From d72df903cf3b783515903e7bb65543a66b90b4f2 Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Tue, 14 Apr 2026 20:37:56 -0600 Subject: [PATCH 1/2] Add new halo exchange group in mpas_atm_halos to exchange eddy viscosity fields This commit introduces a new halo exchange group in mpas_atm_halos in order to perform halo exchanges for the two eddy viscosity fields relevant to LES model runs. This group is added to both mpas_dmpar and mpas_halo halo exchange methods in mpas_atm_halos. The halo exchanges of eddy_visc_horz and eddy_visc_vert appear to fix the parallel reproducibility issues found in some LES runs, when using 3d_smagorinsky model option. The addition of this halo exchange will appear in the subsequent commit. --- src/core_atmosphere/mpas_atm_halos.F | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/core_atmosphere/mpas_atm_halos.F b/src/core_atmosphere/mpas_atm_halos.F index 983c529673..0e92c2fae3 100644 --- a/src/core_atmosphere/mpas_atm_halos.F +++ b/src/core_atmosphere/mpas_atm_halos.F @@ -122,6 +122,9 @@ subroutine atm_build_halo_groups(domain, ierr) call mpas_dmpar_exch_group_add_field(domain, 'dynamics:theta_m,pressure_p,rtheta_p', 'rtheta_p', & timeLevel=1, haloLayers=(/1,2/)) + call mpas_dmpar_exch_group_create(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert') + call mpas_dmpar_exch_group_add_field(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert', 'eddy_visc_horz', timeLevel=1, haloLayers=(/1,2/)) + call mpas_dmpar_exch_group_add_field(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert', 'eddy_visc_vert', timeLevel=1, haloLayers=(/1,2/)) call mpas_dmpar_exch_group_create(domain, 'dynamics:exner') call mpas_dmpar_exch_group_add_field(domain, 'dynamics:exner', 'exner', timeLevel=1, haloLayers=(/1,2/)) @@ -243,6 +246,10 @@ subroutine atm_build_halo_groups(domain, ierr) timeLevel=1, haloLayers=(/1,2/)) call mpas_halo_exch_group_complete(domain, 'dynamics:theta_m,pressure_p,rtheta_p') + call mpas_halo_exch_group_create(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert') + call mpas_halo_exch_group_add_field(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert', 'eddy_visc_horz', timeLevel=1, haloLayers=(/1,2/)) + call mpas_halo_exch_group_add_field(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert', 'eddy_visc_vert', timeLevel=1, haloLayers=(/1,2/)) + call mpas_halo_exch_group_complete(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert') call mpas_halo_exch_group_create(domain, 'dynamics:exner') call mpas_halo_exch_group_add_field(domain, 'dynamics:exner', 'exner', timeLevel=1, haloLayers=(/1,2/)) @@ -367,6 +374,7 @@ subroutine atm_destroy_halo_groups(domain, ierr) call mpas_dmpar_exch_group_destroy(domain, 'dynamics:w,pv_edge,rho_edge,scalars') call mpas_dmpar_exch_group_destroy(domain, 'dynamics:theta_m,pressure_p,rtheta_p') + call mpas_dmpar_exch_group_destroy(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert') call mpas_dmpar_exch_group_destroy(domain, 'dynamics:exner') call mpas_dmpar_exch_group_destroy(domain, 'dynamics:tend_u') call mpas_dmpar_exch_group_destroy(domain, 'dynamics:rho_pp') @@ -403,6 +411,7 @@ subroutine atm_destroy_halo_groups(domain, ierr) call mpas_halo_exch_group_destroy(domain, 'dynamics:w,pv_edge,rho_edge,scalars') call mpas_halo_exch_group_destroy(domain, 'dynamics:theta_m,pressure_p,rtheta_p') + call mpas_halo_exch_group_destroy(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert') call mpas_halo_exch_group_destroy(domain, 'dynamics:exner') call mpas_halo_exch_group_destroy(domain, 'dynamics:tend_u') call mpas_halo_exch_group_destroy(domain, 'dynamics:rho_pp') From a6bc145d38bcd0b3afb62933ce62354e83571b7a Mon Sep 17 00:00:00 2001 From: Abishek Gopal Date: Tue, 14 Apr 2026 20:50:53 -0600 Subject: [PATCH 2/2] Adding halo exhanges for LES eddy viscosity fields in atm_compute_dyn_tend This commit introduces a new halo exchange for the two eddy viscosity fields relevant to LES runs, eddy_visc_horz and eddy_visc_vert, immediately after the call to les_models in atm_compute_dyn_tend. This change also requires the addition of new arguments to the atm_compute_dyn_tend and atm_compute_dyn_tend_work subroutines. This changes is required to fix parallel reproducibility issues observed in LES model runs when config_les_model= 3d_smagorinsky, affecting both CPU and GPU runs. By trial and error, it was observed that performing halo exchanges of the two fieds seemed to fix this error. The exact source of this issue has not been understood yet and would need to be revisited. --- .../dynamics/mpas_atm_time_integration.F | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 238ca7235f..4e4736bb35 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -2254,7 +2254,8 @@ subroutine atm_srk3(domain, dt, itimestep, exchange_halo_group) edgeThreadStart(thread), edgeThreadEnd(thread), & cellSolveThreadStart(thread), cellSolveThreadEnd(thread), & vertexSolveThreadStart(thread), vertexSolveThreadEnd(thread), & - edgeSolveThreadStart(thread), edgeSolveThreadEnd(thread)) + edgeSolveThreadStart(thread), edgeSolveThreadEnd(thread), & + domain, config_gpu_aware_mpi, exchange_halo_group) end do !$OMP END PARALLEL DO @@ -5787,7 +5788,8 @@ end subroutine atm_advance_scalars_mono_work subroutine atm_compute_dyn_tend(tend, tend_physics, state, diag, mesh, diag_physics, configs, nVertLevels, rk_step, dynamics_substep, dt, & cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd, & - cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd) + cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd, & + domain, config_gpu_aware_mpi, exchange_halo_group) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Compute height and normal wind tendencies, as well as diagnostic variables ! @@ -5820,6 +5822,9 @@ subroutine atm_compute_dyn_tend(tend, tend_physics, state, diag, mesh, diag_phys real (kind=RKIND), intent(in) :: dt integer, intent(in) :: cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd integer, intent(in) :: cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd + type (domain_type), intent(inout) :: domain + logical, intent(in) :: config_gpu_aware_mpi + procedure (halo_exchange_routine) :: exchange_halo_group ! ! Local variables @@ -6092,7 +6097,8 @@ subroutine atm_compute_dyn_tend(tend, tend_physics, state, diag, mesh, diag_phys config_surface_heat_flux, config_surface_moisture_flux, config_surface_drag_coefficient, & rthdynten, ustm, hfx, qfx, & cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd, & - cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd) + cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd, & + domain, config_gpu_aware_mpi, exchange_halo_group) if (nopbl) then deallocate(ustm) @@ -6127,7 +6133,8 @@ subroutine atm_compute_dyn_tend_work(nCells, nEdges, nVertices, nVertLevels_dumm config_surface_heat_flux, config_surface_moisture_flux, config_surface_drag_coefficient, & rthdynten, ustm, hfx, qfx, & cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd, & - cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd) + cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd, & + domain, config_gpu_aware_mpi, exchange_halo_group) use mpas_atm_dimensions @@ -6266,6 +6273,9 @@ subroutine atm_compute_dyn_tend_work(nCells, nEdges, nVertices, nVertLevels_dumm integer, intent(in) :: cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd integer, intent(in) :: cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd + type (domain_type), intent(inout) :: domain + logical, intent(in) :: config_gpu_aware_mpi + procedure (halo_exchange_routine) :: exchange_halo_group ! ! Local variables @@ -6385,6 +6395,10 @@ subroutine atm_compute_dyn_tend_work(nCells, nEdges, nVertices, nVertLevels_dumm cellStart, cellEnd, nEdgesOnCell, edgesOnCell, cellsOnEdge, & nCells, nEdges, nVertLevels, maxEdges, num_scalars ) + !$acc update self(eddy_visc_horz, eddy_visc_vert) if (.not. config_gpu_aware_mpi) + call exchange_halo_group(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert', config_gpu_aware_mpi) + !$acc update device(eddy_visc_horz, eddy_visc_vert) if (.not. config_gpu_aware_mpi) + end if if (config_mpas_cam_coef > 0.0) then