diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 238ca7235f..4e4736bb35 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -2254,7 +2254,8 @@ subroutine atm_srk3(domain, dt, itimestep, exchange_halo_group) edgeThreadStart(thread), edgeThreadEnd(thread), & cellSolveThreadStart(thread), cellSolveThreadEnd(thread), & vertexSolveThreadStart(thread), vertexSolveThreadEnd(thread), & - edgeSolveThreadStart(thread), edgeSolveThreadEnd(thread)) + edgeSolveThreadStart(thread), edgeSolveThreadEnd(thread), & + domain, config_gpu_aware_mpi, exchange_halo_group) end do !$OMP END PARALLEL DO @@ -5787,7 +5788,8 @@ end subroutine atm_advance_scalars_mono_work subroutine atm_compute_dyn_tend(tend, tend_physics, state, diag, mesh, diag_physics, configs, nVertLevels, rk_step, dynamics_substep, dt, & cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd, & - cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd) + cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd, & + domain, config_gpu_aware_mpi, exchange_halo_group) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Compute height and normal wind tendencies, as well as diagnostic variables ! @@ -5820,6 +5822,9 @@ subroutine atm_compute_dyn_tend(tend, tend_physics, state, diag, mesh, diag_phys real (kind=RKIND), intent(in) :: dt integer, intent(in) :: cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd integer, intent(in) :: cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd + type (domain_type), intent(inout) :: domain + logical, intent(in) :: config_gpu_aware_mpi + procedure (halo_exchange_routine) :: exchange_halo_group ! ! Local variables @@ -6092,7 +6097,8 @@ subroutine atm_compute_dyn_tend(tend, tend_physics, state, diag, mesh, diag_phys config_surface_heat_flux, config_surface_moisture_flux, config_surface_drag_coefficient, & rthdynten, ustm, hfx, qfx, & cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd, & - cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd) + cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd, & + domain, config_gpu_aware_mpi, exchange_halo_group) if (nopbl) then deallocate(ustm) @@ -6127,7 +6133,8 @@ subroutine atm_compute_dyn_tend_work(nCells, nEdges, nVertices, nVertLevels_dumm config_surface_heat_flux, config_surface_moisture_flux, config_surface_drag_coefficient, & rthdynten, ustm, hfx, qfx, & cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd, & - cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd) + cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd, & + domain, config_gpu_aware_mpi, exchange_halo_group) use mpas_atm_dimensions @@ -6266,6 +6273,9 @@ subroutine atm_compute_dyn_tend_work(nCells, nEdges, nVertices, nVertLevels_dumm integer, intent(in) :: cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd integer, intent(in) :: cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd + type (domain_type), intent(inout) :: domain + logical, intent(in) :: config_gpu_aware_mpi + procedure (halo_exchange_routine) :: exchange_halo_group ! ! Local variables @@ -6385,6 +6395,10 @@ subroutine atm_compute_dyn_tend_work(nCells, nEdges, nVertices, nVertLevels_dumm cellStart, cellEnd, nEdgesOnCell, edgesOnCell, cellsOnEdge, & nCells, nEdges, nVertLevels, maxEdges, num_scalars ) + !$acc update self(eddy_visc_horz, eddy_visc_vert) if (.not. config_gpu_aware_mpi) + call exchange_halo_group(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert', config_gpu_aware_mpi) + !$acc update device(eddy_visc_horz, eddy_visc_vert) if (.not. config_gpu_aware_mpi) + end if if (config_mpas_cam_coef > 0.0) then diff --git a/src/core_atmosphere/mpas_atm_halos.F b/src/core_atmosphere/mpas_atm_halos.F index 983c529673..0e92c2fae3 100644 --- a/src/core_atmosphere/mpas_atm_halos.F +++ b/src/core_atmosphere/mpas_atm_halos.F @@ -122,6 +122,9 @@ subroutine atm_build_halo_groups(domain, ierr) call mpas_dmpar_exch_group_add_field(domain, 'dynamics:theta_m,pressure_p,rtheta_p', 'rtheta_p', & timeLevel=1, haloLayers=(/1,2/)) + call mpas_dmpar_exch_group_create(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert') + call mpas_dmpar_exch_group_add_field(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert', 'eddy_visc_horz', timeLevel=1, haloLayers=(/1,2/)) + call mpas_dmpar_exch_group_add_field(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert', 'eddy_visc_vert', timeLevel=1, haloLayers=(/1,2/)) call mpas_dmpar_exch_group_create(domain, 'dynamics:exner') call mpas_dmpar_exch_group_add_field(domain, 'dynamics:exner', 'exner', timeLevel=1, haloLayers=(/1,2/)) @@ -243,6 +246,10 @@ subroutine atm_build_halo_groups(domain, ierr) timeLevel=1, haloLayers=(/1,2/)) call mpas_halo_exch_group_complete(domain, 'dynamics:theta_m,pressure_p,rtheta_p') + call mpas_halo_exch_group_create(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert') + call mpas_halo_exch_group_add_field(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert', 'eddy_visc_horz', timeLevel=1, haloLayers=(/1,2/)) + call mpas_halo_exch_group_add_field(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert', 'eddy_visc_vert', timeLevel=1, haloLayers=(/1,2/)) + call mpas_halo_exch_group_complete(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert') call mpas_halo_exch_group_create(domain, 'dynamics:exner') call mpas_halo_exch_group_add_field(domain, 'dynamics:exner', 'exner', timeLevel=1, haloLayers=(/1,2/)) @@ -367,6 +374,7 @@ subroutine atm_destroy_halo_groups(domain, ierr) call mpas_dmpar_exch_group_destroy(domain, 'dynamics:w,pv_edge,rho_edge,scalars') call mpas_dmpar_exch_group_destroy(domain, 'dynamics:theta_m,pressure_p,rtheta_p') + call mpas_dmpar_exch_group_destroy(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert') call mpas_dmpar_exch_group_destroy(domain, 'dynamics:exner') call mpas_dmpar_exch_group_destroy(domain, 'dynamics:tend_u') call mpas_dmpar_exch_group_destroy(domain, 'dynamics:rho_pp') @@ -403,6 +411,7 @@ subroutine atm_destroy_halo_groups(domain, ierr) call mpas_halo_exch_group_destroy(domain, 'dynamics:w,pv_edge,rho_edge,scalars') call mpas_halo_exch_group_destroy(domain, 'dynamics:theta_m,pressure_p,rtheta_p') + call mpas_halo_exch_group_destroy(domain, 'dynamics:eddy_visc_horz,eddy_visc_vert') call mpas_halo_exch_group_destroy(domain, 'dynamics:exner') call mpas_halo_exch_group_destroy(domain, 'dynamics:tend_u') call mpas_halo_exch_group_destroy(domain, 'dynamics:rho_pp')