diff --git a/src/Communicate/BufferHandler.h b/src/Communicate/BufferHandler.h index 9975a3631..720e24f0c 100644 --- a/src/Communicate/BufferHandler.h +++ b/src/Communicate/BufferHandler.h @@ -154,8 +154,8 @@ namespace ippl { buffer_type reallocateLargestFreeBuffer(size_type requiredSize); buffer_type allocateNewBuffer(size_type requiredSize); - size_type usedSize_m; ///< Total size of all allocated buffers - size_type freeSize_m; ///< Total size of all free buffers + size_type usedSize_m = 0; ///< Total size of all allocated buffers + size_type freeSize_m = 0; ///< Total size of all free buffers protected: buffer_set_type used_buffers{ diff --git a/src/Field/HaloCells.hpp b/src/Field/HaloCells.hpp index 542e067fc..0f15339af 100644 --- a/src/Field/HaloCells.hpp +++ b/src/Field/HaloCells.hpp @@ -12,6 +12,79 @@ namespace ippl { namespace detail { + // These kernels are instantiated in multiple translation units. Named functors avoid + // NVCC's translation-unit-local extended-lambda copy/delete helper state. + template + struct HaloPackFunctor { + Subview subview; + Buffer buffer; + + KOKKOS_INLINE_FUNCTION void operator()(const IndexArray& args) const { + int l = 0; + + for (unsigned d1 = 0; d1 < Dim; ++d1) { + int next = args[d1]; + for (unsigned d2 = 0; d2 < d1; ++d2) { + next *= subview.extent(d2); + } + l += next; + } + + buffer(l) = apply(subview, args); + } + }; + + template + struct HaloUnpackFunctor { + Subview subview; + Buffer buffer; + Op op; + + KOKKOS_INLINE_FUNCTION void operator()(const IndexArray& args) const { + int l = 0; + + for (unsigned d1 = 0; d1 < Dim; ++d1) { + int next = args[d1]; + for (unsigned d2 = 0; d2 < d1; ++d2) { + next *= subview.extent(d2); + } + l += next; + } + + op(apply(subview, args), buffer(l)); + } + }; + + template + struct HaloPeriodicFunctor { + View view; + Op op; + unsigned dimension; + int extent; + int nghost; + + KOKKOS_INLINE_FUNCTION void operator()(IndexArray coords) const { + // The ghosts are filled starting from the inside of the domain proceeding outwards + // for both lower and upper faces. The extra brackets and explicit mention + + // nghost + i + coords[dimension] += nghost; + auto&& left = apply(view, coords); + + // N - nghost - i + coords[dimension] = extent - coords[dimension]; + auto&& right = apply(view, coords); + + // nghost - 1 - i + coords[dimension] += 2 * nghost - 1 - extent; + op(apply(view, coords), right); + + // N - (nghost - 1 - i) = N - (nghost - 1) + i + coords[dimension] = extent - coords[dimension]; + op(apply(view, coords), left); + } + }; + template HaloCells::HaloCells() {} @@ -171,9 +244,8 @@ namespace ippl { auto subview = makeSubview(view, range); auto& buffer = fd.buffer; - - size_t size = subview.size(); - nsends = size; + size_t size = subview.size(); + nsends = size; if (buffer.size() < size) { int overalloc = Comm->getDefaultOverallocation(); Kokkos::realloc(buffer, size * overalloc); @@ -181,21 +253,11 @@ namespace ippl { using index_array_type = typename RangePolicy::index_array_type; - ippl::parallel_for( - "HaloCells::pack()", getRangePolicy(subview), - KOKKOS_LAMBDA(const index_array_type& args) { - int l = 0; - - for (unsigned d1 = 0; d1 < Dim; d1++) { - int next = args[d1]; - for (unsigned d2 = 0; d2 < d1; d2++) { - next *= subview.extent(d2); - } - l += next; - } - - buffer(l) = apply(subview, args); - }); + using buffer_view_type = typename databuffer_type::view_type; + using functor_type = + HaloPackFunctor; + ippl::parallel_for("HaloCells::pack()", getRangePolicy(subview), + functor_type{subview, buffer}); Kokkos::fence(); } @@ -212,21 +274,10 @@ namespace ippl { using index_array_type = typename RangePolicy::index_array_type; - ippl::parallel_for( - "HaloCells::unpack()", getRangePolicy(subview), - KOKKOS_LAMBDA(const index_array_type& args) { - int l = 0; - - for (unsigned d1 = 0; d1 < Dim; d1++) { - int next = args[d1]; - for (unsigned d2 = 0; d2 < d1; d2++) { - next *= subview.extent(d2); - } - l += next; - } - - op(apply(subview, args), buffer(l)); - }); + using functor_type = + HaloUnpackFunctor; + ippl::parallel_for("HaloCells::unpack()", getRangePolicy(subview), + functor_type{subview, buffer, op}); Kokkos::fence(); } @@ -271,30 +322,10 @@ namespace ippl { using index_array_type = typename RangePolicy::index_array_type; - ippl::parallel_for( - "applyPeriodicSerialDim", createRangePolicy(begin, end), - KOKKOS_LAMBDA(index_array_type & coords) { - // The ghosts are filled starting from the inside - // of the domain proceeding outwards for both lower - // and upper faces. The extra brackets and explicit - // mention - - // nghost + i - coords[d] += nghost; - auto&& left = apply(view, coords); - - // N - nghost - i - coords[d] = N - coords[d]; - auto&& right = apply(view, coords); - - // nghost - 1 - i - coords[d] += 2 * nghost - 1 - N; - op(apply(view, coords), right); - - // N - (nghost - 1 - i) = N - (nghost - 1) + i - coords[d] = N - coords[d]; - op(apply(view, coords), left); - }); + using functor_type = HaloPeriodicFunctor; + ippl::parallel_for("applyPeriodicSerialDim", + createRangePolicy(begin, end), + functor_type{view, op, d, N, nghost}); Kokkos::fence(); }