diff --git a/include/xsimd/arch/xsimd_avx.hpp b/include/xsimd/arch/xsimd_avx.hpp index 7d745f140..e048e4d37 100644 --- a/include/xsimd/arch/xsimd_avx.hpp +++ b/include/xsimd/arch/xsimd_avx.hpp @@ -16,6 +16,7 @@ #include "../types/xsimd_avx_register.hpp" #include "../types/xsimd_batch_constant.hpp" +#include #include #include #include @@ -1041,6 +1042,8 @@ namespace xsimd // exactly the lower 128-bit half: one plain load, upper lanes zero XSIMD_IF_CONSTEXPR(mask.prefix() == half_size) { + // cross-check the plain move via countr_one/countl_zero (independent of prefix()) + assert(mask.countr_one() >= half_size && mask.countl_zero() >= half_size && "lower half fully active, upper empty"); return detail::zero_extend_lo(half_batch::load(mem, Mode {})); } // lower 128-bit half: stay in the value domain so the half kernel can @@ -1051,9 +1054,21 @@ namespace xsimd const auto lo = load_masked(mem, mlo, convert {}, Mode {}, half_arch {}); return detail::zero_extend_lo(lo); } + // prefix crossing the 128-bit boundary: plain lower half + + // prefix-masked upper half (mirrors the store side) + else XSIMD_IF_CONSTEXPR(mask.prefix() > half_size && mask.prefix() < batch::size) + { + // the plain lower-half load reads every lower lane, so they must all be active + assert(mask.countr_one() >= half_size && "plain lower-half load needs the lower half fully active"); + const half_batch lo = half_batch::load(mem, Mode {}); + constexpr auto mhi = ::xsimd::detail::upper_half(mask); + const half_batch hi = load_masked(mem + half_size, mhi, convert {}, Mode {}, half_arch {}); + return detail::merge_sse(lo.data, hi.data); + } // exactly the upper 128-bit half: one plain load into the upper lanes else XSIMD_IF_CONSTEXPR(mask.suffix() == half_size) { + assert(mask.countl_one() >= half_size && mask.countr_zero() >= half_size && "upper half fully active, lower empty"); return detail::zero_extend(half_batch::load(mem + half_size, Mode {})); } // upper 128-bit half @@ -1103,6 +1118,9 @@ namespace xsimd // exactly the lower 128-bit half: one plain store XSIMD_IF_CONSTEXPR(mask.prefix() == half_size) { + // a plain store writes every lower lane and no upper lane, so the mask + // must have the lower half fully active and the upper half empty + assert(mask.countr_one() >= half_size && mask.countl_zero() >= half_size && "lower half fully active, upper empty"); const half_batch lo = detail::lower_half(src); lo.store(mem, Mode {}); } @@ -1110,6 +1128,7 @@ namespace xsimd // upper half. Never emits vmaskmov, which does not store-forward. else XSIMD_IF_CONSTEXPR(mask.prefix() > half_size && mask.prefix() < batch::size) { + assert(mask.countr_one() >= half_size && "plain lower-half store needs the lower half fully active"); const half_batch lo = detail::lower_half(src); lo.store(mem, Mode {}); constexpr auto mhi = ::xsimd::detail::upper_half(mask); @@ -1119,6 +1138,7 @@ namespace xsimd // exactly the upper 128-bit half: one plain store else XSIMD_IF_CONSTEXPR(mask.suffix() == half_size) { + assert(mask.countl_one() >= half_size && mask.countr_zero() >= half_size && "upper half fully active, lower empty"); const half_batch hi = detail::upper_half(src); hi.store(mem + half_size, Mode {}); } @@ -1172,6 +1192,45 @@ namespace xsimd } } + namespace detail + { + // Reinterpret a constant-mask 4/8-byte load/store as same-width float + // and run DstArch's kernel, which lowers prefix/suffix shapes to plain + // moves. Shared by the int/EVEX 128- and 256-bit archs. + template + XSIMD_INLINE batch plain_move_load(T const* mem, batch_bool_constant, convert, Mode) noexcept + { + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "plain-move delegation only supports 4/8-byte lanes"); + using F = std::conditional_t; + // same-width float has the same lane count, so the V... mask pack and the + // memory reinterpret line up one-to-one; wrong here would load the wrong lanes + static_assert(batch::size == batch::size, "same-width float must preserve lane count"); + static_assert(sizeof...(V) == batch::size, "mask pack width must match the batch"); + // the plain-move path emits aligned moves in aligned/stream mode, which fault + // on a misaligned pointer (the old vmaskmov tolerated it) + assert((std::is_same::value || ::xsimd::is_aligned(mem)) && "aligned/stream masked load needs an aligned pointer"); + // qualify: an unqualified call resolves to detail::load_masked (a different + // helper) under MSVC's two-phase lookup; we want the kernel-level overload + return bitwise_cast(batch(::xsimd::kernel::load_masked(reinterpret_cast(mem), batch_bool_constant {}, convert {}, Mode {}, DstArch {}))); + } + + // Re-tag to DstArch so the vector-mask store kernel is used (the AVX + // store is gated off EVEX k-register archs). + template + XSIMD_INLINE void plain_move_store(T* mem, batch const& src, batch_bool_constant, Mode) noexcept + { + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "plain-move delegation only supports 4/8-byte lanes"); + using F = std::conditional_t; + static_assert(batch::size == batch::size, "same-width float must preserve lane count"); + static_assert(sizeof...(V) == batch::size, "mask pack width must match the batch"); + assert((std::is_same::value || ::xsimd::is_aligned(mem)) && "aligned/stream masked store needs an aligned pointer"); + const auto fsrc = bitwise_cast(src); + // qualify: an unqualified call resolves to detail::store_masked (a different + // helper) under MSVC's two-phase lookup; we want the kernel-level overload + ::xsimd::kernel::store_masked(reinterpret_cast(mem), batch(fsrc.data), batch_bool_constant {}, Mode {}, DstArch {}); + } + } + // lt template XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept diff --git a/include/xsimd/arch/xsimd_avx2.hpp b/include/xsimd/arch/xsimd_avx2.hpp index 570709124..50d9e1be8 100644 --- a/include/xsimd/arch/xsimd_avx2.hpp +++ b/include/xsimd/arch/xsimd_avx2.hpp @@ -150,19 +150,14 @@ namespace xsimd _mm256_maskstore_epi64(reinterpret_cast(mem), mask, src); } } - - XSIMD_INLINE __m256i zero_extend(__m128i hi) noexcept - { - return _mm256_insertf128_si256(_mm256_setzero_si256(), hi, 1); - } } - // no half-split shortcut for load; forward to runtime + // Constant masks: prefix/suffix shapes lower to plain moves. template XSIMD_INLINE std::enable_if_t::value && (sizeof(T) == 4 || sizeof(T) == 8), batch> load_masked(T const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept { - return load_masked(mem, mask.as_batch_bool(), convert {}, Mode {}, avx2 {}); + return detail::plain_move_load(mem, mask, convert {}, Mode {}); } template @@ -176,28 +171,7 @@ namespace xsimd typename = std::enable_if_t::value && (sizeof(T) == 4 || sizeof(T) == 8)>> XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept { - constexpr size_t lanes_per_half = batch::size / 2; - using half_batch = ::xsimd::make_sized_batch_t; - using half_arch = typename half_batch::arch_type; - - // lower 128-bit half - XSIMD_IF_CONSTEXPR(mask.countl_zero() >= lanes_per_half) - { - constexpr auto mlo = ::xsimd::detail::lower_half(mask); - const half_batch lo = detail::lower_half(src); - store_masked(mem, lo, mlo, Mode {}, half_arch {}); - } - // upper 128-bit half - else XSIMD_IF_CONSTEXPR(mask.countr_zero() >= lanes_per_half) - { - constexpr auto mhi = ::xsimd::detail::upper_half(mask); - const half_batch hi = detail::upper_half(src); - store_masked(mem + lanes_per_half, hi, mhi, Mode {}, half_arch {}); - } - else - { - detail::maskstore(mem, mask.as_batch(), src); - } + detail::plain_move_store(mem, src, mask, Mode {}); } template diff --git a/include/xsimd/arch/xsimd_avx2_128.hpp b/include/xsimd/arch/xsimd_avx2_128.hpp index aad0bc3c1..e1068ccae 100644 --- a/include/xsimd/arch/xsimd_avx2_128.hpp +++ b/include/xsimd/arch/xsimd_avx2_128.hpp @@ -149,8 +149,7 @@ namespace xsimd { XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask)) { - using F = std::conditional_t; - return bitwise_cast(batch(load_masked(reinterpret_cast(mem), batch_bool_constant {}, convert {}, Mode {}, sse2 {}))); + return detail::plain_move_load(mem, mask, convert {}, Mode {}); } else { @@ -164,8 +163,7 @@ namespace xsimd { XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask)) { - using F = std::conditional_t; - store_masked(reinterpret_cast(mem), bitwise_cast(src), batch_bool_constant {}, Mode {}, sse2 {}); + detail::plain_move_store(mem, src, mask, Mode {}); } else { diff --git a/include/xsimd/arch/xsimd_avx512vl_128.hpp b/include/xsimd/arch/xsimd_avx512vl_128.hpp index def5726da..2df3c45fa 100644 --- a/include/xsimd/arch/xsimd_avx512vl_128.hpp +++ b/include/xsimd/arch/xsimd_avx512vl_128.hpp @@ -307,11 +307,20 @@ namespace xsimd } } + // Constant masks: prefix/suffix shapes lower to plain moves; interior + // masks keep the EVEX path. template XSIMD_INLINE batch load_masked(T const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept { - return detail::maskload128(mem, mask.mask(), Mode {}); + XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask)) + { + return detail::plain_move_load(mem, mask, convert {}, Mode {}); + } + else + { + return detail::maskload128(mem, mask.mask(), Mode {}); + } } template XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept { - detail::maskstore128(mem, src, mask.mask(), Mode {}); + XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask)) + { + detail::plain_move_store(mem, src, mask, Mode {}); + } + else + { + detail::maskstore128(mem, src, mask.mask(), Mode {}); + } } template ::value && (sizeof(T) == 4 || sizeof(T) == 8)>> XSIMD_INLINE batch load_masked(T const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept { - return detail::maskload256(mem, mask.mask(), Mode {}); + // all() reaches here only via the avx512f half-split cascade. + XSIMD_IF_CONSTEXPR(mask.all()) + { + return batch::load(mem, Mode {}); + } + else XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask)) + { + return detail::plain_move_load(mem, mask, convert {}, Mode {}); + } + else + { + return detail::maskload256(mem, mask.mask(), Mode {}); + } } template ::value && (sizeof(T) == 4 || sizeof(T) == 8)>> XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept { - detail::maskstore256(mem, src, mask.mask(), Mode {}); + XSIMD_IF_CONSTEXPR(mask.all()) + { + src.store(mem, Mode {}); + } + else XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask)) + { + detail::plain_move_store(mem, src, mask, Mode {}); + } + else + { + detail::maskstore256(mem, src, mask.mask(), Mode {}); + } } template