diff --git a/bitmapcontainer.go b/bitmapcontainer.go index 416da732..bdd2b2cd 100644 --- a/bitmapcontainer.go +++ b/bitmapcontainer.go @@ -938,9 +938,7 @@ func (bc *bitmapContainer) ixorRun16(value2 *runContainer16) container { func (bc *bitmapContainer) ixorBitmap(value2 *bitmapContainer) container { newCardinality := int(popcntXorSlice(bc.bitmap, value2.bitmap)) if newCardinality > arrayDefaultMaxSize { - for k := 0; k < len(bc.bitmap); k++ { - bc.bitmap[k] = bc.bitmap[k] ^ value2.bitmap[k] - } + xorSliceInPlace(bc.bitmap, value2.bitmap) bc.cardinality = newCardinality return bc } diff --git a/bitmapcontainer_bench_test.go b/bitmapcontainer_bench_test.go index 530772ac..dc6e1799 100644 --- a/bitmapcontainer_bench_test.go +++ b/bitmapcontainer_bench_test.go @@ -68,3 +68,115 @@ func BenchmarkParOrBitmapContainers(b *testing.B) { } } } + +const bitmapXorDenseWord = uint64(0xaaaaaaaaaaaaaaaa) + +// newBitmapContainerXorFixture creates two bitmap containers with a chosen +// symmetric-difference cardinality. Both inputs remain dense bitmap containers. +func newBitmapContainerXorFixture(xorCardinality int) (*bitmapContainer, *bitmapContainer) { + left := newBitmapContainer() + right := newBitmapContainer() + for i := range left.bitmap { + left.bitmap[i] = bitmapXorDenseWord + right.bitmap[i] = bitmapXorDenseWord + } + for i := 0; i < xorCardinality; i++ { + right.bitmap[i/64] ^= uint64(1) << (i % 64) + } + left.computeCardinality() + right.computeCardinality() + return left, right +} + +func newDenseXorBitmapFixture(containers int) (*Bitmap, *Bitmap) { + leftWords := make([]uint64, containers*bitmapContainerSize) + rightWords := make([]uint64, len(leftWords)) + for i := range leftWords { + leftWords[i] = bitmapXorDenseWord + rightWords[i] = ^bitmapXorDenseWord + } + return FromDense(leftWords, true), FromDense(rightWords, true) +} + +// BenchmarkBitmapXorDenseContainers measures Bitmap.Xor with matching dense +// keys. Each operation alternates between a half-full and full bitmap result, +// so every matching pair takes ixorBitmap's bitmap-result branch. +func BenchmarkBitmapXorDenseContainers(b *testing.B) { + for _, benchmark := range []struct { + name string + containers int + }{ + {name: "one", containers: 1}, + {name: "fifty", containers: 50}, + {name: "two-fifty-six", containers: 256}, + } { + b.Run(benchmark.name, func(b *testing.B) { + left, right := newDenseXorBitmapFixture(benchmark.containers) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + left.Xor(right) + } + b.StopTimer() + + want := uint64(benchmark.containers * maxCapacity / 2) + if b.N%2 != 0 { + want *= 2 + } + if got := left.GetCardinality(); got != want { + b.Fatalf("unexpected cardinality: got %d, want %d", got, want) + } + }) + } +} + +func newDenseXorThresholdFixture() (*Bitmap, *Bitmap) { + leftWords := make([]uint64, bitmapContainerSize) + rightWords := make([]uint64, bitmapContainerSize) + for i := range leftWords { + leftWords[i] = bitmapXorDenseWord + rightWords[i] = bitmapXorDenseWord + } + for i := 0; i < arrayDefaultMaxSize; i++ { + rightWords[i/64] ^= uint64(1) << (i % 64) + } + return FromDense(leftWords, true), FromDense(rightWords, true) +} + +// BenchmarkBitmapXorDenseThresholdBatch measures independent ordinary Xor +// calls at the array-result threshold. It resets receivers outside the timed +// region and batches calls to reduce timer noise without changing the path. +func BenchmarkBitmapXorDenseThresholdBatch(b *testing.B) { + const batchSize = 16 + + lefts := make([]*Bitmap, batchSize) + originals := make([]container, batchSize) + _, right := newDenseXorThresholdFixture() + for i := range lefts { + lefts[i], _ = newDenseXorThresholdFixture() + originals[i] = lefts[i].highlowcontainer.getContainerAtIndex(0) + } + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + for _, left := range lefts { + left.Xor(right) + } + b.StopTimer() + for i, left := range lefts { + left.highlowcontainer.setContainerAtIndex(0, originals[i]) + } + b.StartTimer() + } + b.StopTimer() + + lefts[0].Xor(right) + if got := lefts[0].GetCardinality(); got != arrayDefaultMaxSize { + b.Fatalf("unexpected cardinality: got %d, want %d", got, arrayDefaultMaxSize) + } + if _, ok := lefts[0].highlowcontainer.getContainerAtIndex(0).(*arrayContainer); !ok { + b.Fatal("expected an array result") + } +} diff --git a/bitmapcontainer_test.go b/bitmapcontainer_test.go index b74fb5c6..952522c8 100644 --- a/bitmapcontainer_test.go +++ b/bitmapcontainer_test.go @@ -639,3 +639,64 @@ func TestBitmapContainerFillLeastSignificant16bitsProperties(t *testing.T) { runTest(t, vals, 0x55550000) }) } + +func TestBitmapContainerIXorBitmapDense(t *testing.T) { + left, right := newBitmapContainerXorFixture(maxCapacity) + + result := left.ixorBitmap(right) + actual, ok := result.(*bitmapContainer) + require.True(t, ok) + assert.Equal(t, maxCapacity, actual.cardinality) + for _, word := range actual.bitmap { + assert.Equal(t, ^uint64(0), word) + } + assert.NoError(t, actual.validate()) +} + +func TestBitmapContainerIXorBitmapThreshold(t *testing.T) { + left, right := newBitmapContainerXorFixture(arrayDefaultMaxSize) + original := left.clone().(*bitmapContainer) + + result := left.ixorBitmap(right) + actual, ok := result.(*arrayContainer) + require.True(t, ok) + require.Len(t, actual.content, arrayDefaultMaxSize) + for i, value := range actual.content { + assert.Equal(t, uint16(i), value) + } + assert.True(t, original.equals(left)) + assert.NoError(t, actual.validate()) +} + +func TestBitmapXorDenseThreshold(t *testing.T) { + left, right := newDenseXorThresholdFixture() + + left.Xor(right) + actual, ok := left.highlowcontainer.getContainerAtIndex(0).(*arrayContainer) + require.True(t, ok) + require.Len(t, actual.content, arrayDefaultMaxSize) + for i, value := range actual.content { + assert.Equal(t, uint16(i), value) + } + assert.NoError(t, left.Validate()) +} + +func TestBitmapXorDenseCopyOnWrite(t *testing.T) { + const containers = 2 + left, right := newDenseXorBitmapFixture(containers) + original := left.Clone() + expectedWords := make([]uint64, containers*bitmapContainerSize) + for i := range expectedWords { + expectedWords[i] = ^uint64(0) + } + expected := FromDense(expectedWords, true) + + left.SetCopyOnWrite(true) + alias := left.Clone() + alias.Xor(right) + + assert.True(t, original.Equals(left)) + assert.True(t, expected.Equals(alias)) + assert.NoError(t, left.Validate()) + assert.NoError(t, alias.Validate()) +} diff --git a/popcnt_avx2_amd64.go b/popcnt_avx2_amd64.go index 25b6065c..b44175bf 100644 --- a/popcnt_avx2_amd64.go +++ b/popcnt_avx2_amd64.go @@ -27,6 +27,9 @@ func _popcntOrSliceAVX2(s, m []uint64) uint64 //go:noescape func _popcntXorSliceAVX2(s, m []uint64) uint64 +//go:noescape +func _xorSliceInPlaceAVX2(s, m []uint64) + // useAVX2 selects the AVX2 assembly implementations when the running CPU // supports AVX2. It is evaluated once at package initialization. var useAVX2 = _hasAVX2() @@ -65,3 +68,11 @@ func popcntXorSlice(s, m []uint64) uint64 { } return popcntXorSliceGo(s, m) } + +func xorSliceInPlace(s, m []uint64) { + if useAVX2 { + _xorSliceInPlaceAVX2(s, m) + return + } + xorSliceInPlaceGo(s, m) +} diff --git a/popcnt_avx2_amd64.s b/popcnt_avx2_amd64.s index 1c61a237..e5fca88c 100644 --- a/popcnt_avx2_amd64.s +++ b/popcnt_avx2_amd64.s @@ -316,6 +316,70 @@ maskdone: MOVQ AX, ret+48(FP) RET +// func _xorSliceInPlaceAVX2(s, m []uint64) +// XORs m into s. Bitmap containers contain 1024 words, so the main loop handles +// four 256-bit vectors (128 bytes, 16 words) per iteration. A vector tail and +// scalar tail keep the helper correct for the shorter slices used by tests. +TEXT ยท_xorSliceInPlaceAVX2(SB), NOSPLIT, $0-48 + MOVQ s_base+0(FP), SI + MOVQ m_base+24(FP), DI + MOVQ s_len+8(FP), CX + MOVQ CX, R8 + SHRQ $4, R8 // 16 words per unrolled iteration + TESTQ R8, R8 + JZ xorstoretail +xorstoreloop: + VMOVDQU 0(SI), Y0 + VMOVDQU 32(SI), Y1 + VMOVDQU 64(SI), Y2 + VMOVDQU 96(SI), Y3 + VMOVDQU 0(DI), Y4 + VMOVDQU 32(DI), Y5 + VMOVDQU 64(DI), Y6 + VMOVDQU 96(DI), Y7 + VPXOR Y4, Y0, Y0 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VMOVDQU Y0, 0(SI) + VMOVDQU Y1, 32(SI) + VMOVDQU Y2, 64(SI) + VMOVDQU Y3, 96(SI) + ADDQ $128, SI + ADDQ $128, DI + DECQ R8 + JNZ xorstoreloop +xorstoretail: + ANDQ $15, CX + MOVQ CX, R8 + SHRQ $2, R8 // 4 words per vector tail iteration + TESTQ R8, R8 + JZ xorstorescalartail +xorstorevectorloop: + VMOVDQU (SI), Y0 + VMOVDQU (DI), Y1 + VPXOR Y1, Y0, Y0 + VMOVDQU Y0, (SI) + ADDQ $32, SI + ADDQ $32, DI + DECQ R8 + JNZ xorstorevectorloop +xorstorescalartail: + ANDQ $3, CX + TESTQ CX, CX + JZ xorstoredone +xorstorescalarloop: + MOVQ (SI), AX + XORQ (DI), AX + MOVQ AX, (SI) + ADDQ $8, SI + ADDQ $8, DI + DECQ CX + JNZ xorstorescalarloop +xorstoredone: + VZEROUPPER + RET + // func _hasAVX2() bool // Reports whether the CPU supports AVX2 and the OS has enabled the wide (YMM) // register state. All three checks must pass; otherwise the Go wrappers fall diff --git a/popcnt_avx2_amd64_test.go b/popcnt_avx2_amd64_test.go index c0e91b0f..5db0113b 100644 --- a/popcnt_avx2_amd64_test.go +++ b/popcnt_avx2_amd64_test.go @@ -80,6 +80,30 @@ func BenchmarkPopcntSlice1024Go(b *testing.B) { _ = sink } +// BenchmarkBitmapXorDenseScalarFallback measures the ordinary dense Xor path +// with AVX2 disabled, including the portable count and store fallbacks. +func BenchmarkBitmapXorDenseScalarFallback(b *testing.B) { + saved := useAVX2 + useAVX2 = false + defer func() { useAVX2 = saved }() + + left, right := newDenseXorBitmapFixture(50) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + left.Xor(right) + } + b.StopTimer() + + want := uint64(50 * maxCapacity / 2) + if b.N%2 != 0 { + want *= 2 + } + if got := left.GetCardinality(); got != want { + b.Fatalf("unexpected cardinality: got %d, want %d", got, want) + } +} + func TestAVX2PopcntDispatch(t *testing.T) { // Verify the runtime dispatch wrappers agree with the Go reference both // when AVX2 is selected and when the scalar fallback is forced. @@ -127,3 +151,61 @@ func TestAVX2PopcntDifferential(t *testing.T) { } } } + +func TestAVX2XorSliceDispatch(t *testing.T) { + saved := useAVX2 + defer func() { useAVX2 = saved }() + + r := rand.New(rand.NewSource(99)) + for _, on := range []bool{false, true} { + if on && !saved { + continue + } + useAVX2 = on + for _, n := range avx2TestLengths { + input := randomUint64Slice(r, n) + mask := randomUint64Slice(r, n) + want := make([]uint64, len(input)) + copy(want, input) + xorSliceInPlaceGo(want, mask) + + xorSliceInPlace(input, mask) + assert.Equalf(t, want, input, "xorSliceInPlace avx2=%v len=%d", on, n) + + alias := randomUint64Slice(r, n) + xorSliceInPlace(alias, alias) + assert.Equalf(t, make([]uint64, n), alias, "xorSliceInPlace alias avx2=%v len=%d", on, n) + } + } +} + +func TestAVX2XorSliceDifferential(t *testing.T) { + if !useAVX2 { + t.Skip("AVX2 not available on this CPU") + } + r := rand.New(rand.NewSource(100)) + for _, n := range avx2TestLengths { + for iter := 0; iter < 64; iter++ { + input := randomUint64Slice(r, n) + mask := randomUint64Slice(r, n) + want := make([]uint64, len(input)) + copy(want, input) + xorSliceInPlaceGo(want, mask) + + _xorSliceInPlaceAVX2(input, mask) + assert.Equalf(t, want, input, "_xorSliceInPlaceAVX2 len=%d", n) + } + } +} + +func TestBitmapXorDenseScalarFallback(t *testing.T) { + saved := useAVX2 + useAVX2 = false + defer func() { useAVX2 = saved }() + + left, right := newDenseXorBitmapFixture(2) + left.Xor(right) + + assert.Equal(t, uint64(2*maxCapacity), left.GetCardinality()) + assert.NoError(t, left.Validate()) +} diff --git a/popcnt_generic.go b/popcnt_generic.go index 63235fe4..33986fa9 100644 --- a/popcnt_generic.go +++ b/popcnt_generic.go @@ -22,3 +22,7 @@ func popcntOrSlice(s, m []uint64) uint64 { func popcntXorSlice(s, m []uint64) uint64 { return popcntXorSliceGo(s, m) } + +func xorSliceInPlace(s, m []uint64) { + xorSliceInPlaceGo(s, m) +} diff --git a/popcnt_neon_arm64.go b/popcnt_neon_arm64.go index c348688f..a43a6cf6 100644 --- a/popcnt_neon_arm64.go +++ b/popcnt_neon_arm64.go @@ -63,3 +63,7 @@ func popcntXorSlice(s, m []uint64) uint64 { } return popcntXorSliceGo(s, m) } + +func xorSliceInPlace(s, m []uint64) { + xorSliceInPlaceGo(s, m) +} diff --git a/popcnt_slices.go b/popcnt_slices.go index 4afdd713..fb793359 100644 --- a/popcnt_slices.go +++ b/popcnt_slices.go @@ -41,3 +41,9 @@ func popcntXorSliceGo(s, m []uint64) uint64 { } return cnt } + +func xorSliceInPlaceGo(s, m []uint64) { + for i := range s { + s[i] ^= m[i] + } +}