Use AVX2 for dense in-place XOR

bitmapcontainer.go

@@ -938,9 +938,7 @@ func (bc *bitmapContainer) ixorRun16(value2 *runContainer16) container {
 func (bc *bitmapContainer) ixorBitmap(value2 *bitmapContainer) container {
 	newCardinality := int(popcntXorSlice(bc.bitmap, value2.bitmap))
 	if newCardinality > arrayDefaultMaxSize {
-		for k := 0; k < len(bc.bitmap); k++ {
-			bc.bitmap[k] = bc.bitmap[k] ^ value2.bitmap[k]
-		}
+		xorSliceInPlace(bc.bitmap, value2.bitmap)
 		bc.cardinality = newCardinality
 		return bc
 	}

bitmapcontainer_bench_test.go

@@ -68,3 +68,115 @@ func BenchmarkParOrBitmapContainers(b *testing.B) {
 		}
 	}
 }
+
+const bitmapXorDenseWord = uint64(0xaaaaaaaaaaaaaaaa)
+
+// newBitmapContainerXorFixture creates two bitmap containers with a chosen
+// symmetric-difference cardinality. Both inputs remain dense bitmap containers.
+func newBitmapContainerXorFixture(xorCardinality int) (*bitmapContainer, *bitmapContainer) {
+	left := newBitmapContainer()
+	right := newBitmapContainer()
+	for i := range left.bitmap {
+		left.bitmap[i] = bitmapXorDenseWord
+		right.bitmap[i] = bitmapXorDenseWord
+	}
+	for i := 0; i < xorCardinality; i++ {
+		right.bitmap[i/64] ^= uint64(1) << (i % 64)
+	}
+	left.computeCardinality()
+	right.computeCardinality()
+	return left, right
+}
+
+func newDenseXorBitmapFixture(containers int) (*Bitmap, *Bitmap) {
+	leftWords := make([]uint64, containers*bitmapContainerSize)
+	rightWords := make([]uint64, len(leftWords))
+	for i := range leftWords {
+		leftWords[i] = bitmapXorDenseWord
+		rightWords[i] = ^bitmapXorDenseWord
+	}
+	return FromDense(leftWords, true), FromDense(rightWords, true)
+}
+
+// BenchmarkBitmapXorDenseContainers measures Bitmap.Xor with matching dense
+// keys. Each operation alternates between a half-full and full bitmap result,
+// so every matching pair takes ixorBitmap's bitmap-result branch.
+func BenchmarkBitmapXorDenseContainers(b *testing.B) {
+	for _, benchmark := range []struct {
+		name       string
+		containers int
+	}{
+		{name: "one", containers: 1},
+		{name: "fifty", containers: 50},
+		{name: "two-fifty-six", containers: 256},
+	} {
+		b.Run(benchmark.name, func(b *testing.B) {
+			left, right := newDenseXorBitmapFixture(benchmark.containers)
+
+			b.ReportAllocs()
+			b.ResetTimer()
+			for b.Loop() {
+				left.Xor(right)
+			}
+			b.StopTimer()
+
+			want := uint64(benchmark.containers * maxCapacity / 2)
+			if b.N%2 != 0 {
+				want *= 2
+			}
+			if got := left.GetCardinality(); got != want {
+				b.Fatalf("unexpected cardinality: got %d, want %d", got, want)
+			}
+		})
+	}
+}
+
+func newDenseXorThresholdFixture() (*Bitmap, *Bitmap) {
+	leftWords := make([]uint64, bitmapContainerSize)
+	rightWords := make([]uint64, bitmapContainerSize)
+	for i := range leftWords {
+		leftWords[i] = bitmapXorDenseWord
+		rightWords[i] = bitmapXorDenseWord
+	}
+	for i := 0; i < arrayDefaultMaxSize; i++ {
+		rightWords[i/64] ^= uint64(1) << (i % 64)
+	}
+	return FromDense(leftWords, true), FromDense(rightWords, true)
+}
+
+// BenchmarkBitmapXorDenseThresholdBatch measures independent ordinary Xor
+// calls at the array-result threshold. It resets receivers outside the timed
+// region and batches calls to reduce timer noise without changing the path.
+func BenchmarkBitmapXorDenseThresholdBatch(b *testing.B) {
+	const batchSize = 16
+
+	lefts := make([]*Bitmap, batchSize)
+	originals := make([]container, batchSize)
+	_, right := newDenseXorThresholdFixture()
+	for i := range lefts {
+		lefts[i], _ = newDenseXorThresholdFixture()
+		originals[i] = lefts[i].highlowcontainer.getContainerAtIndex(0)
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for b.Loop() {
+		for _, left := range lefts {
+			left.Xor(right)
+		}
+		b.StopTimer()
+		for i, left := range lefts {
+			left.highlowcontainer.setContainerAtIndex(0, originals[i])
+		}
+		b.StartTimer()
+	}
+	b.StopTimer()
+
+	lefts[0].Xor(right)
+	if got := lefts[0].GetCardinality(); got != arrayDefaultMaxSize {
+		b.Fatalf("unexpected cardinality: got %d, want %d", got, arrayDefaultMaxSize)
+	}
+	if _, ok := lefts[0].highlowcontainer.getContainerAtIndex(0).(*arrayContainer); !ok {
+		b.Fatal("expected an array result")
+	}
+}

bitmapcontainer_test.go

@@ -639,3 +639,64 @@ func TestBitmapContainerFillLeastSignificant16bitsProperties(t *testing.T) {
 		runTest(t, vals, 0x55550000)
 	})
 }
+
+func TestBitmapContainerIXorBitmapDense(t *testing.T) {
+	left, right := newBitmapContainerXorFixture(maxCapacity)
+
+	result := left.ixorBitmap(right)
+	actual, ok := result.(*bitmapContainer)
+	require.True(t, ok)
+	assert.Equal(t, maxCapacity, actual.cardinality)
+	for _, word := range actual.bitmap {
+		assert.Equal(t, ^uint64(0), word)
+	}
+	assert.NoError(t, actual.validate())
+}
+
+func TestBitmapContainerIXorBitmapThreshold(t *testing.T) {
+	left, right := newBitmapContainerXorFixture(arrayDefaultMaxSize)
+	original := left.clone().(*bitmapContainer)
+
+	result := left.ixorBitmap(right)
+	actual, ok := result.(*arrayContainer)
+	require.True(t, ok)
+	require.Len(t, actual.content, arrayDefaultMaxSize)
+	for i, value := range actual.content {
+		assert.Equal(t, uint16(i), value)
+	}
+	assert.True(t, original.equals(left))
+	assert.NoError(t, actual.validate())
+}
+
+func TestBitmapXorDenseThreshold(t *testing.T) {
+	left, right := newDenseXorThresholdFixture()
+
+	left.Xor(right)
+	actual, ok := left.highlowcontainer.getContainerAtIndex(0).(*arrayContainer)
+	require.True(t, ok)
+	require.Len(t, actual.content, arrayDefaultMaxSize)
+	for i, value := range actual.content {
+		assert.Equal(t, uint16(i), value)
+	}
+	assert.NoError(t, left.Validate())
+}
+
+func TestBitmapXorDenseCopyOnWrite(t *testing.T) {
+	const containers = 2
+	left, right := newDenseXorBitmapFixture(containers)
+	original := left.Clone()
+	expectedWords := make([]uint64, containers*bitmapContainerSize)
+	for i := range expectedWords {
+		expectedWords[i] = ^uint64(0)
+	}
+	expected := FromDense(expectedWords, true)
+
+	left.SetCopyOnWrite(true)
+	alias := left.Clone()
+	alias.Xor(right)
+
+	assert.True(t, original.Equals(left))
+	assert.True(t, expected.Equals(alias))
+	assert.NoError(t, left.Validate())
+	assert.NoError(t, alias.Validate())
+}

popcnt_avx2_amd64.go

@@ -27,6 +27,9 @@ func _popcntOrSliceAVX2(s, m []uint64) uint64
 //go:noescape
 func _popcntXorSliceAVX2(s, m []uint64) uint64
 
+//go:noescape
+func _xorSliceInPlaceAVX2(s, m []uint64)
+
 // useAVX2 selects the AVX2 assembly implementations when the running CPU
 // supports AVX2. It is evaluated once at package initialization.
 var useAVX2 = _hasAVX2()
@@ -65,3 +68,11 @@ func popcntXorSlice(s, m []uint64) uint64 {
 	}
 	return popcntXorSliceGo(s, m)
 }
+
+func xorSliceInPlace(s, m []uint64) {
+	if useAVX2 {
+		_xorSliceInPlaceAVX2(s, m)
+		return
+	}
+	xorSliceInPlaceGo(s, m)
+}

popcnt_avx2_amd64.s

@@ -316,6 +316,70 @@ maskdone:
 	MOVQ AX, ret+48(FP)
 	RET
 
+// func _xorSliceInPlaceAVX2(s, m []uint64)
+// XORs m into s. Bitmap containers contain 1024 words, so the main loop handles
+// four 256-bit vectors (128 bytes, 16 words) per iteration. A vector tail and
+// scalar tail keep the helper correct for the shorter slices used by tests.
+TEXT ·_xorSliceInPlaceAVX2(SB), NOSPLIT, $0-48
+	MOVQ s_base+0(FP), SI
+	MOVQ m_base+24(FP), DI
+	MOVQ s_len+8(FP), CX
+	MOVQ CX, R8
+	SHRQ $4, R8 // 16 words per unrolled iteration
+	TESTQ R8, R8
+	JZ xorstoretail
+xorstoreloop:
+	VMOVDQU 0(SI), Y0
+	VMOVDQU 32(SI), Y1
+	VMOVDQU 64(SI), Y2
+	VMOVDQU 96(SI), Y3
+	VMOVDQU 0(DI), Y4
+	VMOVDQU 32(DI), Y5
+	VMOVDQU 64(DI), Y6
+	VMOVDQU 96(DI), Y7
+	VPXOR Y4, Y0, Y0
+	VPXOR Y5, Y1, Y1
+	VPXOR Y6, Y2, Y2
+	VPXOR Y7, Y3, Y3
+	VMOVDQU Y0, 0(SI)
+	VMOVDQU Y1, 32(SI)
+	VMOVDQU Y2, 64(SI)
+	VMOVDQU Y3, 96(SI)
+	ADDQ $128, SI
+	ADDQ $128, DI
+	DECQ R8
+	JNZ xorstoreloop
+xorstoretail:
+	ANDQ $15, CX
+	MOVQ CX, R8
+	SHRQ $2, R8 // 4 words per vector tail iteration
+	TESTQ R8, R8
+	JZ xorstorescalartail
+xorstorevectorloop:
+	VMOVDQU (SI), Y0
+	VMOVDQU (DI), Y1
+	VPXOR Y1, Y0, Y0
+	VMOVDQU Y0, (SI)
+	ADDQ $32, SI
+	ADDQ $32, DI
+	DECQ R8
+	JNZ xorstorevectorloop
+xorstorescalartail:
+	ANDQ $3, CX
+	TESTQ CX, CX
+	JZ xorstoredone
+xorstorescalarloop:
+	MOVQ (SI), AX
+	XORQ (DI), AX
+	MOVQ AX, (SI)
+	ADDQ $8, SI
+	ADDQ $8, DI
+	DECQ CX
+	JNZ xorstorescalarloop
+xorstoredone:
+	VZEROUPPER
+	RET
+
 // func _hasAVX2() bool
 // Reports whether the CPU supports AVX2 and the OS has enabled the wide (YMM)
 // register state. All three checks must pass; otherwise the Go wrappers fall

popcnt_avx2_amd64_test.go

@@ -80,6 +80,30 @@ func BenchmarkPopcntSlice1024Go(b *testing.B) {
 	_ = sink
 }
 
+// BenchmarkBitmapXorDenseScalarFallback measures the ordinary dense Xor path
+// with AVX2 disabled, including the portable count and store fallbacks.
+func BenchmarkBitmapXorDenseScalarFallback(b *testing.B) {
+	saved := useAVX2
+	useAVX2 = false
+	defer func() { useAVX2 = saved }()
+
+	left, right := newDenseXorBitmapFixture(50)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for b.Loop() {
+		left.Xor(right)
+	}
+	b.StopTimer()
+
+	want := uint64(50 * maxCapacity / 2)
+	if b.N%2 != 0 {
+		want *= 2
+	}
+	if got := left.GetCardinality(); got != want {
+		b.Fatalf("unexpected cardinality: got %d, want %d", got, want)
+	}
+}
+
 func TestAVX2PopcntDispatch(t *testing.T) {
 	// Verify the runtime dispatch wrappers agree with the Go reference both
 	// when AVX2 is selected and when the scalar fallback is forced.
@@ -127,3 +151,61 @@ func TestAVX2PopcntDifferential(t *testing.T) {
 		}
 	}
 }
+
+func TestAVX2XorSliceDispatch(t *testing.T) {
+	saved := useAVX2
+	defer func() { useAVX2 = saved }()
+
+	r := rand.New(rand.NewSource(99))
+	for _, on := range []bool{false, true} {
+		if on && !saved {
+			continue
+		}
+		useAVX2 = on
+		for _, n := range avx2TestLengths {
+			input := randomUint64Slice(r, n)
+			mask := randomUint64Slice(r, n)
+			want := make([]uint64, len(input))
+			copy(want, input)
+			xorSliceInPlaceGo(want, mask)
+
+			xorSliceInPlace(input, mask)
+			assert.Equalf(t, want, input, "xorSliceInPlace avx2=%v len=%d", on, n)
+
+			alias := randomUint64Slice(r, n)
+			xorSliceInPlace(alias, alias)
+			assert.Equalf(t, make([]uint64, n), alias, "xorSliceInPlace alias avx2=%v len=%d", on, n)
+		}
+	}
+}
+
+func TestAVX2XorSliceDifferential(t *testing.T) {
+	if !useAVX2 {
+		t.Skip("AVX2 not available on this CPU")
+	}
+	r := rand.New(rand.NewSource(100))
+	for _, n := range avx2TestLengths {
+		for iter := 0; iter < 64; iter++ {
+			input := randomUint64Slice(r, n)
+			mask := randomUint64Slice(r, n)
+			want := make([]uint64, len(input))
+			copy(want, input)
+			xorSliceInPlaceGo(want, mask)
+
+			_xorSliceInPlaceAVX2(input, mask)
+			assert.Equalf(t, want, input, "_xorSliceInPlaceAVX2 len=%d", n)
+		}
+	}
+}
+
+func TestBitmapXorDenseScalarFallback(t *testing.T) {
+	saved := useAVX2
+	useAVX2 = false
+	defer func() { useAVX2 = saved }()
+
+	left, right := newDenseXorBitmapFixture(2)
+	left.Xor(right)
+
+	assert.Equal(t, uint64(2*maxCapacity), left.GetCardinality())
+	assert.NoError(t, left.Validate())
+}

popcnt_generic.go

@@ -22,3 +22,7 @@ func popcntOrSlice(s, m []uint64) uint64 {
 func popcntXorSlice(s, m []uint64) uint64 {
 	return popcntXorSliceGo(s, m)
 }
+
+func xorSliceInPlace(s, m []uint64) {
+	xorSliceInPlaceGo(s, m)
+}

popcnt_neon_arm64.go

@@ -63,3 +63,7 @@ func popcntXorSlice(s, m []uint64) uint64 {
 	}
 	return popcntXorSliceGo(s, m)
 }
+
+func xorSliceInPlace(s, m []uint64) {
+	xorSliceInPlaceGo(s, m)
+}