perf(parquet): vectorize amd64 SSE4/AVX2 bool unpacking for ~9x throughput

Replace the scalar bit-by-bit implementations of _bytes_to_bools_sse4
and _bytes_to_bools_avx2 with actual SIMD vectorized code. The previous
implementations were auto-generated by c2goasm from clang output that
failed to auto-vectorize, resulting in purely scalar code (movzx/shr/and/
mov one bit at a time) despite being labeled as SSE4 and AVX2.

SSE4: uses PSHUFB to broadcast 2 input bytes into 16 XMM lanes, then
PAND+PCMPEQB for parallel bit-test and PAND to normalize to 0/1.
Processes 2 bytes (16 bools) per iteration.

AVX2: uses VPBROADCASTD+VPSHUFB to broadcast 4 input bytes into 32 YMM
lanes, then VPAND+VPCMPEQB+VPAND for parallel bit-test and normalize.
Processes 4 bytes (32 bools) per iteration. Includes VZEROUPPER to
avoid SSE-AVX transition penalties.

Both include scalar tails for edge cases with <vector-width output slots.

Benchmarks on AMD Ryzen 7 7800X3D (linux/amd64):

  BytesToBools/64B   146.0ns -> 15.60ns  (9.4x, 418->3913 MiB/s)
  BytesToBools/256B  562.3ns -> 63.36ns  (8.9x, 434->3853 MiB/s)
  BytesToBools/1KB   2247ns  -> 253.9ns  (8.8x, 435->3846 MiB/s)
  BytesToBools/4KB   8970ns  -> 1018ns   (8.8x, 436->3838 MiB/s)
  BytesToBools/16KB  35798ns -> 4044ns   (8.9x, 437->3864 MiB/s)

  geomean: -88.8% latency, +795% throughput

Author: zeroshade

.github/workflows/test.yml

@@ -330,7 +330,7 @@ jobs:
     name: TinyGo
     runs-on: ubuntu-latest
     env:
-      TINYGO_VERSION: 0.38.0
+      TINYGO_VERSION: 0.40.1
     timeout-minutes: 20
     steps:
       - name: Checkout

go.mod

@@ -16,7 +16,7 @@
 
 module github.com/apache/arrow-go/v18
 
-go 1.24.0
+go 1.25.0
 
 require (
 	github.com/andybalholm/brotli v1.2.0

parquet/internal/utils/unpack_bool.go

@@ -20,7 +20,11 @@ package utils
 func bytesToBoolsGo(in []byte, out []bool) {
 	for i, b := range in {
 		for j := 0; j < 8; j++ {
-			out[8*i+j] = (b & (1 << j)) != 0
+			idx := 8*i + j
+			if idx >= len(out) {
+				return
+			}
+			out[idx] = (b & (1 << j)) != 0
 		}
 	}
 }

parquet/internal/utils/unpack_bool_avx2_amd64.s

@@ -1,88 +1,127 @@
 //+build !noasm !appengine
-// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT
 
-TEXT ·_bytes_to_bools_avx2(SB), $0-32
+// AVX2 vectorized bytes-to-bools using VPBROADCASTD + VPSHUFB + VPCMPEQB.
+// Processes 4 input bytes → 32 output bools per vector iteration.
+// Replaces the original c2goasm-generated scalar code which used zero SIMD.
+
+#include "textflag.h"
+
+// VPSHUFB operates on two independent 128-bit lanes.
+// Lower lane: byte 0 → lanes 0-7, byte 1 → lanes 8-15
+// Upper lane: byte 2 → lanes 16-23, byte 3 → lanes 24-31
+DATA shuffle_avx2<>+0x00(SB)/8, $0x0000000000000000
+DATA shuffle_avx2<>+0x08(SB)/8, $0x0101010101010101
+DATA shuffle_avx2<>+0x10(SB)/8, $0x0202020202020202
+DATA shuffle_avx2<>+0x18(SB)/8, $0x0303030303030303
+GLOBL shuffle_avx2<>(SB), (NOPTR+RODATA), $32
+
+// [1, 2, 4, 8, 16, 32, 64, 128] × 4
+DATA bitmask_avx2<>+0x00(SB)/8, $0x8040201008040201
+DATA bitmask_avx2<>+0x08(SB)/8, $0x8040201008040201
+DATA bitmask_avx2<>+0x10(SB)/8, $0x8040201008040201
+DATA bitmask_avx2<>+0x18(SB)/8, $0x8040201008040201
+GLOBL bitmask_avx2<>(SB), (NOPTR+RODATA), $32
+
+// [1, 1, 1, ...] × 32
+DATA ones_avx2<>+0x00(SB)/8, $0x0101010101010101
+DATA ones_avx2<>+0x08(SB)/8, $0x0101010101010101
+DATA ones_avx2<>+0x10(SB)/8, $0x0101010101010101
+DATA ones_avx2<>+0x18(SB)/8, $0x0101010101010101
+GLOBL ones_avx2<>(SB), (NOPTR+RODATA), $32
+
+TEXT ·_bytes_to_bools_avx2(SB), NOSPLIT, $0-32
 
 	MOVQ in+0(FP), DI
 	MOVQ len+8(FP), SI
 	MOVQ out+16(FP), DX
-	MOVQ outlen+24(FP), CX
-
-	WORD $0xf685             // test    esi, esi
-	JLE  LBB0_5
-	WORD $0x8941; BYTE $0xf0 // mov    r8d, esi
-	LONG $0x03e0c149         // shl    r8, 3
-	WORD $0x3145; BYTE $0xd2 // xor    r10d, r10d
-	JMP  LBB0_2
-
-LBB0_4:
-	LONG $0x08c28349         // add    r10, 8
-	LONG $0x01c78348         // add    rdi, 1
-	WORD $0x394d; BYTE $0xd0 // cmp    r8, r10
-	JE   LBB0_5
-
-LBB0_2:
-	WORD $0x3941; BYTE $0xca // cmp    r10d, ecx
-	JGE  LBB0_4
-	WORD $0x8945; BYTE $0xd1 // mov    r9d, r10d
-	WORD $0xb60f; BYTE $0x07 // movzx    eax, byte [rdi]
-	WORD $0x0124             // and    al, 1
-	LONG $0x0a048842         // mov    byte [rdx + r9], al
-	WORD $0x894c; BYTE $0xce // mov    rsi, r9
-	LONG $0x01ce8348         // or    rsi, 1
-	WORD $0xce39             // cmp    esi, ecx
-	JGE  LBB0_4
-	WORD $0xb60f; BYTE $0x07 // movzx    eax, byte [rdi]
-	WORD $0xe8d0             // shr    al, 1
-	WORD $0x0124             // and    al, 1
-	WORD $0x0488; BYTE $0x32 // mov    byte [rdx + rsi], al
-	WORD $0x894c; BYTE $0xce // mov    rsi, r9
-	LONG $0x02ce8348         // or    rsi, 2
-	WORD $0xce39             // cmp    esi, ecx
-	JGE  LBB0_4
-	WORD $0xb60f; BYTE $0x07 // movzx    eax, byte [rdi]
-	WORD $0xe8c0; BYTE $0x02 // shr    al, 2
-	WORD $0x0124             // and    al, 1
-	WORD $0x0488; BYTE $0x32 // mov    byte [rdx + rsi], al
-	WORD $0x894c; BYTE $0xce // mov    rsi, r9
-	LONG $0x03ce8348         // or    rsi, 3
-	WORD $0xce39             // cmp    esi, ecx
-	JGE  LBB0_4
-	WORD $0xb60f; BYTE $0x07 // movzx    eax, byte [rdi]
-	WORD $0xe8c0; BYTE $0x03 // shr    al, 3
-	WORD $0x0124             // and    al, 1
-	WORD $0x0488; BYTE $0x32 // mov    byte [rdx + rsi], al
-	WORD $0x894c; BYTE $0xce // mov    rsi, r9
-	LONG $0x04ce8348         // or    rsi, 4
-	WORD $0xce39             // cmp    esi, ecx
-	JGE  LBB0_4
-	WORD $0xb60f; BYTE $0x07 // movzx    eax, byte [rdi]
-	WORD $0xe8c0; BYTE $0x04 // shr    al, 4
-	WORD $0x0124             // and    al, 1
-	WORD $0x0488; BYTE $0x32 // mov    byte [rdx + rsi], al
-	WORD $0x894c; BYTE $0xce // mov    rsi, r9
-	LONG $0x05ce8348         // or    rsi, 5
-	WORD $0xce39             // cmp    esi, ecx
-	JGE  LBB0_4
-	WORD $0xb60f; BYTE $0x07 // movzx    eax, byte [rdi]
-	WORD $0xe8c0; BYTE $0x05 // shr    al, 5
-	WORD $0x0124             // and    al, 1
-	WORD $0x0488; BYTE $0x32 // mov    byte [rdx + rsi], al
-	WORD $0x894c; BYTE $0xce // mov    rsi, r9
-	LONG $0x06ce8348         // or    rsi, 6
-	WORD $0xce39             // cmp    esi, ecx
-	JGE  LBB0_4
-	WORD $0xb60f; BYTE $0x07 // movzx    eax, byte [rdi]
-	WORD $0xe8c0; BYTE $0x06 // shr    al, 6
-	WORD $0x0124             // and    al, 1
-	WORD $0x0488; BYTE $0x32 // mov    byte [rdx + rsi], al
-	LONG $0x07c98349         // or    r9, 7
-	WORD $0x3941; BYTE $0xc9 // cmp    r9d, ecx
-	JGE  LBB0_4
-	WORD $0xb60f; BYTE $0x07 // movzx    eax, byte [rdi]
-	WORD $0xe8c0; BYTE $0x07 // shr    al, 7
-	LONG $0x0a048842         // mov    byte [rdx + r9], al
-	JMP  LBB0_4
-
-LBB0_5:
+	MOVQ outlen+24(FP), R13
+
+	TESTL SI, SI
+	JLE  done
+
+	VMOVDQU shuffle_avx2<>(SB), Y3
+	VMOVDQU bitmask_avx2<>(SB), Y4
+	VMOVDQU ones_avx2<>(SB), Y5
+
+	XORQ R8, R8
+	XORQ R9, R9
+
+loop32:
+	MOVQ SI, AX
+	SUBQ R8, AX
+	CMPQ AX, $4
+	JL   loop8
+
+	MOVQ R13, AX
+	SUBQ R9, AX
+	CMPQ AX, $32
+	JL   loop8
+
+	MOVL (DI)(R8*1), AX
+	MOVD AX, X0
+	VPBROADCASTD X0, Y0
+	VPSHUFB Y3, Y0, Y0
+	VPAND Y4, Y0, Y1
+	VPCMPEQB Y4, Y1, Y1
+	VPAND Y5, Y1, Y1
+	VMOVDQU Y1, (DX)(R9*1)
+
+	ADDQ $4, R8
+	ADDQ $32, R9
+	JMP  loop32
+
+loop8:
+	CMPQ R8, SI
+	JGE  avx_done
+
+	MOVQ R13, AX
+	SUBQ R9, AX
+	CMPQ AX, $8
+	JL   scalar
+
+	MOVBLZX (DI)(R8*1), AX
+	MOVD AX, X0
+	VPBROADCASTD X0, Y0
+	VPSHUFB Y3, Y0, Y0
+	VPAND Y4, Y0, Y1
+	VPCMPEQB Y4, Y1, Y1
+	VPAND Y5, Y1, Y1
+	MOVQ X1, (DX)(R9*1)
+
+	ADDQ $1, R8
+	ADDQ $8, R9
+	JMP  loop8
+
+scalar:
+	CMPQ R8, SI
+	JGE  avx_done
+	CMPQ R9, R13
+	JGE  avx_done
+
+	MOVBLZX (DI)(R8*1), AX
+	XORQ CX, CX
+
+scalar_bit:
+	CMPQ CX, $8
+	JGE  scalar_next
+	CMPQ R9, R13
+	JGE  avx_done
+
+	MOVL AX, R11
+	SHRL CL, R11
+	ANDL $1, R11
+	MOVB R11, (DX)(R9*1)
+
+	INCQ CX
+	INCQ R9
+	JMP  scalar_bit
+
+scalar_next:
+	INCQ R8
+	JMP  scalar
+
+avx_done:
+	VZEROUPPER
+
+done:
 	RET

parquet/internal/utils/unpack_bool_benchmark_test.go

@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils_test
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/apache/arrow-go/v18/parquet/internal/utils"
+)
+
+func BenchmarkBytesToBools(b *testing.B) {
+	for _, nBytes := range []int{64, 256, 1024, 4096, 16384} {
+		in := make([]byte, nBytes)
+		rng := rand.New(rand.NewSource(42))
+		for i := range in {
+			in[i] = byte(rng.Intn(256))
+		}
+		out := make([]bool, nBytes*8)
+
+		b.Run(fmt.Sprintf("bytes=%d", nBytes), func(b *testing.B) {
+			b.SetBytes(int64(nBytes))
+			for i := 0; i < b.N; i++ {
+				utils.BytesToBools(in, out)
+			}
+		})
+	}
+}
+
+func TestBytesToBoolsCorrectness(t *testing.T) {
+	rng := rand.New(rand.NewSource(12345))
+
+	for _, nBytes := range []int{1, 2, 3, 7, 8, 15, 16, 31, 32, 63, 64, 100, 256, 1024} {
+		t.Run(fmt.Sprintf("bytes=%d", nBytes), func(t *testing.T) {
+			in := make([]byte, nBytes)
+			for i := range in {
+				in[i] = byte(rng.Intn(256))
+			}
+
+			outlen := nBytes * 8
+			got := make([]bool, outlen)
+			want := make([]bool, outlen)
+
+			for i, b := range in {
+				for j := 0; j < 8; j++ {
+					want[8*i+j] = (b & (1 << j)) != 0
+				}
+			}
+
+			utils.BytesToBools(in, got)
+
+			for i := 0; i < outlen; i++ {
+				if got[i] != want[i] {
+					byteIdx := i / 8
+					bitIdx := i % 8
+					t.Fatalf("mismatch at index %d (byte %d, bit %d): got %v, want %v (input byte = 0x%02x)",
+						i, byteIdx, bitIdx, got[i], want[i], in[byteIdx])
+				}
+			}
+		})
+	}
+}
+
+func TestBytesToBoolsOutlenSmaller(t *testing.T) {
+	in := []byte{0xFF, 0xAA, 0x55}
+	for outlen := 1; outlen <= 24; outlen++ {
+		t.Run(fmt.Sprintf("outlen=%d", outlen), func(t *testing.T) {
+			got := make([]bool, outlen)
+			want := make([]bool, outlen)
+
+			for i, b := range in {
+				for j := 0; j < 8; j++ {
+					idx := 8*i + j
+					if idx >= outlen {
+						break
+					}
+					want[idx] = (b & (1 << j)) != 0
+				}
+			}
+
+			utils.BytesToBools(in, got)
+
+			for i := 0; i < outlen; i++ {
+				if got[i] != want[i] {
+					t.Fatalf("outlen=%d: mismatch at index %d: got %v, want %v", outlen, i, got[i], want[i])
+				}
+			}
+		})
+	}
+}