Skip to content

Add AArch64 PMULL implementation for CRC #87

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Xor-el merged 3 commits into from
Jun 24, 2026
Merged

Add AArch64 PMULL implementation for CRC #87

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
73f6bfa
Add AArch64 PMULL implementation for Crc
Xor-el Jun 24, 2026
89cc686
enable benchmark
Xor-el Jun 24, 2026
c0ecfb0
disable benchmark
Xor-el Jun 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion HashLib/src/Checksum/HlpCRCDispatch.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ procedure CRC_UpdateViaBitSerial(AData: PByte; ADataLength, AIndex: Int32;
//
// i386: SSE2
// x86_64: VPCLMULQDQ, PCLMULQDQ, SSE2
// aarch64: —
// aarch64: PMULL
// =============================================================================

{$IFDEF HASHLIB_X86_64_ASM}
Expand Down Expand Up @@ -450,6 +450,22 @@ function CRC_Fold_Forward_Vpclmul(AData: PByte; ALength: UInt32;

{$ENDIF HASHLIB_X86_64_ASM}

{$IFDEF HASHLIB_AARCH64_ASM}

function CRC_Fold_Reflected_Pmull(AData: PByte; ALength: UInt32;
AState: Pointer; AConstants: Pointer): UInt64;
{$I ..\Include\Simd\Common\SimdProc4Begin_aarch64.inc}
{$I ..\Include\Simd\CRC\CRCFoldReflectedPmull_aarch64.inc}
end;

function CRC_Fold_Forward_Pmull(AData: PByte; ALength: UInt32;
AState: Pointer; AConstants: Pointer): UInt64;
{$I ..\Include\Simd\Common\SimdProc4Begin_aarch64.inc}
{$I ..\Include\Simd\CRC\CRCFoldForwardPmull_aarch64.inc}
end;

{$ENDIF HASHLIB_AARCH64_ASM}

// =============================================================================
// Dispatch initialization
// =============================================================================
Expand All @@ -469,6 +485,17 @@ procedure InitDispatch();
CRC_Fold_Reflected32 := @CRC_Fold_Reflected32_Scalar;
CRC_Fold_UsesCarrylessMul := False;

{$IFDEF HASHLIB_AARCH64_ASM}
if TCpuFeatures.Arm.HasPMULL() then
begin
CRC_Fold_Reflected := @CRC_Fold_Reflected_Pmull;
CRC_Fold_Forward := @CRC_Fold_Forward_Pmull;
CRC_Fold_Reflected32 := @CRC_Fold_Reflected_Pmull;
CRC_Fold_UsesCarrylessMul := True;
Exit;
end;
{$ENDIF HASHLIB_AARCH64_ASM}

{$IFDEF HASHLIB_X86_64_ASM}
if TCpuFeatures.X86.HasVPCLMULQDQ() then
begin
Expand Down
131 changes: 131 additions & 0 deletions HashLib/src/Include/Simd/CRC/CRCFoldForwardPmull_aarch64.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
// CRC folding (PMULL/PMULL2 + TBL byte-reverse) + Barrett reduction for
// non-reflected (MSB-first) CRCs with width 8..64.
// Expects AAPCS64: x0 = AData, w1 = ALength (>= 16, multiple of 16),
// x2 = AState, x3 = AConstants (PCRCFoldRuntimeCtx; FoldConstants at +0).
// AState: [0..7] = initial CRC pre-shifted left by (64 - Width), [8..15] = 0.
// AConstants: [+0] Fold_4x128, [+16] Fold_1x128, [+32] Barrett,
// [+64] BswapMask, [+80] CrcBits, [+88] BarrettShift.
// Returns final CRC in x0.
// Leaf nostackframe; caller-saved GPR/vector only (v0-v7, v17).
// Reference: Linux kernel crc-pclmul-template.S (Eric Biggers),
// HashLib CRCFoldForwardPclmul_x86_64.inc.
// AArch64 vector instructions are .long-encoded for broad assembler compatibility.
// Register map:
// v0-v3 = fold accumulators (xmm0-3)
// v4 = pmull-hi temp (xmm4)
// v5 = loaded 16-byte block (xmm5)
// v6 = BswapMask then Barrett consts [G : mu_lo] (xmm6)
// v7 = Fold_4x128 then Fold_1x128 (xmm7)
// v17 = dup staging for clmul hi*lo ($01) Barrett mix
.long 0x3dc01066 // ldr q6, [x3, #64]
.long 0x7101003f // cmp w1, #64
b.hs .Lcrc_fwd_large
.long 0x3dc00467 // ldr q7, [x3, #16]
.long 0x3dc00000 // ldr q0, [x0]
.long 0x4e060000 // tbl v0.16b, {v0.16b}, v6.16b
.long 0xf9400044 // ldr x4, [x2]
.long 0x6e241c84 // eor v4.16b, v4.16b, v4.16b
.long 0x4e181c84 // ins v4.d[1], x4
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x91004000 // add x0, x0, #16
.long 0x51004021 // sub w1, w1, #16
b .Lcrc_fwd_fold1_check
.Lcrc_fwd_large:
.long 0x3dc00067 // ldr q7, [x3]
.long 0x3dc00000 // ldr q0, [x0, #0]
.long 0x4e060000 // tbl v0.16b, {v0.16b}, v6.16b
.long 0x3dc00401 // ldr q1, [x0, #16]
.long 0x4e060021 // tbl v1.16b, {v1.16b}, v6.16b
.long 0x3dc00802 // ldr q2, [x0, #32]
.long 0x4e060042 // tbl v2.16b, {v2.16b}, v6.16b
.long 0x3dc00c03 // ldr q3, [x0, #48]
.long 0x4e060063 // tbl v3.16b, {v3.16b}, v6.16b
.long 0xf9400044 // ldr x4, [x2]
.long 0x6e241c84 // eor v4.16b, v4.16b, v4.16b
.long 0x4e181c84 // ins v4.d[1], x4
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x91010000 // add x0, x0, #64
.long 0x51010021 // sub w1, w1, #64
.long 0x7101003f // cmp w1, #64
b.lo .Lcrc_fwd_fold4_done
.Lcrc_fwd_fold4_loop:
.long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b
.long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x3dc00005 // ldr q5, [x0, #0]
.long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b
.long 0x6e251c00 // eor v0.16b, v0.16b, v5.16b
.long 0x4ea11c24 // orr v4.16b, v1.16b, v1.16b
.long 0x0ee7e021 // pmull v1.1q, v1.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c21 // eor v1.16b, v1.16b, v4.16b
.long 0x3dc00405 // ldr q5, [x0, #16]
.long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b
.long 0x6e251c21 // eor v1.16b, v1.16b, v5.16b
.long 0x4ea21c44 // orr v4.16b, v2.16b, v2.16b
.long 0x0ee7e042 // pmull v2.1q, v2.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b
.long 0x3dc00805 // ldr q5, [x0, #32]
.long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b
.long 0x6e251c42 // eor v2.16b, v2.16b, v5.16b
.long 0x4ea31c64 // orr v4.16b, v3.16b, v3.16b
.long 0x0ee7e063 // pmull v3.1q, v3.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c63 // eor v3.16b, v3.16b, v4.16b
.long 0x3dc00c05 // ldr q5, [x0, #48]
.long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b
.long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b
.long 0x91010000 // add x0, x0, #64
.long 0x51010021 // sub w1, w1, #64
.long 0x7101003f // cmp w1, #64
b.hs .Lcrc_fwd_fold4_loop
.Lcrc_fwd_fold4_done:
.long 0x3dc00467 // ldr q7, [x3, #16]
.long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b
.long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
.long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b
.long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x6e221c00 // eor v0.16b, v0.16b, v2.16b
.long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b
.long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x6e231c00 // eor v0.16b, v0.16b, v3.16b
.Lcrc_fwd_fold1_check:
.long 0x7100403f // cmp w1, #16
b.lo .Lcrc_fwd_fold1_done
.Lcrc_fwd_fold1_loop:
.long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b
.long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x3dc00005 // ldr q5, [x0, #0]
.long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b
.long 0x6e251c00 // eor v0.16b, v0.16b, v5.16b
.long 0x91004000 // add x0, x0, #16
.long 0x51004021 // sub w1, w1, #16
.long 0x7100403f // cmp w1, #16
b.hs .Lcrc_fwd_fold1_loop
.Lcrc_fwd_fold1_done:
.long 0x3dc00866 // ldr q6, [x3, #32]
.long 0x4e180411 // dup v17.2d, v0.d[1]
.long 0x0ee6e221 // pmull v1.1q, v17.1d, v6.1d
.long 0x6e201c21 // eor v1.16b, v1.16b, v0.16b
.long 0x4ee6e021 // pmull2 v1.1q, v1.2d, v6.2d
.long 0xfd402c62 // ldr d2, [x3, #88]
.long 0x4e080442 // dup v2.2d, v2.d[0]
.long 0x6ee24421 // ushl v1.2d, v1.2d, v2.2d
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
.long 0x0ee6e001 // pmull v1.1q, v0.1d, v6.1d
.long 0x6e004000 // ext v0.16b, v0.16b, v0.16b, #8
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
.long 0x4ee6e000 // pmull2 v0.1q, v0.2d, v6.2d
.long 0x4e083c00 // umov x0, v0.d[0]
ret
125 changes: 125 additions & 0 deletions HashLib/src/Include/Simd/CRC/CRCFoldReflectedPmull_aarch64.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
// CRC folding (PMULL/PMULL2) + Barrett reduction for reflected (LSB-first)
// CRCs with width 8..64. CRC-64 uses a special Barrett path for the 65-bit G.
// Expects AAPCS64: x0 = AData, w1 = ALength (>= 16, multiple of 16),
// x2 = AState, x3 = AConstants (PCRCFoldRuntimeCtx; FoldConstants at +0).
// AState: [0..7] = initial CRC (reflected), [8..15] = 0.
// AConstants: [+0] Fold_4x128, [+16] Fold_1x128, [+32] Barrett,
// [+80] CrcBits. Returns final CRC in x0.
// Leaf nostackframe; caller-saved GPR/vector only (v0-v7, v16-v18).
// Reference: Linux kernel crc-pclmul-template.S (Eric Biggers),
// HashLib CRCFoldReflectedPclmul_x86_64.inc.
// AArch64 vector instructions are .long-encoded for broad assembler compatibility.
// Register map:
// v0-v3 = fold accumulators (xmm0-3)
// v4 = pmull-hi temp (xmm4)
// v5 = loaded 16-byte block (xmm5)
// v6 = Fold_4x128 then Barrett consts (xmm6)
// v7 = Fold_1x128 (xmm7)
// v16 = zeroed scratch (psrldq emulation via ext)
// v17 = dup staging for clmul lo*hi ($10) Barrett mixes
// v18 = saved Barrett q (CRC-64 x^0 correction)
.long 0x3dc00467 // ldr q7, [x3, #16]
.long 0x7101003f // cmp w1, #64
b.hs .Lcrc_refl_large
.long 0x3dc00000 // ldr q0, [x0]
.long 0xfd400044 // ldr d4, [x2]
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x91004000 // add x0, x0, #16
.long 0x51004021 // sub w1, w1, #16
b .Lcrc_refl_fold1_check
.Lcrc_refl_large:
.long 0x3dc00066 // ldr q6, [x3]
.long 0x3dc00000 // ldr q0, [x0]
.long 0x3dc00401 // ldr q1, [x0, #16]
.long 0x3dc00802 // ldr q2, [x0, #32]
.long 0x3dc00c03 // ldr q3, [x0, #48]
.long 0xfd400044 // ldr d4, [x2]
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x91010000 // add x0, x0, #64
.long 0x51010021 // sub w1, w1, #64
.long 0x7101003f // cmp w1, #64
b.lo .Lcrc_refl_fold4_done
.Lcrc_refl_fold4_loop:
.long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b
.long 0x0ee6e000 // pmull v0.1q, v0.1d, v6.1d
.long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x3dc00005 // ldr q5, [x0, #0]
.long 0x6e251c00 // eor v0.16b, v0.16b, v5.16b
.long 0x4ea11c24 // orr v4.16b, v1.16b, v1.16b
.long 0x0ee6e021 // pmull v1.1q, v1.1d, v6.1d
.long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d
.long 0x6e241c21 // eor v1.16b, v1.16b, v4.16b
.long 0x3dc00405 // ldr q5, [x0, #16]
.long 0x6e251c21 // eor v1.16b, v1.16b, v5.16b
.long 0x4ea21c44 // orr v4.16b, v2.16b, v2.16b
.long 0x0ee6e042 // pmull v2.1q, v2.1d, v6.1d
.long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d
.long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b
.long 0x3dc00805 // ldr q5, [x0, #32]
.long 0x6e251c42 // eor v2.16b, v2.16b, v5.16b
.long 0x4ea31c64 // orr v4.16b, v3.16b, v3.16b
.long 0x0ee6e063 // pmull v3.1q, v3.1d, v6.1d
.long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d
.long 0x6e241c63 // eor v3.16b, v3.16b, v4.16b
.long 0x3dc00c05 // ldr q5, [x0, #48]
.long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b
.long 0x91010000 // add x0, x0, #64
.long 0x51010021 // sub w1, w1, #64
.long 0x7101003f // cmp w1, #64
b.hs .Lcrc_refl_fold4_loop
.Lcrc_refl_fold4_done:
.long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b
.long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
.long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b
.long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x6e221c00 // eor v0.16b, v0.16b, v2.16b
.long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b
.long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x6e231c00 // eor v0.16b, v0.16b, v3.16b
.Lcrc_refl_fold1_check:
.long 0x7100403f // cmp w1, #16
b.lo .Lcrc_refl_fold1_done
.Lcrc_refl_fold1_loop:
.long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b
.long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d
.long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d
.long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b
.long 0x3dc00005 // ldr q5, [x0, #0]
.long 0x6e251c00 // eor v0.16b, v0.16b, v5.16b
.long 0x91004000 // add x0, x0, #16
.long 0x51004021 // sub w1, w1, #16
.long 0x7100403f // cmp w1, #16
b.hs .Lcrc_refl_fold1_loop
.Lcrc_refl_fold1_done:
.long 0x4e1804f1 // dup v17.2d, v7.d[1]
.long 0x0ef1e001 // pmull v1.1q, v0.1d, v17.1d
.long 0x6e301e10 // eor v16.16b, v16.16b, v16.16b
.long 0x6e104000 // ext v0.16b, v0.16b, v16.16b, #8
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
.long 0x3dc00866 // ldr q6, [x3, #32]
.long 0x0ee6e001 // pmull v1.1q, v0.1d, v6.1d
.long 0xf9402864 // ldr x4, [x3, #80]
.long 0xf101009f // cmp x4, #64
b.eq .Lcrc_refl_barrett64
.long 0x4e1804d1 // dup v17.2d, v6.d[1]
.long 0x0ef1e021 // pmull v1.1q, v1.1d, v17.1d
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
b .Lcrc_refl_extract
.Lcrc_refl_barrett64:
.long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b
.long 0x4e1804d1 // dup v17.2d, v6.d[1]
.long 0x0ef1e021 // pmull v1.1q, v1.1d, v17.1d
.long 0x4e080652 // dup v18.2d, v18.d[0]
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
.long 0x6e321c00 // eor v0.16b, v0.16b, v18.16b
.Lcrc_refl_extract:
.long 0x4e183c00 // umov x0, v0.d[1]
ret
Loading