From 78aaa9206c8549c29ea4ad9e54c00311025bfc24 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 19:44:29 +0200 Subject: [PATCH] loong64: emit Go's XVMOVQ .V4 broadcast load instead of a WORD Go's loong64 assembler does have the broadcast load: it is spelled as an arrangement, XVMOVQ off(R), X.V4, and assembles to xvldrepl.d (checked with Go 1.26.4 and 1.27.1: the same word as GNU as). v0.2.5's generator emitted it through go-asmgen's WORD encoder and its docs said Go lacked it; both are corrected. The machine code of sum_loong64.s is byte-identical before and after (compared from `go tool asm -S`), so the hardware results stand. Only the fused multiply-add (xvfmadd.d) remains a WORD. Co-Authored-By: Claude Opus 5.5 --- docs/perf.md | 23 +++++++++++++---------- internal/kernels/asmgen/loong64/gen.go | 11 ++++++----- internal/kernels/gemm_loong64.go | 3 ++- internal/kernels/sum_loong64.s | 16 ++++++++-------- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/docs/perf.md b/docs/perf.md index 7e51ce0..c5299b8 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -605,13 +605,16 @@ baseline is POWER8. ## loong64: LASX kernels (2026-10-04) -Go's loong64 assembler has LASX vector float64 add/sub/mul/div, square root and -256-bit loads, but no fused multiply-add and no broadcast load; go-asmgen -v0.14.0 encodes `xvfmadd.d` and `xvldrepl.d` as `WORD`s (pinned against GNU as -2.43, bit-identical to `math.FMA` on a Loongson 3C5000L; transitional until -cmd/asm names them). go-ndarray uses them for sum, dot, sqrt, add/sub/mul/div -and an 8×8 GEMM micro-kernel (16 four-lane accumulators, A broadcast by -`xvldrepl.d`). LASX is not on every LoongArch CPU, so the kernels run only when +Go's loong64 assembler has LASX vector float64 add/sub/mul/div, square root, +256-bit loads and the broadcast load (spelled as an arrangement, +`XVMOVQ off(R), X.V4`, which assembles to `xvldrepl.d`), but no fused +multiply-add; go-asmgen v0.14.0 encodes `xvfmadd.d` as a `WORD` (pinned +against GNU as 2.43, bit-identical to `math.FMA` on a Loongson 3C5000L; +transitional until cmd/asm names it). go-ndarray uses them for sum, dot, sqrt, +add/sub/mul/div and an 8×8 GEMM micro-kernel (16 four-lane accumulators, A +broadcast by `xvldrepl.d`). (v0.2.5 said Go had no broadcast load and emitted +it as a WORD too; it has one, and v0.2.6 emits the mnemonic, the same machine +code.) LASX is not on every LoongArch CPU, so the kernels run only when the kernel reports it in AT_HWCAP (`/proc/self/auxv`), with the scalar code as the fallback; the parsing is a pure function tested on every target. @@ -672,9 +675,9 @@ host has no route to package mirrors), so there is no external reference. **ppc64le** has no vector-double arithmetic (no `XVADDDP`/`XVMADDADP`), which go-asmgen v0.13.0 now encodes as `WORD`s, so ppc64le has kernels; **loong64** has vector-double add/sub/mul/div (`VADDD` assembles to - `vfadd.d`, `VMULD` to `vfmul.d`, `XVADDD` to `xvfadd.d`) but no vector FMA - or broadcast load, which go-asmgen v0.14.0 encodes (transitionally), so - loong64 has kernels; + `vfadd.d`, `VMULD` to `vfmul.d`, `XVADDD` to `xvfadd.d`) and the broadcast + load (`XVMOVQ off(R), X.V4` = `xvldrepl.d`), but no vector FMA, which + go-asmgen v0.14.0 encodes (transitionally), so loong64 has kernels; **s390x** has them, FMA included (`VFADB`, `VFMADB`); **riscv64** has them (`VFADDVV`, `VFMACCVV`), but the V extension is optional and needs a run-time check. So s390x and riscv64 kernels are work not yet done, not a diff --git a/internal/kernels/asmgen/loong64/gen.go b/internal/kernels/asmgen/loong64/gen.go index c2c3611..21e6cb4 100644 --- a/internal/kernels/asmgen/loong64/gen.go +++ b/internal/kernels/asmgen/loong64/gen.go @@ -3,9 +3,10 @@ // // Go's loong64 assembler has the LASX vector float64 add/sub/mul/div // (XVADDD, XVSUBD, XVMULD, XVDIVD), the square root (XVFSQRTD) and the -// 256-bit loads and stores (XVMOVQ, with immediate offsets), but no fused -// multiply-add and no broadcast load; those come from go-asmgen's encoders -// (Builder.XVFMADDD, XVLDREPLD), which emit WORDs. LASX is not on every +// 256-bit loads and stores (XVMOVQ, with immediate offsets), and its +// broadcast load is the arrangement form XVMOVQ off(R), X.V4 (xvldrepl.d). It +// has no fused multiply-add: that comes from go-asmgen's XVFMADDD, which +// emits a WORD (transitional until cmd/asm names it). LASX is not on every // LoongArch CPU, so the Go side calls these kernels only when the kernel // reports it (AT_HWCAP). // @@ -171,7 +172,7 @@ func binKernel(name, vec, sca string) *emit.Function { // for p in [0,kc): C[r][col] += pa[p*8+r] * pb[p*8+col] (r<8, col<8) // // 16 accumulators X0..X15 (row r in X(2r), X(2r+1), four columns each), the B -// row in X16, X17, and each A value broadcast by xvldrepl.d into X18..X25, +// row in X16, X17, and each A value broadcast (XVMOVQ .V4, xvldrepl.d) into X18..X25, // then xvfmadd.d: 16 fused multiply-adds per k step against 10 loads. Fused, // like the other FMA kernels; on integer-valued data it equals the scalar // ikj oracle exactly. @@ -186,7 +187,7 @@ func gemmKernel() *emit.Function { b.Label("gloop") b.Raw("XVMOVQ 0(R7), X16").Raw("XVMOVQ 32(R7), X17") for r := 0; r < gemmMR; r++ { - b.XVLDREPLD(18+r, 6, 8*r) + b.Raw("XVMOVQ %d(R6), X%d.V4", 8*r, 18+r) // xvldrepl.d: pa[r] in all four lanes } for r := 0; r < gemmMR; r++ { b.XVFMADDD(2*r, 18+r, 16, 2*r) diff --git a/internal/kernels/gemm_loong64.go b/internal/kernels/gemm_loong64.go index 592c3c0..f926b6f 100644 --- a/internal/kernels/gemm_loong64.go +++ b/internal/kernels/gemm_loong64.go @@ -2,7 +2,8 @@ package kernels // loong64 GEMM micro-kernel: an 8x8 LASX tile (gemmMicro8x8LASX in // sum_loong64.s, generated by go-asmgen): 16 four-lane accumulators, the B row -// in two vectors, each A value broadcast by xvldrepl.d and a fused xvfmadd.d. +// in two vectors, each A value broadcast by xvldrepl.d (XVMOVQ .V4) and a +// fused xvfmadd.d. // Without LASX, gemmMicroGo computes the same tile in scalar code. // MR, NR are the micro-kernel's register-block dimensions. diff --git a/internal/kernels/sum_loong64.s b/internal/kernels/sum_loong64.s index 55d17ba..b9f165c 100644 --- a/internal/kernels/sum_loong64.s +++ b/internal/kernels/sum_loong64.s @@ -340,14 +340,14 @@ TEXT ·gemmMicro8x8LASX(SB), NOSPLIT, $0-40 gloop: XVMOVQ 0(R7), X16 XVMOVQ 32(R7), X17 - WORD $0x321000d2 // xvldrepl.d $xr18, $r6, 0 - WORD $0x321004d3 // xvldrepl.d $xr19, $r6, 8 - WORD $0x321008d4 // xvldrepl.d $xr20, $r6, 16 - WORD $0x32100cd5 // xvldrepl.d $xr21, $r6, 24 - WORD $0x321010d6 // xvldrepl.d $xr22, $r6, 32 - WORD $0x321014d7 // xvldrepl.d $xr23, $r6, 40 - WORD $0x321018d8 // xvldrepl.d $xr24, $r6, 48 - WORD $0x32101cd9 // xvldrepl.d $xr25, $r6, 56 + XVMOVQ 0(R6), X18.V4 + XVMOVQ 8(R6), X19.V4 + XVMOVQ 16(R6), X20.V4 + XVMOVQ 24(R6), X21.V4 + XVMOVQ 32(R6), X22.V4 + XVMOVQ 40(R6), X23.V4 + XVMOVQ 48(R6), X24.V4 + XVMOVQ 56(R6), X25.V4 WORD $0x0a204240 // xvfmadd.d $xr0, $xr18, $xr16, $xr0 WORD $0x0a20c641 // xvfmadd.d $xr1, $xr18, $xr17, $xr1 WORD $0x0a214262 // xvfmadd.d $xr2, $xr19, $xr16, $xr2