diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 39ae88cf7a..f0de431e8e 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -283,4 +283,5 @@ hheei * Aadityansha Verma * [2026-07-14] Add independent transpose support for C in GEADD (sgeadd/dgeadd/cgeadd/zgeadd). - +* Hugo Meiland + * [2026-08-09] Add Cortex-A72 DGEMM 6x8 microkernel and blocking diff --git a/Makefile.system b/Makefile.system index e8e732e9c8..b3525b651b 100644 --- a/Makefile.system +++ b/Makefile.system @@ -725,6 +725,7 @@ ifeq ($(ARCH), arm64) DYNAMIC_CORE = ARMV8 DYNAMIC_CORE += CORTEXA53 DYNAMIC_CORE += CORTEXA57 +DYNAMIC_CORE += CORTEXA72 DYNAMIC_CORE += NEOVERSEN1 ifneq ($(NO_SVE), 1) DYNAMIC_CORE += NEOVERSEV1 diff --git a/cmake/arch.cmake b/cmake/arch.cmake index b902937a05..84caff6e22 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -33,7 +33,7 @@ endif () if (DYNAMIC_ARCH) if (ARM64) - set(DYNAMIC_CORE ARMV8 CORTEXA53 CORTEXA57 THUNDERX THUNDERX2T99 TSV110 EMAG8180 NEOVERSEN1 THUNDERX3T110) + set(DYNAMIC_CORE ARMV8 CORTEXA53 CORTEXA57 CORTEXA72 THUNDERX THUNDERX2T99 TSV110 EMAG8180 NEOVERSEN1 THUNDERX3T110) if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU") if (${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) # SVE ACLE supported in GCC >= 10 set(DYNAMIC_CORE ${DYNAMIC_CORE} NEOVERSEV1 NEOVERSEN2 ARMV8SVE A64FX) diff --git a/driver/others/dynamic_arm64.c b/driver/others/dynamic_arm64.c index 0fdf72d56a..43ede41af5 100644 --- a/driver/others/dynamic_arm64.c +++ b/driver/others/dynamic_arm64.c @@ -168,10 +168,12 @@ extern gotoblas_t gotoblas_CORTEXA53; #define gotoblas_CORTEXA55 gotoblas_CORTEXA53 #ifdef OS_DARWIN #define gotoblas_CORTEXA57 gotoblas_ARMV8 +#define gotoblas_CORTEXA72 gotoblas_ARMV8 #else extern gotoblas_t gotoblas_CORTEXA57; +/* Darwin DYNAMIC_CORE does not build CORTEXA72; Linux DYNAMIC_ARCH does. */ +extern gotoblas_t gotoblas_CORTEXA72; #endif -#define gotoblas_CORTEXA72 gotoblas_CORTEXA57 #define gotoblas_CORTEXA73 gotoblas_CORTEXA57 #define gotoblas_FALKOR gotoblas_CORTEXA57 #ifdef OS_DARWIN diff --git a/kernel/arm64/KERNEL.CORTEXA72 b/kernel/arm64/KERNEL.CORTEXA72 index 007b2ce267..f7d247e7ec 100644 --- a/kernel/arm64/KERNEL.CORTEXA72 +++ b/kernel/arm64/KERNEL.CORTEXA72 @@ -1,3 +1,28 @@ include $(KERNELDIR)/KERNEL.CORTEXA57 +# Cortex-A72 DGEMM uses a dedicated 6x8 microkernel (see +# dgemm_kernel_6x8_cortexa72.S). Stock TARGET=CORTEXA72 previously +# aliased CORTEXA57 (8x4). Keep SGEMM/CGEMM/ZGEMM from A57 for now. +# +# Stock generic/gemm_{t,n}copy_6.c packs as 4+2, not contiguous MR=6 — +# use the panel packers. NR=8 reuses the asm dgemm_{n,t}copy_8.S. +# No dtrmm_kernel_6x8.S yet; fall back to generic 2x2 for DTRMM. +# Stock generic/trsm_kernel_*.c assumes power-of-2 UNROLL_M; use the +# UNROLLM6-aware kernels (same idea as loongarch64 UNROLLN6). +DGEMMKERNEL = dgemm_kernel_6x8_cortexa72.S +DTRMMKERNEL = ../generic/trmmkernel_2x2.c +DTRSMKERNEL_LN = trsm_kernel_LN_UNROLLM6.c +DTRSMKERNEL_LT = trsm_kernel_LT_UNROLLM6.c +DTRSMKERNEL_RN = trsm_kernel_RN_UNROLLM6.c +DTRSMKERNEL_RT = trsm_kernel_RT_UNROLLM6.c + +DGEMMINCOPY = gemm_ncopy_6_panel.c +DGEMMITCOPY = gemm_tcopy_6_panel.c +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) + +DGEMMONCOPY = dgemm_ncopy_8.S +DGEMMOTCOPY = dgemm_tcopy_8.S +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) diff --git a/kernel/arm64/dgemm_kernel_6x8_cortexa72.S b/kernel/arm64/dgemm_kernel_6x8_cortexa72.S new file mode 100644 index 0000000000..54478af0b3 --- /dev/null +++ b/kernel/arm64/dgemm_kernel_6x8_cortexa72.S @@ -0,0 +1,1072 @@ +/******************************************************************************* +Copyright (c) 2015, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#define ASSEMBLER +#include "common.h" + +/* X0 X1 X2 s0 X3 x4 x5 x6 */ +/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc )*/ + +#define origM x0 +#define origN x1 +#define origK x2 +#define origPA x3 +#define origPB x4 +#define pC x5 +#define LDC x6 +#define temp x7 +#define counterL x8 +#define counterI x9 +#define counterJ x10 +#define pB x11 +#define pCRow0 x12 +#define pCRow1 x13 +#define pCRow2 x14 +#define pA x15 + +/* + * Cortex-A72 DGEMM 6x8 microkernel (mr=6, nr=8). + * + * Register map (BLIS bli_dgemm_armv8a_asm_6x8 inspired): + * A: v0,v1,v2 = 6 packed doubles / k-step (48 B) + * B: v3,v4,v5,v6 = 8 packed doubles / k-step (64 B) + * alpha: d7 / v7.d[0] + * C col j: v(8+3*j), v(9+3*j), v(10+3*j) -> v8..v31 (24 accumulators) + * + * B/FLOP = 1.17 vs 8x4's 1.50; FLOPs/A-byte = 2.0 (NR/4). + */ +#define A_PRE_SIZE 2560 +#define B_PRE_SIZE 448 +#define C_PRE_SIZE 128 + +#define alpha0 d7 +#define alphaV0 v7.d[0] + +.macro INIT6x8 + fmov d8, xzr + fmov d9, xzr + fmov d10, xzr + fmov d11, xzr + fmov d12, xzr + fmov d13, xzr + fmov d14, xzr + fmov d15, xzr + fmov d16, xzr + fmov d17, xzr + fmov d18, xzr + fmov d19, xzr + fmov d20, xzr + fmov d21, xzr + fmov d22, xzr + fmov d23, xzr + fmov d24, xzr + fmov d25, xzr + fmov d26, xzr + fmov d27, xzr + fmov d28, xzr + fmov d29, xzr + fmov d30, xzr + fmov d31, xzr +.endm + +.macro KERNEL6x8_SUB + ld1 {v0.2d, v1.2d, v2.2d}, [pA], #48 + ld1 {v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64 + prfm PLDL1KEEP, [pA, #A_PRE_SIZE] + prfm PLDL1KEEP, [pB, #B_PRE_SIZE] + fmla v8.2d, v0.2d, v3.d[0] + fmla v9.2d, v1.2d, v3.d[0] + fmla v10.2d, v2.2d, v3.d[0] + fmla v11.2d, v0.2d, v3.d[1] + fmla v12.2d, v1.2d, v3.d[1] + fmla v13.2d, v2.2d, v3.d[1] + fmla v14.2d, v0.2d, v4.d[0] + fmla v15.2d, v1.2d, v4.d[0] + fmla v16.2d, v2.2d, v4.d[0] + fmla v17.2d, v0.2d, v4.d[1] + fmla v18.2d, v1.2d, v4.d[1] + fmla v19.2d, v2.2d, v4.d[1] + fmla v20.2d, v0.2d, v5.d[0] + fmla v21.2d, v1.2d, v5.d[0] + fmla v22.2d, v2.2d, v5.d[0] + fmla v23.2d, v0.2d, v5.d[1] + fmla v24.2d, v1.2d, v5.d[1] + fmla v25.2d, v2.2d, v5.d[1] + fmla v26.2d, v0.2d, v6.d[0] + fmla v27.2d, v1.2d, v6.d[0] + fmla v28.2d, v2.2d, v6.d[0] + fmla v29.2d, v0.2d, v6.d[1] + fmla v30.2d, v1.2d, v6.d[1] + fmla v31.2d, v2.2d, v6.d[1] +.endm + +.macro SAVE6x8 + mov pCRow1, pCRow0 + prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE] + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v8.2d, alphaV0 + fmla v1.2d, v9.2d, alphaV0 + fmla v2.2d, v10.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE] + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v11.2d, alphaV0 + fmla v1.2d, v12.2d, alphaV0 + fmla v2.2d, v13.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE] + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v14.2d, alphaV0 + fmla v1.2d, v15.2d, alphaV0 + fmla v2.2d, v16.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE] + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v17.2d, alphaV0 + fmla v1.2d, v18.2d, alphaV0 + fmla v2.2d, v19.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE] + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v20.2d, alphaV0 + fmla v1.2d, v21.2d, alphaV0 + fmla v2.2d, v22.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE] + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v23.2d, alphaV0 + fmla v1.2d, v24.2d, alphaV0 + fmla v2.2d, v25.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE] + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v26.2d, alphaV0 + fmla v1.2d, v27.2d, alphaV0 + fmla v2.2d, v28.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE] + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v29.2d, alphaV0 + fmla v1.2d, v30.2d, alphaV0 + fmla v2.2d, v31.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + add pCRow0, pCRow0, #48 +.endm + +.macro INIT4x8 + fmov d16, xzr + fmov d17, xzr + fmov d18, xzr + fmov d19, xzr + fmov d20, xzr + fmov d21, xzr + fmov d22, xzr + fmov d23, xzr + fmov d24, xzr + fmov d25, xzr + fmov d26, xzr + fmov d27, xzr + fmov d28, xzr + fmov d29, xzr + fmov d30, xzr + fmov d31, xzr +.endm + +.macro KERNEL4x8_SUB + ld1 {v0.2d, v1.2d}, [pA], #32 + ld1 {v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64 + fmla v16.2d, v0.2d, v3.d[0] + fmla v17.2d, v1.2d, v3.d[0] + fmla v18.2d, v0.2d, v3.d[1] + fmla v19.2d, v1.2d, v3.d[1] + fmla v20.2d, v0.2d, v4.d[0] + fmla v21.2d, v1.2d, v4.d[0] + fmla v22.2d, v0.2d, v4.d[1] + fmla v23.2d, v1.2d, v4.d[1] + fmla v24.2d, v0.2d, v5.d[0] + fmla v25.2d, v1.2d, v5.d[0] + fmla v26.2d, v0.2d, v5.d[1] + fmla v27.2d, v1.2d, v5.d[1] + fmla v28.2d, v0.2d, v6.d[0] + fmla v29.2d, v1.2d, v6.d[0] + fmla v30.2d, v0.2d, v6.d[1] + fmla v31.2d, v1.2d, v6.d[1] +.endm + +.macro SAVE4x8 + mov pCRow1, pCRow0 + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v16.2d, alphaV0 + fmla v1.2d, v17.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v18.2d, alphaV0 + fmla v1.2d, v19.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v20.2d, alphaV0 + fmla v1.2d, v21.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v22.2d, alphaV0 + fmla v1.2d, v23.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v24.2d, alphaV0 + fmla v1.2d, v25.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v26.2d, alphaV0 + fmla v1.2d, v27.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v28.2d, alphaV0 + fmla v1.2d, v29.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v30.2d, alphaV0 + fmla v1.2d, v31.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + add pCRow0, pCRow0, #32 +.endm + +.macro INIT2x8 + fmov d16, xzr + fmov d18, xzr + fmov d20, xzr + fmov d22, xzr + fmov d24, xzr + fmov d26, xzr + fmov d28, xzr + fmov d30, xzr +.endm + +.macro KERNEL2x8_SUB + ld1 {v0.2d}, [pA], #16 + ld1 {v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64 + fmla v16.2d, v0.2d, v3.d[0] + fmla v18.2d, v0.2d, v3.d[1] + fmla v20.2d, v0.2d, v4.d[0] + fmla v22.2d, v0.2d, v4.d[1] + fmla v24.2d, v0.2d, v5.d[0] + fmla v26.2d, v0.2d, v5.d[1] + fmla v28.2d, v0.2d, v6.d[0] + fmla v30.2d, v0.2d, v6.d[1] +.endm + +.macro SAVE2x8 + mov pCRow1, pCRow0 + ldr q0, [pCRow1] + fmla v0.2d, v16.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v18.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v20.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v22.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v24.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v26.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v28.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v30.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + add pCRow0, pCRow0, #16 +.endm + +.macro INIT1x8 + fmov d16, xzr + fmov d17, xzr + fmov d18, xzr + fmov d19, xzr + fmov d20, xzr + fmov d21, xzr + fmov d22, xzr + fmov d23, xzr +.endm + +.macro KERNEL1x8_SUB + ldr d0, [pA], #8 + ld1 {v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64 + fmadd d16, d0, d3, d16 + fmul d1, d0, v3.d[1] + fadd d17, d17, d1 + fmadd d18, d0, d4, d18 + fmul d1, d0, v4.d[1] + fadd d19, d19, d1 + fmadd d20, d0, d5, d20 + fmul d1, d0, v5.d[1] + fadd d21, d21, d1 + fmadd d22, d0, d6, d22 + fmul d1, d0, v6.d[1] + fadd d23, d23, d1 +.endm + +.macro SAVE1x8 + mov pCRow1, pCRow0 + ldr d0, [pCRow1] + fmadd d0, d16, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d17, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d18, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d19, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d20, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d21, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d22, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d23, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + add pCRow0, pCRow0, #8 +.endm + +.macro INIT6x4 + fmov d8, xzr + fmov d9, xzr + fmov d10, xzr + fmov d11, xzr + fmov d12, xzr + fmov d13, xzr + fmov d14, xzr + fmov d15, xzr + fmov d16, xzr + fmov d17, xzr + fmov d18, xzr + fmov d19, xzr +.endm + +.macro KERNEL6x4_SUB + ld1 {v0.2d, v1.2d, v2.2d}, [pA], #48 + ld1 {v3.2d, v4.2d}, [pB], #32 + fmla v8.2d, v0.2d, v3.d[0] + fmla v9.2d, v1.2d, v3.d[0] + fmla v10.2d, v2.2d, v3.d[0] + fmla v11.2d, v0.2d, v3.d[1] + fmla v12.2d, v1.2d, v3.d[1] + fmla v13.2d, v2.2d, v3.d[1] + fmla v14.2d, v0.2d, v4.d[0] + fmla v15.2d, v1.2d, v4.d[0] + fmla v16.2d, v2.2d, v4.d[0] + fmla v17.2d, v0.2d, v4.d[1] + fmla v18.2d, v1.2d, v4.d[1] + fmla v19.2d, v2.2d, v4.d[1] +.endm + +.macro SAVE6x4 + mov pCRow1, pCRow0 + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v8.2d, alphaV0 + fmla v1.2d, v9.2d, alphaV0 + fmla v2.2d, v10.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v11.2d, alphaV0 + fmla v1.2d, v12.2d, alphaV0 + fmla v2.2d, v13.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v14.2d, alphaV0 + fmla v1.2d, v15.2d, alphaV0 + fmla v2.2d, v16.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v17.2d, alphaV0 + fmla v1.2d, v18.2d, alphaV0 + fmla v2.2d, v19.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + add pCRow0, pCRow0, #48 +.endm + +.macro INIT6x2 + fmov d8, xzr + fmov d9, xzr + fmov d10, xzr + fmov d11, xzr + fmov d12, xzr + fmov d13, xzr +.endm + +.macro KERNEL6x2_SUB + ld1 {v0.2d, v1.2d, v2.2d}, [pA], #48 + ld1 {v3.2d}, [pB], #16 + fmla v8.2d, v0.2d, v3.d[0] + fmla v9.2d, v1.2d, v3.d[0] + fmla v10.2d, v2.2d, v3.d[0] + fmla v11.2d, v0.2d, v3.d[1] + fmla v12.2d, v1.2d, v3.d[1] + fmla v13.2d, v2.2d, v3.d[1] +.endm + +.macro SAVE6x2 + mov pCRow1, pCRow0 + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v8.2d, alphaV0 + fmla v1.2d, v9.2d, alphaV0 + fmla v2.2d, v10.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + fmla v0.2d, v11.2d, alphaV0 + fmla v1.2d, v12.2d, alphaV0 + fmla v2.2d, v13.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow1] + add pCRow0, pCRow0, #48 +.endm + +.macro INIT6x1 + fmov d8, xzr + fmov d9, xzr + fmov d10, xzr +.endm + +.macro KERNEL6x1_SUB + ld1 {v0.2d, v1.2d, v2.2d}, [pA], #48 + ldr d3, [pB], #8 + fmla v8.2d, v0.2d, v3.d[0] + fmla v9.2d, v1.2d, v3.d[0] + fmla v10.2d, v2.2d, v3.d[0] +.endm + +.macro SAVE6x1 + ld1 {v0.2d, v1.2d, v2.2d}, [pCRow0] + fmla v0.2d, v8.2d, alphaV0 + fmla v1.2d, v9.2d, alphaV0 + fmla v2.2d, v10.2d, alphaV0 + st1 {v0.2d, v1.2d, v2.2d}, [pCRow0] + add pCRow0, pCRow0, #48 +.endm + + +.macro INIT4x4 + fmov d16, xzr + fmov d17, xzr + fmov d18, xzr + fmov d19, xzr + fmov d20, xzr + fmov d21, xzr + fmov d22, xzr + fmov d23, xzr +.endm +.macro KERNEL4x4_SUB + ld1 {v0.2d, v1.2d}, [pA], #32 + ld1 {v3.2d, v4.2d}, [pB], #32 + fmla v16.2d, v0.2d, v3.d[0] + fmla v17.2d, v1.2d, v3.d[0] + fmla v18.2d, v0.2d, v3.d[1] + fmla v19.2d, v1.2d, v3.d[1] + fmla v20.2d, v0.2d, v4.d[0] + fmla v21.2d, v1.2d, v4.d[0] + fmla v22.2d, v0.2d, v4.d[1] + fmla v23.2d, v1.2d, v4.d[1] +.endm +.macro SAVE4x4 + mov pCRow1, pCRow0 + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v16.2d, alphaV0 + fmla v1.2d, v17.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v18.2d, alphaV0 + fmla v1.2d, v19.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v20.2d, alphaV0 + fmla v1.2d, v21.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v22.2d, alphaV0 + fmla v1.2d, v23.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow0, pCRow0, #32 +.endm + +.macro INIT2x4 + fmov d16, xzr + fmov d18, xzr + fmov d20, xzr + fmov d22, xzr +.endm +.macro KERNEL2x4_SUB + ld1 {v0.2d}, [pA], #16 + ld1 {v3.2d, v4.2d}, [pB], #32 + fmla v16.2d, v0.2d, v3.d[0] + fmla v18.2d, v0.2d, v3.d[1] + fmla v20.2d, v0.2d, v4.d[0] + fmla v22.2d, v0.2d, v4.d[1] +.endm +.macro SAVE2x4 + mov pCRow1, pCRow0 + ldr q0, [pCRow1] + fmla v0.2d, v16.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v18.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v20.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v22.2d, alphaV0 + str q0, [pCRow1] + add pCRow0, pCRow0, #16 +.endm + +.macro INIT1x4 + fmov d16, xzr + fmov d17, xzr + fmov d18, xzr + fmov d19, xzr +.endm +.macro KERNEL1x4_SUB + ldr d0, [pA], #8 + ld1 {v3.2d, v4.2d}, [pB], #32 + fmadd d16, d0, d3, d16 + fmul d1, d0, v3.d[1] + fadd d17, d17, d1 + fmadd d18, d0, d4, d18 + fmul d1, d0, v4.d[1] + fadd d19, d19, d1 +.endm +.macro SAVE1x4 + mov pCRow1, pCRow0 + ldr d0, [pCRow1] + fmadd d0, d16, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d17, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d18, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d19, alpha0, d0 + str d0, [pCRow1] + add pCRow0, pCRow0, #8 +.endm + +.macro INIT4x2 + fmov d16, xzr + fmov d17, xzr + fmov d18, xzr + fmov d19, xzr +.endm +.macro KERNEL4x2_SUB + ld1 {v0.2d, v1.2d}, [pA], #32 + ld1 {v3.2d}, [pB], #16 + fmla v16.2d, v0.2d, v3.d[0] + fmla v17.2d, v1.2d, v3.d[0] + fmla v18.2d, v0.2d, v3.d[1] + fmla v19.2d, v1.2d, v3.d[1] +.endm +.macro SAVE4x2 + mov pCRow1, pCRow0 + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v16.2d, alphaV0 + fmla v1.2d, v17.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow1, pCRow1, LDC + ld1 {v0.2d, v1.2d}, [pCRow1] + fmla v0.2d, v18.2d, alphaV0 + fmla v1.2d, v19.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow1] + add pCRow0, pCRow0, #32 +.endm + +.macro INIT2x2 + fmov d16, xzr + fmov d18, xzr +.endm +.macro KERNEL2x2_SUB + ld1 {v0.2d}, [pA], #16 + ld1 {v3.2d}, [pB], #16 + fmla v16.2d, v0.2d, v3.d[0] + fmla v18.2d, v0.2d, v3.d[1] +.endm +.macro SAVE2x2 + mov pCRow1, pCRow0 + ldr q0, [pCRow1] + fmla v0.2d, v16.2d, alphaV0 + str q0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr q0, [pCRow1] + fmla v0.2d, v18.2d, alphaV0 + str q0, [pCRow1] + add pCRow0, pCRow0, #16 +.endm + +.macro INIT1x2 + fmov d16, xzr + fmov d17, xzr +.endm +.macro KERNEL1x2_SUB + ldr d0, [pA], #8 + ld1 {v3.2d}, [pB], #16 + fmadd d16, d0, d3, d16 + fmul d1, d0, v3.d[1] + fadd d17, d17, d1 +.endm +.macro SAVE1x2 + mov pCRow1, pCRow0 + ldr d0, [pCRow1] + fmadd d0, d16, alpha0, d0 + str d0, [pCRow1] + add pCRow1, pCRow1, LDC + ldr d0, [pCRow1] + fmadd d0, d17, alpha0, d0 + str d0, [pCRow1] + add pCRow0, pCRow0, #8 +.endm + +.macro INIT4x1 + fmov d16, xzr + fmov d17, xzr +.endm +.macro KERNEL4x1_SUB + ld1 {v0.2d, v1.2d}, [pA], #32 + ldr d3, [pB], #8 + fmla v16.2d, v0.2d, v3.d[0] + fmla v17.2d, v1.2d, v3.d[0] +.endm +.macro SAVE4x1 + ld1 {v0.2d, v1.2d}, [pCRow0] + fmla v0.2d, v16.2d, alphaV0 + fmla v1.2d, v17.2d, alphaV0 + st1 {v0.2d, v1.2d}, [pCRow0] + add pCRow0, pCRow0, #32 +.endm + +.macro INIT2x1 + fmov d16, xzr +.endm +.macro KERNEL2x1_SUB + ld1 {v0.2d}, [pA], #16 + ldr d3, [pB], #8 + fmla v16.2d, v0.2d, v3.d[0] +.endm +.macro SAVE2x1 + ldr q0, [pCRow0] + fmla v0.2d, v16.2d, alphaV0 + str q0, [pCRow0] + add pCRow0, pCRow0, #16 +.endm + +.macro INIT1x1 + fmov d16, xzr +.endm +.macro KERNEL1x1_SUB + ldr d0, [pA], #8 + ldr d3, [pB], #8 + fmadd d16, d0, d3, d16 +.endm +.macro SAVE1x1 + ldr d0, [pCRow0] + fmadd d0, d16, alpha0, d0 + str d0, [pCRow0] + add pCRow0, pCRow0, #8 +.endm + + PROLOGUE + .align 5 + add sp, sp, #-(11 * 16) + stp d8, d9, [sp, #(0 * 16)] + stp d10, d11, [sp, #(1 * 16)] + stp d12, d13, [sp, #(2 * 16)] + stp d14, d15, [sp, #(3 * 16)] + stp d16, d17, [sp, #(4 * 16)] + stp x18, x19, [sp, #(5 * 16)] + stp x20, x21, [sp, #(6 * 16)] + stp x22, x23, [sp, #(7 * 16)] + stp x24, x25, [sp, #(8 * 16)] + stp x26, x27, [sp, #(9 * 16)] + str x28, [sp, #(10 * 16)] + + fmov alpha0, d0 + lsl LDC, LDC, #3 + + mov counterJ, origN + asr counterJ, counterJ, #3 + cbz counterJ, .L6x8_L4_BEGIN + +.L6x8_L8_BEGIN: + mov pCRow0, pC + add pC, pC, LDC, lsl #3 + mov pA, origPA + + mov temp, #6 + udiv counterI, origM, temp + cbz counterI, .L6x8_L8_Mrem + +.L6x8_L8_M6: + INIT6x8 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L8_M6s +.L6x8_L8_M6l: + KERNEL6x8_SUB + subs counterL, counterL, #1 + bgt .L6x8_L8_M6l +.L6x8_L8_M6s: + SAVE6x8 + subs counterI, counterI, #1 + bgt .L6x8_L8_M6 + +.L6x8_L8_Mrem: + mov temp, #6 + udiv counterI, origM, temp + msub counterI, counterI, temp, origM + cmp counterI, #4 + blt .L6x8_L8_M2c + INIT4x8 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L8_M4s +.L6x8_L8_M4l: + KERNEL4x8_SUB + subs counterL, counterL, #1 + bgt .L6x8_L8_M4l +.L6x8_L8_M4s: + SAVE4x8 + sub counterI, counterI, #4 +.L6x8_L8_M2c: + cmp counterI, #2 + blt .L6x8_L8_M1c + INIT2x8 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L8_M2s +.L6x8_L8_M2l: + KERNEL2x8_SUB + subs counterL, counterL, #1 + bgt .L6x8_L8_M2l +.L6x8_L8_M2s: + SAVE2x8 + sub counterI, counterI, #2 +.L6x8_L8_M1c: + cbz counterI, .L6x8_L8_END + INIT1x8 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L8_M1s +.L6x8_L8_M1l: + KERNEL1x8_SUB + subs counterL, counterL, #1 + bgt .L6x8_L8_M1l +.L6x8_L8_M1s: + SAVE1x8 + +.L6x8_L8_END: + lsl temp, origK, #6 /* + K*8*8 bytes */ + add origPB, origPB, temp + subs counterJ, counterJ, #1 + bgt .L6x8_L8_BEGIN + +.L6x8_L4_BEGIN: + tst origN, #7 + beq .L6x8_L999 + tst origN, #4 + beq .L6x8_L2_BEGIN + + mov pCRow0, pC + add pC, pC, LDC, lsl #2 + mov pA, origPA + + mov temp, #6 + udiv counterI, origM, temp + cbz counterI, .L6x8_L4_Mrem +.L6x8_L4_M6: + INIT6x4 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L4_M6s +.L6x8_L4_M6l: + KERNEL6x4_SUB + subs counterL, counterL, #1 + bgt .L6x8_L4_M6l +.L6x8_L4_M6s: + SAVE6x4 + subs counterI, counterI, #1 + bgt .L6x8_L4_M6 +.L6x8_L4_Mrem: + mov temp, #6 + udiv counterI, origM, temp + msub counterI, counterI, temp, origM + cmp counterI, #4 + blt .L6x8_L4_M2c + INIT4x4 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L4_M4s +.L6x8_L4_M4l: + KERNEL4x4_SUB + subs counterL, counterL, #1 + bgt .L6x8_L4_M4l +.L6x8_L4_M4s: + SAVE4x4 + sub counterI, counterI, #4 +.L6x8_L4_M2c: + cmp counterI, #2 + blt .L6x8_L4_M1c + INIT2x4 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L4_M2s +.L6x8_L4_M2l: + KERNEL2x4_SUB + subs counterL, counterL, #1 + bgt .L6x8_L4_M2l +.L6x8_L4_M2s: + SAVE2x4 + sub counterI, counterI, #2 +.L6x8_L4_M1c: + cbz counterI, .L6x8_L4_END + INIT1x4 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L4_M1s +.L6x8_L4_M1l: + KERNEL1x4_SUB + subs counterL, counterL, #1 + bgt .L6x8_L4_M1l +.L6x8_L4_M1s: + SAVE1x4 +.L6x8_L4_END: + add origPB, origPB, origK, lsl #5 /* +K*4*8 */ + +.L6x8_L2_BEGIN: + tst origN, #2 + beq .L6x8_L1_BEGIN + mov pCRow0, pC + add pC, pC, LDC, lsl #1 + mov pA, origPA + mov temp, #6 + udiv counterI, origM, temp + cbz counterI, .L6x8_L2_Mrem +.L6x8_L2_M6: + INIT6x2 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L2_M6s +.L6x8_L2_M6l: + KERNEL6x2_SUB + subs counterL, counterL, #1 + bgt .L6x8_L2_M6l +.L6x8_L2_M6s: + SAVE6x2 + subs counterI, counterI, #1 + bgt .L6x8_L2_M6 +.L6x8_L2_Mrem: + mov temp, #6 + udiv counterI, origM, temp + msub counterI, counterI, temp, origM + cmp counterI, #4 + blt .L6x8_L2_M2c + INIT4x2 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L2_M4s +.L6x8_L2_M4l: + KERNEL4x2_SUB + subs counterL, counterL, #1 + bgt .L6x8_L2_M4l +.L6x8_L2_M4s: + SAVE4x2 + sub counterI, counterI, #4 +.L6x8_L2_M2c: + cmp counterI, #2 + blt .L6x8_L2_M1c + INIT2x2 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L2_M2s +.L6x8_L2_M2l: + KERNEL2x2_SUB + subs counterL, counterL, #1 + bgt .L6x8_L2_M2l +.L6x8_L2_M2s: + SAVE2x2 + sub counterI, counterI, #2 +.L6x8_L2_M1c: + cbz counterI, .L6x8_L2_END + INIT1x2 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L2_M1s +.L6x8_L2_M1l: + KERNEL1x2_SUB + subs counterL, counterL, #1 + bgt .L6x8_L2_M1l +.L6x8_L2_M1s: + SAVE1x2 +.L6x8_L2_END: + add origPB, origPB, origK, lsl #4 + +.L6x8_L1_BEGIN: + tst origN, #1 + beq .L6x8_L999 + mov pCRow0, pC + mov pA, origPA + mov temp, #6 + udiv counterI, origM, temp + cbz counterI, .L6x8_L1_Mrem +.L6x8_L1_M6: + INIT6x1 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L1_M6s +.L6x8_L1_M6l: + KERNEL6x1_SUB + subs counterL, counterL, #1 + bgt .L6x8_L1_M6l +.L6x8_L1_M6s: + SAVE6x1 + subs counterI, counterI, #1 + bgt .L6x8_L1_M6 +.L6x8_L1_Mrem: + mov temp, #6 + udiv counterI, origM, temp + msub counterI, counterI, temp, origM + cmp counterI, #4 + blt .L6x8_L1_M2c + INIT4x1 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L1_M4s +.L6x8_L1_M4l: + KERNEL4x1_SUB + subs counterL, counterL, #1 + bgt .L6x8_L1_M4l +.L6x8_L1_M4s: + SAVE4x1 + sub counterI, counterI, #4 +.L6x8_L1_M2c: + cmp counterI, #2 + blt .L6x8_L1_M1c + INIT2x1 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L1_M2s +.L6x8_L1_M2l: + KERNEL2x1_SUB + subs counterL, counterL, #1 + bgt .L6x8_L1_M2l +.L6x8_L1_M2s: + SAVE2x1 + sub counterI, counterI, #2 +.L6x8_L1_M1c: + cbz counterI, .L6x8_L999 + INIT1x1 + mov pB, origPB + mov counterL, origK + cbz counterL, .L6x8_L1_M1s +.L6x8_L1_M1l: + KERNEL1x1_SUB + subs counterL, counterL, #1 + bgt .L6x8_L1_M1l +.L6x8_L1_M1s: + SAVE1x1 + +.L6x8_L999: + mov x0, #0 + ldp d8, d9, [sp, #(0 * 16)] + ldp d10, d11, [sp, #(1 * 16)] + ldp d12, d13, [sp, #(2 * 16)] + ldp d14, d15, [sp, #(3 * 16)] + ldp d16, d17, [sp, #(4 * 16)] + ldp x18, x19, [sp, #(5 * 16)] + ldp x20, x21, [sp, #(6 * 16)] + ldp x22, x23, [sp, #(7 * 16)] + ldp x24, x25, [sp, #(8 * 16)] + ldp x26, x27, [sp, #(9 * 16)] + ldr x28, [sp, #(10 * 16)] + add sp, sp, #(11 * 16) + ret + EPILOGUE diff --git a/kernel/arm64/gemm_ncopy_6_panel.c b/kernel/arm64/gemm_ncopy_6_panel.c new file mode 100644 index 0000000000..c9769cc89b --- /dev/null +++ b/kernel/arm64/gemm_ncopy_6_panel.c @@ -0,0 +1,94 @@ +/* True MR=6 no-transpose pack (INCOPY) for OpenBLAS DGEMM 6xN. + * + * Used when A is transposed (TN/TT/...): INCOPY(m=Kc, n=Mc). + * Packs 6 columns at a time into contiguous 6-wide micropanels along m + * (same layout as loongarch64/gemm_ncopy_6.prefx.c). + * + * Stock generic/gemm_ncopy_6.c packs as 4-wide and is wrong for 6x8. + */ +#include "common.h" + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b) +{ + BLASLONG i, j; + FLOAT *aoffset, *aoffset1, *aoffset2, *aoffset3, *aoffset4, *aoffset5, *aoffset6; + FLOAT *boffset; + FLOAT c1, c2, c3, c4, c5, c6; + + aoffset = a; + boffset = b; + + j = n / 6; + if (j > 0) { + do { + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset5 = aoffset4 + lda; + aoffset6 = aoffset5 + lda; + aoffset += 6 * lda; + + i = m; + if (i > 0) { + do { + c1 = *(aoffset1); + c2 = *(aoffset2); + c3 = *(aoffset3); + c4 = *(aoffset4); + c5 = *(aoffset5); + c6 = *(aoffset6); + aoffset1++; aoffset2++; aoffset3++; + aoffset4++; aoffset5++; aoffset6++; + *(boffset + 0) = c1; + *(boffset + 1) = c2; + *(boffset + 2) = c3; + *(boffset + 3) = c4; + *(boffset + 4) = c5; + *(boffset + 5) = c6; + boffset += 6; + i--; + } while (i > 0); + } + j--; + } while (j > 0); + } + + /* n remainder: 4, 2, 1 column groups (matches ukernel N-edge order for A-side) */ + { + BLASLONG nr = n - (n / 6) * 6; + if (nr >= 4) { + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset += 4 * lda; + for (i = 0; i < m; i++) { + boffset[0] = *aoffset1++; + boffset[1] = *aoffset2++; + boffset[2] = *aoffset3++; + boffset[3] = *aoffset4++; + boffset += 4; + } + nr -= 4; + } + if (nr >= 2) { + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset += 2 * lda; + for (i = 0; i < m; i++) { + boffset[0] = *aoffset1++; + boffset[1] = *aoffset2++; + boffset += 2; + } + nr -= 2; + } + if (nr >= 1) { + aoffset1 = aoffset; + for (i = 0; i < m; i++) { + *boffset++ = *aoffset1++; + } + } + } + return 0; +} diff --git a/kernel/arm64/gemm_tcopy_6_panel.c b/kernel/arm64/gemm_tcopy_6_panel.c new file mode 100644 index 0000000000..b8dd7ab15e --- /dev/null +++ b/kernel/arm64/gemm_tcopy_6_panel.c @@ -0,0 +1,63 @@ +/* True MR=6 transpose pack (ITCOPY) for OpenBLAS DGEMM 6xN. + * + * Call signature matches gemm_tcopy_*: (m, n, a, lda, b) where for NN + * ICOPY uses ITCOPY(m=Kc, n=Mc). Output is Goto-style micropanels: + * for each panel of 6 rows: Kc contiguous packs of 6 doubles. + * Remainders after full panels: 4, then 2, then 1 (kernel cascade). + * + * Stock generic/gemm_tcopy_6.c is a clone of tcopy_4 (4+2) and does NOT + * match a contiguous 6-wide ukernel — do not use it for 6x8. + */ +#include "common.h" + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b) +{ + BLASLONG j, js; + FLOAT *b_ptr = b; + BLASLONG n6 = n / 6; + BLASLONG nr = n - n6 * 6; + + for (js = 0; js < n6; js++) { + FLOAT *a_row = a + js * 6; + for (j = 0; j < m; j++) { + FLOAT *ap = a_row + j * lda; + b_ptr[0] = ap[0]; + b_ptr[1] = ap[1]; + b_ptr[2] = ap[2]; + b_ptr[3] = ap[3]; + b_ptr[4] = ap[4]; + b_ptr[5] = ap[5]; + b_ptr += 6; + } + } + + FLOAT *a_row = a + n6 * 6; + if (nr >= 4) { + for (j = 0; j < m; j++) { + FLOAT *ap = a_row + j * lda; + b_ptr[0] = ap[0]; + b_ptr[1] = ap[1]; + b_ptr[2] = ap[2]; + b_ptr[3] = ap[3]; + b_ptr += 4; + } + a_row += 4; + nr -= 4; + } + if (nr >= 2) { + for (j = 0; j < m; j++) { + FLOAT *ap = a_row + j * lda; + b_ptr[0] = ap[0]; + b_ptr[1] = ap[1]; + b_ptr += 2; + } + a_row += 2; + nr -= 2; + } + if (nr >= 1) { + for (j = 0; j < m; j++) { + *b_ptr++ = a_row[j * lda]; + } + } + return 0; +} diff --git a/kernel/arm64/trsm_kernel_LN_UNROLLM6.c b/kernel/arm64/trsm_kernel_LN_UNROLLM6.c new file mode 100644 index 0000000000..44fd8e8619 --- /dev/null +++ b/kernel/arm64/trsm_kernel_LN_UNROLLM6.c @@ -0,0 +1,346 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include "common.h" + +/* UNROLL_M=6-aware TRSM kernel. + * Stock generic/trsm_kernel_*.c assumes power-of-2 UNROLL_M via + * GEMM_UNROLL_M_SHIFT (incorrectly set to 2 for M=6) and bit masks + * (m & (M-1)). That overruns packed buffers and corrupts the heap — + * reproduces as free()/double-free in HPL panel factor (dtrsm). + * Pattern mirrors loongarch64/trsm_kernel_*_UNROLLN6.c for N=6. + */ + +static FLOAT dm1 = -1.; + +#ifdef CONJ +#define GEMM_KERNEL GEMM_KERNEL_L +#else +#define GEMM_KERNEL GEMM_KERNEL_N +#endif + +#if GEMM_DEFAULT_UNROLL_M == 1 +#define GEMM_UNROLL_M_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 2 +#define GEMM_UNROLL_M_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 4 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 6 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 8 +#define GEMM_UNROLL_M_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 16 +#define GEMM_UNROLL_M_SHIFT 4 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 1 +#define GEMM_UNROLL_N_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 2 +#define GEMM_UNROLL_N_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 4 +#define GEMM_UNROLL_N_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 8 +#define GEMM_UNROLL_N_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 16 +#define GEMM_UNROLL_N_SHIFT 4 +#endif + +#ifndef COMPLEX + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa, bb; + + int i, j, k; + + a += (m - 1) * m; + b += (m - 1) * n; + + for (i = m - 1; i >= 0; i--) { + + aa = *(a + i); + + for (j = 0; j < n; j ++) { + bb = *(c + i + j * ldc); + bb *= aa; + *b = bb; + *(c + i + j * ldc) = bb; + b ++; + + for (k = 0; k < i; k ++){ + *(c + k + j * ldc) -= bb * *(a + k); + } + + } + a -= m; + b -= 2 * n; + } + +} + +#else + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa1, aa2; + FLOAT bb1, bb2; + FLOAT cc1, cc2; + + int i, j, k; + + ldc *= 2; + a += (m - 1) * m * 2; + b += (m - 1) * n * 2; + + for (i = m - 1; i >= 0; i--) { + + aa1 = *(a + i * 2 + 0); + aa2 = *(a + i * 2 + 1); + + for (j = 0; j < n; j ++) { + bb1 = *(c + i * 2 + 0 + j * ldc); + bb2 = *(c + i * 2 + 1 + j * ldc); + +#ifndef CONJ + cc1 = aa1 * bb1 - aa2 * bb2; + cc2 = aa1 * bb2 + aa2 * bb1; +#else + cc1 = aa1 * bb1 + aa2 * bb2; + cc2 = aa1 * bb2 - aa2 * bb1; +#endif + + + *(b + 0) = cc1; + *(b + 1) = cc2; + *(c + i * 2 + 0 + j * ldc) = cc1; + *(c + i * 2 + 1 + j * ldc) = cc2; + b += 2; + + for (k = 0; k < i; k ++){ +#ifndef CONJ + *(c + k * 2 + 0 + j * ldc) -= cc1 * *(a + k * 2 + 0) - cc2 * *(a + k * 2 + 1); + *(c + k * 2 + 1 + j * ldc) -= cc1 * *(a + k * 2 + 1) + cc2 * *(a + k * 2 + 0); +#else + *(c + k * 2 + 0 + j * ldc) -= cc1 * *(a + k * 2 + 0) + cc2 * *(a + k * 2 + 1); + *(c + k * 2 + 1 + j * ldc) -= - cc1 * *(a + k * 2 + 1) + cc2 * *(a + k * 2 + 0); +#endif + } + + } + a -= m * 2; + b -= 4 * n; + } + +} + +#endif + + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, +#ifdef COMPLEX + FLOAT dummy2, +#endif + FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ + + BLASLONG i, j; + FLOAT *aa, *cc; + BLASLONG kk; + BLASLONG mmodM = m - (m / GEMM_UNROLL_M) * GEMM_UNROLL_M; + +#if 0 + fprintf(stderr, "TRSM KERNEL LN : m = %3ld n = %3ld k = %3ld offset = %3ld\n", + m, n, k, offset); +#endif + + j = (n >> GEMM_UNROLL_N_SHIFT); + + while (j > 0) { + + kk = m + offset; + + if (mmodM) { + for (i = 1; i < GEMM_UNROLL_M; i *= 2){ + if (mmodM & i) { + aa = a + ((m & ~(i - 1)) - i) * k * COMPSIZE; + cc = c + ((m & ~(i - 1)) - i) * COMPSIZE; + + if (k - kk > 0) { + GEMM_KERNEL(i, GEMM_UNROLL_N, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + i * kk * COMPSIZE, + b + GEMM_UNROLL_N * kk * COMPSIZE, + cc, + ldc); + } + + solve(i, GEMM_UNROLL_N, + aa + (kk - i) * i * COMPSIZE, + b + (kk - i) * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + kk -= i; + } + } + } + + i = (m / GEMM_UNROLL_M); + if (i > 0) { + aa = a + ((m / GEMM_UNROLL_M) * GEMM_UNROLL_M - GEMM_UNROLL_M) * k * COMPSIZE; + cc = c + ((m / GEMM_UNROLL_M) * GEMM_UNROLL_M - GEMM_UNROLL_M) * COMPSIZE; + + do { + if (k - kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + GEMM_UNROLL_M * kk * COMPSIZE, + b + GEMM_UNROLL_N * kk * COMPSIZE, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, GEMM_UNROLL_N, + aa + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_M * COMPSIZE, + b + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa -= GEMM_UNROLL_M * k * COMPSIZE; + cc -= GEMM_UNROLL_M * COMPSIZE; + kk -= GEMM_UNROLL_M; + i --; + } while (i > 0); + } + + b += GEMM_UNROLL_N * k * COMPSIZE; + c += GEMM_UNROLL_N * ldc * COMPSIZE; + j --; + } + + if (n & (GEMM_UNROLL_N - 1)) { + + j = (GEMM_UNROLL_N >> 1); + while (j > 0) { + if (n & j) { + + kk = m + offset; + + if (mmodM) { + for (i = 1; i < GEMM_UNROLL_M; i *= 2){ + if (mmodM & i) { + aa = a + ((m & ~(i - 1)) - i) * k * COMPSIZE; + cc = c + ((m & ~(i - 1)) - i) * COMPSIZE; + + if (k - kk > 0) { + GEMM_KERNEL(i, j, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + i * kk * COMPSIZE, + b + j * kk * COMPSIZE, + cc, ldc); + } + + solve(i, j, + aa + (kk - i) * i * COMPSIZE, + b + (kk - i) * j * COMPSIZE, + cc, ldc); + + kk -= i; + } + } + } + + i = (m / GEMM_UNROLL_M); + if (i > 0) { + aa = a + ((m / GEMM_UNROLL_M) * GEMM_UNROLL_M - GEMM_UNROLL_M) * k * COMPSIZE; + cc = c + ((m / GEMM_UNROLL_M) * GEMM_UNROLL_M - GEMM_UNROLL_M) * COMPSIZE; + + do { + if (k - kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, j, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + GEMM_UNROLL_M * kk * COMPSIZE, + b + j * kk * COMPSIZE, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, j, + aa + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_M * COMPSIZE, + b + (kk - GEMM_UNROLL_M) * j * COMPSIZE, + cc, ldc); + + aa -= GEMM_UNROLL_M * k * COMPSIZE; + cc -= GEMM_UNROLL_M * COMPSIZE; + kk -= GEMM_UNROLL_M; + i --; + } while (i > 0); + } + + b += j * k * COMPSIZE; + c += j * ldc * COMPSIZE; + } + j >>= 1; + } + } + + return 0; +} diff --git a/kernel/arm64/trsm_kernel_LT_UNROLLM6.c b/kernel/arm64/trsm_kernel_LT_UNROLLM6.c new file mode 100644 index 0000000000..ba5d639c66 --- /dev/null +++ b/kernel/arm64/trsm_kernel_LT_UNROLLM6.c @@ -0,0 +1,330 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include "common.h" + +/* UNROLL_M=6-aware TRSM kernel. + * Stock generic/trsm_kernel_*.c assumes power-of-2 UNROLL_M via + * GEMM_UNROLL_M_SHIFT (incorrectly set to 2 for M=6) and bit masks + * (m & (M-1)). That overruns packed buffers and corrupts the heap — + * reproduces as free()/double-free in HPL panel factor (dtrsm). + * Pattern mirrors loongarch64/trsm_kernel_*_UNROLLN6.c for N=6. + */ + +static FLOAT dm1 = -1.; + +#ifdef CONJ +#define GEMM_KERNEL GEMM_KERNEL_L +#else +#define GEMM_KERNEL GEMM_KERNEL_N +#endif + +#if GEMM_DEFAULT_UNROLL_M == 1 +#define GEMM_UNROLL_M_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 2 +#define GEMM_UNROLL_M_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 4 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 6 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 8 +#define GEMM_UNROLL_M_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 16 +#define GEMM_UNROLL_M_SHIFT 4 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 1 +#define GEMM_UNROLL_N_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 2 +#define GEMM_UNROLL_N_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 4 +#define GEMM_UNROLL_N_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 8 +#define GEMM_UNROLL_N_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 16 +#define GEMM_UNROLL_N_SHIFT 4 +#endif + +#ifndef COMPLEX + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa, bb; + + int i, j, k; + + for (i = 0; i < m; i++) { + + aa = *(a + i); + + for (j = 0; j < n; j ++) { + bb = *(c + i + j * ldc); + bb *= aa; + *b = bb; + *(c + i + j * ldc) = bb; + b ++; + + for (k = i + 1; k < m; k ++){ + *(c + k + j * ldc) -= bb * *(a + k); + } + + } + a += m; + } +} + +#else + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa1, aa2; + FLOAT bb1, bb2; + FLOAT cc1, cc2; + + int i, j, k; + + ldc *= 2; + + for (i = 0; i < m; i++) { + + aa1 = *(a + i * 2 + 0); + aa2 = *(a + i * 2 + 1); + + for (j = 0; j < n; j ++) { + bb1 = *(c + i * 2 + 0 + j * ldc); + bb2 = *(c + i * 2 + 1 + j * ldc); + +#ifndef CONJ + cc1 = aa1 * bb1 - aa2 * bb2; + cc2 = aa1 * bb2 + aa2 * bb1; +#else + cc1 = aa1 * bb1 + aa2 * bb2; + cc2 = aa1 * bb2 - aa2 * bb1; +#endif + + *(b + 0) = cc1; + *(b + 1) = cc2; + *(c + i * 2 + 0 + j * ldc) = cc1; + *(c + i * 2 + 1 + j * ldc) = cc2; + b += 2; + + for (k = i + 1; k < m; k ++){ +#ifndef CONJ + *(c + k * 2 + 0 + j * ldc) -= cc1 * *(a + k * 2 + 0) - cc2 * *(a + k * 2 + 1); + *(c + k * 2 + 1 + j * ldc) -= cc1 * *(a + k * 2 + 1) + cc2 * *(a + k * 2 + 0); +#else + *(c + k * 2 + 0 + j * ldc) -= cc1 * *(a + k * 2 + 0) + cc2 * *(a + k * 2 + 1); + *(c + k * 2 + 1 + j * ldc) -= -cc1 * *(a + k * 2 + 1) + cc2 * *(a + k * 2 + 0); +#endif + } + + } + a += m * 2; + } +} + +#endif + + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, +#ifdef COMPLEX + FLOAT dummy2, +#endif + FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ + + FLOAT *aa, *cc; + BLASLONG kk; + BLASLONG mmodM = m - (m / GEMM_UNROLL_M) * GEMM_UNROLL_M; + BLASLONG i, j, jj; + +#if 0 + fprintf(stderr, "TRSM KERNEL LT : m = %3ld n = %3ld k = %3ld offset = %3ld\n", + m, n, k, offset); +#endif + + jj = 0; + + j = (n >> GEMM_UNROLL_N_SHIFT); + + while (j > 0) { + + kk = offset; + aa = a; + cc = c; + + i = (m / GEMM_UNROLL_M); + + while (i > 0) { + + if (kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, b, cc, ldc); + } + + solve(GEMM_UNROLL_M, GEMM_UNROLL_N, + aa + kk * GEMM_UNROLL_M * COMPSIZE, + b + kk * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + kk += GEMM_UNROLL_M; + i --; + } + + if (mmodM) { + i = 4; /* largest power-of-2 edge < UNROLL_M=6 */ + while (i > 0) { + if (mmodM & i) { + if (kk > 0) { + GEMM_KERNEL(i, GEMM_UNROLL_N, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, b, cc, ldc); + } + solve(i, GEMM_UNROLL_N, + aa + kk * i * COMPSIZE, + b + kk * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + kk += i; + } + i >>= 1; + } + } + + b += GEMM_UNROLL_N * k * COMPSIZE; + c += GEMM_UNROLL_N * ldc * COMPSIZE; + j --; + jj += GEMM_UNROLL_M; + } + + if (n & (GEMM_UNROLL_N - 1)) { + + j = (GEMM_UNROLL_N >> 1); + while (j > 0) { + if (n & j) { + + kk = offset; + aa = a; + cc = c; + + i = (m / GEMM_UNROLL_M); + + while (i > 0) { + if (kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, j, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, + b, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, j, + aa + kk * GEMM_UNROLL_M * COMPSIZE, + b + kk * j * COMPSIZE, cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + kk += GEMM_UNROLL_M; + i --; + } + + if (mmodM) { + i = 4; /* largest power-of-2 edge < UNROLL_M=6 */ + while (i > 0) { + if (mmodM & i) { + if (kk > 0) { + GEMM_KERNEL(i, j, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, + b, + cc, + ldc); + } + + solve(i, j, + aa + kk * i * COMPSIZE, + b + kk * j * COMPSIZE, cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + kk += i; + } + i >>= 1; + } + } + + b += j * k * COMPSIZE; + c += j * ldc * COMPSIZE; + } + j >>= 1; + } + } + + return 0; +} diff --git a/kernel/arm64/trsm_kernel_RN_UNROLLM6.c b/kernel/arm64/trsm_kernel_RN_UNROLLM6.c new file mode 100644 index 0000000000..85e3f48eb9 --- /dev/null +++ b/kernel/arm64/trsm_kernel_RN_UNROLLM6.c @@ -0,0 +1,328 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include "common.h" + +/* UNROLL_M=6-aware TRSM kernel. + * Stock generic/trsm_kernel_*.c assumes power-of-2 UNROLL_M via + * GEMM_UNROLL_M_SHIFT (incorrectly set to 2 for M=6) and bit masks + * (m & (M-1)). That overruns packed buffers and corrupts the heap — + * reproduces as free()/double-free in HPL panel factor (dtrsm). + * Pattern mirrors loongarch64/trsm_kernel_*_UNROLLN6.c for N=6. + */ + +static FLOAT dm1 = -1.; + +#ifdef CONJ +#define GEMM_KERNEL GEMM_KERNEL_R +#else +#define GEMM_KERNEL GEMM_KERNEL_N +#endif + +#if GEMM_DEFAULT_UNROLL_M == 1 +#define GEMM_UNROLL_M_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 2 +#define GEMM_UNROLL_M_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 4 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 6 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 8 +#define GEMM_UNROLL_M_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 16 +#define GEMM_UNROLL_M_SHIFT 4 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 1 +#define GEMM_UNROLL_N_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 2 +#define GEMM_UNROLL_N_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 4 +#define GEMM_UNROLL_N_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 8 +#define GEMM_UNROLL_N_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 16 +#define GEMM_UNROLL_N_SHIFT 4 +#endif + +#ifndef COMPLEX + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa, bb; + + int i, j, k; + + for (i = 0; i < n; i++) { + + bb = *(b + i); + + for (j = 0; j < m; j ++) { + aa = *(c + j + i * ldc); + aa *= bb; + *a = aa; + *(c + j + i * ldc) = aa; + a ++; + + for (k = i + 1; k < n; k ++){ + *(c + j + k * ldc) -= aa * *(b + k); + } + + } + b += n; + } +} + +#else + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa1, aa2; + FLOAT bb1, bb2; + FLOAT cc1, cc2; + + int i, j, k; + + ldc *= 2; + + for (i = 0; i < n; i++) { + + bb1 = *(b + i * 2 + 0); + bb2 = *(b + i * 2 + 1); + + for (j = 0; j < m; j ++) { + aa1 = *(c + j * 2 + 0 + i * ldc); + aa2 = *(c + j * 2 + 1 + i * ldc); + +#ifndef CONJ + cc1 = aa1 * bb1 - aa2 * bb2; + cc2 = aa1 * bb2 + aa2 * bb1; +#else + cc1 = aa1 * bb1 + aa2 * bb2; + cc2 = -aa1 * bb2 + aa2 * bb1; +#endif + + *(a + 0) = cc1; + *(a + 1) = cc2; + *(c + j * 2 + 0 + i * ldc) = cc1; + *(c + j * 2 + 1 + i * ldc) = cc2; + a += 2; + + for (k = i + 1; k < n; k ++){ +#ifndef CONJ + *(c + j * 2 + 0 + k * ldc) -= cc1 * *(b + k * 2 + 0) - cc2 * *(b + k * 2 + 1); + *(c + j * 2 + 1 + k * ldc) -= cc1 * *(b + k * 2 + 1) + cc2 * *(b + k * 2 + 0); +#else + *(c + j * 2 + 0 + k * ldc) -= cc1 * *(b + k * 2 + 0) + cc2 * *(b + k * 2 + 1); + *(c + j * 2 + 1 + k * ldc) -= - cc1 * *(b + k * 2 + 1) + cc2 * *(b + k * 2 + 0); +#endif + } + + } + b += n * 2; + } +} + +#endif + + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, +#ifdef COMPLEX + FLOAT dummy2, +#endif + FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ + + FLOAT *aa, *cc; + BLASLONG kk; + BLASLONG mmodM = m - (m / GEMM_UNROLL_M) * GEMM_UNROLL_M; + BLASLONG i, j, jj; + +#if 0 + fprintf(stderr, "TRSM RN KERNEL m = %3ld n = %3ld k = %3ld offset = %3ld\n", + m, n, k, offset); +#endif + + jj = 0; + j = (n >> GEMM_UNROLL_N_SHIFT); + kk = -offset; + + while (j > 0) { + + aa = a; + cc = c; + + i = (m / GEMM_UNROLL_M); + + if (i > 0) { + do { + if (kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, b, cc, ldc); + } + + solve(GEMM_UNROLL_M, GEMM_UNROLL_N, + aa + kk * GEMM_UNROLL_M * COMPSIZE, + b + kk * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + i --; + } while (i > 0); + } + + + if (mmodM) { + i = 4; /* largest power-of-2 edge < UNROLL_M=6 */ + while (i > 0) { + if (mmodM & i) { + if (kk > 0) { + GEMM_KERNEL(i, GEMM_UNROLL_N, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, b, cc, ldc); + } + solve(i, GEMM_UNROLL_N, + aa + kk * i * COMPSIZE, + b + kk * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + } + i >>= 1; + } + } + + kk += GEMM_UNROLL_N; + b += GEMM_UNROLL_N * k * COMPSIZE; + c += GEMM_UNROLL_N * ldc * COMPSIZE; + j --; + jj += GEMM_UNROLL_M; + } + + if (n & (GEMM_UNROLL_N - 1)) { + + j = (GEMM_UNROLL_N >> 1); + while (j > 0) { + if (n & j) { + + aa = a; + cc = c; + + i = (m / GEMM_UNROLL_M); + + while (i > 0) { + if (kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, j, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, + b, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, j, + aa + kk * GEMM_UNROLL_M * COMPSIZE, + b + kk * j * COMPSIZE, cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + i --; + } + + if (mmodM) { + i = 4; /* largest power-of-2 edge < UNROLL_M=6 */ + while (i > 0) { + if (mmodM & i) { + if (kk > 0) { + GEMM_KERNEL(i, j, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, + b, + cc, + ldc); + } + + solve(i, j, + aa + kk * i * COMPSIZE, + b + kk * j * COMPSIZE, cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + } + i >>= 1; + } + } + + b += j * k * COMPSIZE; + c += j * ldc * COMPSIZE; + kk += j; + } + j >>= 1; + } + } + + return 0; +} diff --git a/kernel/arm64/trsm_kernel_RT_UNROLLM6.c b/kernel/arm64/trsm_kernel_RT_UNROLLM6.c new file mode 100644 index 0000000000..ec7ea1419a --- /dev/null +++ b/kernel/arm64/trsm_kernel_RT_UNROLLM6.c @@ -0,0 +1,355 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include "common.h" + +/* UNROLL_M=6-aware TRSM kernel. + * Stock generic/trsm_kernel_*.c assumes power-of-2 UNROLL_M via + * GEMM_UNROLL_M_SHIFT (incorrectly set to 2 for M=6) and bit masks + * (m & (M-1)). That overruns packed buffers and corrupts the heap — + * reproduces as free()/double-free in HPL panel factor (dtrsm). + * Pattern mirrors loongarch64/trsm_kernel_*_UNROLLN6.c for N=6. + */ + +static FLOAT dm1 = -1.; + +#ifdef CONJ +#define GEMM_KERNEL GEMM_KERNEL_R +#else +#define GEMM_KERNEL GEMM_KERNEL_N +#endif + +#if GEMM_DEFAULT_UNROLL_M == 1 +#define GEMM_UNROLL_M_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 2 +#define GEMM_UNROLL_M_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 4 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 6 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + + +#if GEMM_DEFAULT_UNROLL_M == 8 +#define GEMM_UNROLL_M_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 16 +#define GEMM_UNROLL_M_SHIFT 4 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 1 +#define GEMM_UNROLL_N_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 2 +#define GEMM_UNROLL_N_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 4 +#define GEMM_UNROLL_N_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 8 +#define GEMM_UNROLL_N_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 16 +#define GEMM_UNROLL_N_SHIFT 4 +#endif + + +#ifndef COMPLEX + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa, bb; + + int i, j, k; + + a += (n - 1) * m; + b += (n - 1) * n; + + for (i = n - 1; i >= 0; i--) { + + bb = *(b + i); + + for (j = 0; j < m; j ++) { + aa = *(c + j + i * ldc); + aa *= bb; + *a = aa; + *(c + j + i * ldc) = aa; + a ++; + + for (k = 0; k < i; k ++){ + *(c + j + k * ldc) -= aa * *(b + k); + } + + } + b -= n; + a -= 2 * m; + } + +} + +#else + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa1, aa2; + FLOAT bb1, bb2; + FLOAT cc1, cc2; + + int i, j, k; + + ldc *= 2; + + a += (n - 1) * m * 2; + b += (n - 1) * n * 2; + + for (i = n - 1; i >= 0; i--) { + + bb1 = *(b + i * 2 + 0); + bb2 = *(b + i * 2 + 1); + + for (j = 0; j < m; j ++) { + + aa1 = *(c + j * 2 + 0 + i * ldc); + aa2 = *(c + j * 2 + 1 + i * ldc); + +#ifndef CONJ + cc1 = aa1 * bb1 - aa2 * bb2; + cc2 = aa1 * bb2 + aa2 * bb1; +#else + cc1 = aa1 * bb1 + aa2 * bb2; + cc2 = - aa1 * bb2 + aa2 * bb1; +#endif + + *(a + 0) = cc1; + *(a + 1) = cc2; + + *(c + j * 2 + 0 + i * ldc) = cc1; + *(c + j * 2 + 1 + i * ldc) = cc2; + a += 2; + + for (k = 0; k < i; k ++){ +#ifndef CONJ + *(c + j * 2 + 0 + k * ldc) -= cc1 * *(b + k * 2 + 0) - cc2 * *(b + k * 2 + 1); + *(c + j * 2 + 1 + k * ldc) -= cc1 * *(b + k * 2 + 1) + cc2 * *(b + k * 2 + 0); +#else + *(c + j * 2 + 0 + k * ldc) -= cc1 * *(b + k * 2 + 0) + cc2 * *(b + k * 2 + 1); + *(c + j * 2 + 1 + k * ldc) -= -cc1 * *(b + k * 2 + 1) + cc2 * *(b + k * 2 + 0); +#endif + } + + } + b -= n * 2; + a -= 4 * m; + } + +} + +#endif + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, +#ifdef COMPLEX + FLOAT dummy2, +#endif + FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ + + BLASLONG i, j; + FLOAT *aa, *cc; + BLASLONG kk; + BLASLONG mmodM = m - (m / GEMM_UNROLL_M) * GEMM_UNROLL_M; + +#if 0 + fprintf(stderr, "TRSM RT KERNEL m = %3ld n = %3ld k = %3ld offset = %3ld\n", + m, n, k, offset); +#endif + + kk = n - offset; + c += n * ldc * COMPSIZE; + b += n * k * COMPSIZE; + + if (n & (GEMM_UNROLL_N - 1)) { + + j = 1; + while (j < GEMM_UNROLL_N) { + if (n & j) { + + aa = a; + b -= j * k * COMPSIZE; + c -= j * ldc* COMPSIZE; + cc = c; + + i = (m / GEMM_UNROLL_M); + if (i > 0) { + + do { + if (k - kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, j, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + GEMM_UNROLL_M * kk * COMPSIZE, + b + j * kk * COMPSIZE, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, j, + aa + (kk - j) * GEMM_UNROLL_M * COMPSIZE, + b + (kk - j) * j * COMPSIZE, + cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + i --; + } while (i > 0); + } + + if (mmodM) { + i = 4; /* largest power-of-2 edge < UNROLL_M=6 */ + do { + if (mmodM & i) { + + if (k - kk > 0) { + GEMM_KERNEL(i, j, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + i * kk * COMPSIZE, + b + j * kk * COMPSIZE, + cc, ldc); + } + + solve(i, j, + aa + (kk - j) * i * COMPSIZE, + b + (kk - j) * j * COMPSIZE, + cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + + } + i >>= 1; + } while (i > 0); + } + kk -= j; + } + j <<= 1; + } + } + + j = (n >> GEMM_UNROLL_N_SHIFT); + + if (j > 0) { + + do { + aa = a; + b -= GEMM_UNROLL_N * k * COMPSIZE; + c -= GEMM_UNROLL_N * ldc * COMPSIZE; + cc = c; + + i = (m / GEMM_UNROLL_M); + if (i > 0) { + do { + if (k - kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + GEMM_UNROLL_M * kk * COMPSIZE, + b + GEMM_UNROLL_N * kk * COMPSIZE, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, GEMM_UNROLL_N, + aa + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_M * COMPSIZE, + b + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + i --; + } while (i > 0); + } + + if (mmodM) { + i = 4; /* largest power-of-2 edge < UNROLL_M=6 */ + do { + if (mmodM & i) { + if (k - kk > 0) { + GEMM_KERNEL(i, GEMM_UNROLL_N, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + i * kk * COMPSIZE, + b + GEMM_UNROLL_N * kk * COMPSIZE, + cc, + ldc); + } + + solve(i, GEMM_UNROLL_N, + aa + (kk - GEMM_UNROLL_N) * i * COMPSIZE, + b + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + } + i >>= 1; + } while (i > 0); + } + + kk -= GEMM_UNROLL_N; + j --; + } while (j > 0); + } + + return 0; +} + + diff --git a/kernel/generic/neg_tcopy_6.c b/kernel/generic/neg_tcopy_6.c new file mode 100644 index 0000000000..8a7a25b601 --- /dev/null +++ b/kernel/generic/neg_tcopy_6.c @@ -0,0 +1,445 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include "common.h" + +/* Contiguous MR=6 negating transpose pack (needed when DGEMM_UNROLL_M=6). + * Layout matches the Goto-style panel packing of neg_tcopy_4/8: full + * 6-row panels first, then m remainders of 4 / 2 / 1. */ + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b){ + + BLASLONG i, j; + + FLOAT *a_offset, *a_offset1, *a_offset2, *a_offset3, *a_offset4; + FLOAT *a_offset5, *a_offset6; + FLOAT *b_offset, *b_offset1, *b_offset2, *b_offset3; + FLOAT ctemp1, ctemp2, ctemp3, ctemp4; + FLOAT ctemp5, ctemp6, ctemp7, ctemp8; + FLOAT ctemp9, ctemp10, ctemp11, ctemp12; + FLOAT ctemp13, ctemp14, ctemp15, ctemp16; + FLOAT ctemp17, ctemp18, ctemp19, ctemp20; + FLOAT ctemp21, ctemp22, ctemp23, ctemp24; + + a_offset = a; + b_offset = b; + + b_offset2 = b + m * (n & ~3); + b_offset3 = b + m * (n & ~1); + + j = (m / 6); + if (j > 0){ + do{ + a_offset1 = a_offset; + a_offset2 = a_offset1 + lda; + a_offset3 = a_offset2 + lda; + a_offset4 = a_offset3 + lda; + a_offset5 = a_offset4 + lda; + a_offset6 = a_offset5 + lda; + a_offset += 6 * lda; + + b_offset1 = b_offset; + b_offset += 24; + + i = (n >> 2); + if (i > 0){ + do{ + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset1 + 1); + ctemp3 = *(a_offset1 + 2); + ctemp4 = *(a_offset1 + 3); + + ctemp5 = *(a_offset2 + 0); + ctemp6 = *(a_offset2 + 1); + ctemp7 = *(a_offset2 + 2); + ctemp8 = *(a_offset2 + 3); + + ctemp9 = *(a_offset3 + 0); + ctemp10 = *(a_offset3 + 1); + ctemp11 = *(a_offset3 + 2); + ctemp12 = *(a_offset3 + 3); + + ctemp13 = *(a_offset4 + 0); + ctemp14 = *(a_offset4 + 1); + ctemp15 = *(a_offset4 + 2); + ctemp16 = *(a_offset4 + 3); + + ctemp17 = *(a_offset5 + 0); + ctemp18 = *(a_offset5 + 1); + ctemp19 = *(a_offset5 + 2); + ctemp20 = *(a_offset5 + 3); + + ctemp21 = *(a_offset6 + 0); + ctemp22 = *(a_offset6 + 1); + ctemp23 = *(a_offset6 + 2); + ctemp24 = *(a_offset6 + 3); + + a_offset1 += 4; + a_offset2 += 4; + a_offset3 += 4; + a_offset4 += 4; + a_offset5 += 4; + a_offset6 += 4; + + *(b_offset1 + 0) = -ctemp1; + *(b_offset1 + 1) = -ctemp2; + *(b_offset1 + 2) = -ctemp3; + *(b_offset1 + 3) = -ctemp4; + + *(b_offset1 + 4) = -ctemp5; + *(b_offset1 + 5) = -ctemp6; + *(b_offset1 + 6) = -ctemp7; + *(b_offset1 + 7) = -ctemp8; + + *(b_offset1 + 8) = -ctemp9; + *(b_offset1 + 9) = -ctemp10; + *(b_offset1 + 10) = -ctemp11; + *(b_offset1 + 11) = -ctemp12; + + *(b_offset1 + 12) = -ctemp13; + *(b_offset1 + 13) = -ctemp14; + *(b_offset1 + 14) = -ctemp15; + *(b_offset1 + 15) = -ctemp16; + + *(b_offset1 + 16) = -ctemp17; + *(b_offset1 + 17) = -ctemp18; + *(b_offset1 + 18) = -ctemp19; + *(b_offset1 + 19) = -ctemp20; + + *(b_offset1 + 20) = -ctemp21; + *(b_offset1 + 21) = -ctemp22; + *(b_offset1 + 22) = -ctemp23; + *(b_offset1 + 23) = -ctemp24; + + b_offset1 += m * 4; + i --; + }while(i > 0); + } + + if (n & 2) { + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset1 + 1); + + ctemp3 = *(a_offset2 + 0); + ctemp4 = *(a_offset2 + 1); + + ctemp5 = *(a_offset3 + 0); + ctemp6 = *(a_offset3 + 1); + + ctemp7 = *(a_offset4 + 0); + ctemp8 = *(a_offset4 + 1); + + ctemp9 = *(a_offset5 + 0); + ctemp10 = *(a_offset5 + 1); + + ctemp11 = *(a_offset6 + 0); + ctemp12 = *(a_offset6 + 1); + + a_offset1 += 2; + a_offset2 += 2; + a_offset3 += 2; + a_offset4 += 2; + a_offset5 += 2; + a_offset6 += 2; + + *(b_offset2 + 0) = -ctemp1; + *(b_offset2 + 1) = -ctemp2; + *(b_offset2 + 2) = -ctemp3; + *(b_offset2 + 3) = -ctemp4; + + *(b_offset2 + 4) = -ctemp5; + *(b_offset2 + 5) = -ctemp6; + *(b_offset2 + 6) = -ctemp7; + *(b_offset2 + 7) = -ctemp8; + + *(b_offset2 + 8) = -ctemp9; + *(b_offset2 + 9) = -ctemp10; + *(b_offset2 + 10) = -ctemp11; + *(b_offset2 + 11) = -ctemp12; + + b_offset2 += 12; + } + + if (n & 1) { + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset2 + 0); + ctemp3 = *(a_offset3 + 0); + ctemp4 = *(a_offset4 + 0); + ctemp5 = *(a_offset5 + 0); + ctemp6 = *(a_offset6 + 0); + + *(b_offset3 + 0) = -ctemp1; + *(b_offset3 + 1) = -ctemp2; + *(b_offset3 + 2) = -ctemp3; + *(b_offset3 + 3) = -ctemp4; + *(b_offset3 + 4) = -ctemp5; + *(b_offset3 + 5) = -ctemp6; + + b_offset3 += 6; + } + + j--; + }while(j > 0); + } + + /* Remainders after m/6 panels are 0..5 — cascade 4, then 2, then 1. + * Do not use (m & 2)/(m & 1) on the original m: those bits are not the + * leftover after a non-power-of-2 main panel. */ + j = m % 6; + + if (j >= 4){ + a_offset1 = a_offset; + a_offset2 = a_offset1 + lda; + a_offset3 = a_offset2 + lda; + a_offset4 = a_offset3 + lda; + a_offset += 4 * lda; + + b_offset1 = b_offset; + b_offset += 16; + + i = (n >> 2); + if (i > 0){ + do{ + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset1 + 1); + ctemp3 = *(a_offset1 + 2); + ctemp4 = *(a_offset1 + 3); + + ctemp5 = *(a_offset2 + 0); + ctemp6 = *(a_offset2 + 1); + ctemp7 = *(a_offset2 + 2); + ctemp8 = *(a_offset2 + 3); + + ctemp9 = *(a_offset3 + 0); + ctemp10 = *(a_offset3 + 1); + ctemp11 = *(a_offset3 + 2); + ctemp12 = *(a_offset3 + 3); + + ctemp13 = *(a_offset4 + 0); + ctemp14 = *(a_offset4 + 1); + ctemp15 = *(a_offset4 + 2); + ctemp16 = *(a_offset4 + 3); + + a_offset1 += 4; + a_offset2 += 4; + a_offset3 += 4; + a_offset4 += 4; + + *(b_offset1 + 0) = -ctemp1; + *(b_offset1 + 1) = -ctemp2; + *(b_offset1 + 2) = -ctemp3; + *(b_offset1 + 3) = -ctemp4; + + *(b_offset1 + 4) = -ctemp5; + *(b_offset1 + 5) = -ctemp6; + *(b_offset1 + 6) = -ctemp7; + *(b_offset1 + 7) = -ctemp8; + + *(b_offset1 + 8) = -ctemp9; + *(b_offset1 + 9) = -ctemp10; + *(b_offset1 + 10) = -ctemp11; + *(b_offset1 + 11) = -ctemp12; + + *(b_offset1 + 12) = -ctemp13; + *(b_offset1 + 13) = -ctemp14; + *(b_offset1 + 14) = -ctemp15; + *(b_offset1 + 15) = -ctemp16; + + b_offset1 += m * 4; + i --; + }while(i > 0); + } + + if (n & 2) { + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset1 + 1); + + ctemp3 = *(a_offset2 + 0); + ctemp4 = *(a_offset2 + 1); + + ctemp5 = *(a_offset3 + 0); + ctemp6 = *(a_offset3 + 1); + + ctemp7 = *(a_offset4 + 0); + ctemp8 = *(a_offset4 + 1); + + a_offset1 += 2; + a_offset2 += 2; + a_offset3 += 2; + a_offset4 += 2; + + *(b_offset2 + 0) = -ctemp1; + *(b_offset2 + 1) = -ctemp2; + *(b_offset2 + 2) = -ctemp3; + *(b_offset2 + 3) = -ctemp4; + + *(b_offset2 + 4) = -ctemp5; + *(b_offset2 + 5) = -ctemp6; + *(b_offset2 + 6) = -ctemp7; + *(b_offset2 + 7) = -ctemp8; + + b_offset2 += 8; + } + + if (n & 1) { + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset2 + 0); + ctemp3 = *(a_offset3 + 0); + ctemp4 = *(a_offset4 + 0); + + *(b_offset3 + 0) = -ctemp1; + *(b_offset3 + 1) = -ctemp2; + *(b_offset3 + 2) = -ctemp3; + *(b_offset3 + 3) = -ctemp4; + + b_offset3 += 4; + } + j -= 4; + } + + if (j >= 2){ + a_offset1 = a_offset; + a_offset2 = a_offset1 + lda; + a_offset += 2 * lda; + + b_offset1 = b_offset; + b_offset += 8; + + i = (n >> 2); + if (i > 0){ + do{ + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset1 + 1); + ctemp3 = *(a_offset1 + 2); + ctemp4 = *(a_offset1 + 3); + + ctemp5 = *(a_offset2 + 0); + ctemp6 = *(a_offset2 + 1); + ctemp7 = *(a_offset2 + 2); + ctemp8 = *(a_offset2 + 3); + + a_offset1 += 4; + a_offset2 += 4; + + *(b_offset1 + 0) = -ctemp1; + *(b_offset1 + 1) = -ctemp2; + *(b_offset1 + 2) = -ctemp3; + *(b_offset1 + 3) = -ctemp4; + + *(b_offset1 + 4) = -ctemp5; + *(b_offset1 + 5) = -ctemp6; + *(b_offset1 + 6) = -ctemp7; + *(b_offset1 + 7) = -ctemp8; + + b_offset1 += m * 4; + i --; + }while(i > 0); + } + + if (n & 2) { + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset1 + 1); + + ctemp3 = *(a_offset2 + 0); + ctemp4 = *(a_offset2 + 1); + + a_offset1 += 2; + a_offset2 += 2; + + *(b_offset2 + 0) = -ctemp1; + *(b_offset2 + 1) = -ctemp2; + *(b_offset2 + 2) = -ctemp3; + *(b_offset2 + 3) = -ctemp4; + + b_offset2 += 4; + } + + if (n & 1) { + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset2 + 0); + + *(b_offset3 + 0) = -ctemp1; + *(b_offset3 + 1) = -ctemp2; + b_offset3 += 2; + } + j -= 2; + } + + if (j >= 1){ + a_offset1 = a_offset; + b_offset1 = b_offset; + + i = (n >> 2); + if (i > 0){ + do{ + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset1 + 1); + ctemp3 = *(a_offset1 + 2); + ctemp4 = *(a_offset1 + 3); + + a_offset1 += 4; + + *(b_offset1 + 0) = -ctemp1; + *(b_offset1 + 1) = -ctemp2; + *(b_offset1 + 2) = -ctemp3; + *(b_offset1 + 3) = -ctemp4; + + b_offset1 += 4 * m; + + i --; + }while(i > 0); + } + + if (n & 2) { + ctemp1 = *(a_offset1 + 0); + ctemp2 = *(a_offset1 + 1); + a_offset1 += 2; + + *(b_offset2 + 0) = -ctemp1; + *(b_offset2 + 1) = -ctemp2; + } + + if (n & 1) { + ctemp1 = *(a_offset1 + 0); + *(b_offset3 + 0) = -ctemp1; + } + } + + return 0; +} diff --git a/param.h b/param.h index 129ef4ea6f..f6ed94737e 100644 --- a/param.h +++ b/param.h @@ -3464,8 +3464,55 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SYMV_P 16 -#if defined(CORTEXA57) || defined(CORTEXX1) || \ - defined(CORTEXA72) || defined(CORTEXA73) || \ +#if defined(CORTEXA72) + +/* Cortex-A72 (e.g. AWS Graviton1): dedicated DGEMM 6x8 ukernel + blocking. + * Stock OpenBLAS aliased A72 to A57 (8x4) and shared the NUM_CORES>8 + * "server" Q=512 path, which is a poor fit for A72's ~2 MiB shared L2 + * per 4-core cluster. + * + * Measured on a1.metal (16 cores, HPL N=52000, NB=192, 2026-08-09): + * 6x8 + P=120 Q=240 R=4096 -> 103.40 GFLOPS + * prior 6x8 P=168 Q=128 -> ~100 GFLOPS + * archived 8x4 custom -> ~90 GFLOPS + * BLIS cortexa57 (archived) -> ~107 GFLOPS + * An 8x8 NEON experiment spilled C and was rejected (~58 GFLOPS HPL). + * Single-core slice (NUM_CORES<=2) keeps Q=240 but caps R=768 so the + * packed B panel stays within a privately owned 2 MiB L2. */ + +#define SGEMM_DEFAULT_UNROLL_M 16 +#define SGEMM_DEFAULT_UNROLL_N 4 + +#define DGEMM_DEFAULT_UNROLL_M 6 +#define DGEMM_DEFAULT_UNROLL_N 8 + +#define CGEMM_DEFAULT_UNROLL_M 8 +#define CGEMM_DEFAULT_UNROLL_N 4 + +#define ZGEMM_DEFAULT_UNROLL_M 4 +#define ZGEMM_DEFAULT_UNROLL_N 4 + +#define SGEMM_DEFAULT_P 128 +#define DGEMM_DEFAULT_P 120 +#define CGEMM_DEFAULT_P 128 +#define ZGEMM_DEFAULT_P 128 + +#define SGEMM_DEFAULT_Q 352 +#define DGEMM_DEFAULT_Q 240 +#define CGEMM_DEFAULT_Q 224 +#define ZGEMM_DEFAULT_Q 112 + +#define SGEMM_DEFAULT_R 4096 +#if NUM_CORES > 2 +#define DGEMM_DEFAULT_R 4096 +#else +#define DGEMM_DEFAULT_R 768 +#endif +#define CGEMM_DEFAULT_R 4096 +#define ZGEMM_DEFAULT_R 2048 + +#elif defined(CORTEXA57) || defined(CORTEXX1) || \ + defined(CORTEXA73) || \ defined(FALKOR) || defined(TSV110) || defined(EMAG8180) || defined(VORTEX) || defined(FT2000) || defined(VORTEXM4) #define SGEMM_DEFAULT_UNROLL_M 16