amd/blis
1
0
Fork 0
mirror of https://github.com/amd/blis.git synced 2026-04-19 23:28:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Relax alignment restrictions for haswell sgemm.

Browse Source
This commit is contained in:
Devin Matthews
2016-07-22 12:56:51 -05:00
parent ee2c139df6
commit e0d2fa0d83
1 changed files with 41 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
kernels/x86_64/haswell/3/bli_gemm_asm_d6x8.c
View File

@@ -322,23 +322,6 @@ void bli_sgemm_asm_6x16
"leaq (%%r13,%%rsi,4), %%r10 \n\t" // r10 = 7*cs_c; "leaq (%%r13,%%rsi,4), %%r10 \n\t" // r10 = 7*cs_c;
" \n\t" " \n\t"
" \n\t" " \n\t"
" \n\t"
" \n\t" // determine if
" \n\t" // c % 32 == 0, AND
" \n\t" // 4*rs_c % 32 == 0, AND
" \n\t" // cs_c == 1
" \n\t" // ie: aligned, ldim aligned, and
" \n\t" // row-stored
" \n\t"
"cmpq $4, %%rsi \n\t" // set ZF if (4*cs_c) == 4.
"sete %%bl \n\t" // bl = ( ZF == 1 ? 1 : 0 );
"testq $31, %%rcx \n\t" // set ZF if c & 32 is zero.
"setz %%bh \n\t" // bh = ( ZF == 0 ? 1 : 0 );
"testq $31, %%rdi \n\t" // set ZF if (4*rs_c) & 32 is zero.
"setz %%al \n\t" // al = ( ZF == 0 ? 1 : 0 );
" \n\t" // and(bl,bh) followed by
" \n\t" // and(bh,al) will reveal result
" \n\t"
" \n\t" // now avoid loading C if beta == 0 " \n\t" // now avoid loading C if beta == 0
" \n\t" " \n\t"
"vxorps %%ymm0, %%ymm0, %%ymm0 \n\t" // set ymm0 to zero. "vxorps %%ymm0, %%ymm0, %%ymm0 \n\t" // set ymm0 to zero.
@@ -346,10 +329,8 @@ void bli_sgemm_asm_6x16
"je .SBETAZERO \n\t" // if ZF = 1, jump to beta == 0 case "je .SBETAZERO \n\t" // if ZF = 1, jump to beta == 0 case
" \n\t" " \n\t"
" \n\t" " \n\t"
" \n\t" // check if aligned/row-stored "cmpq $4, %%rsi \n\t" // set ZF if (4*cs_c) == 4.
"andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. "jz .SROWSTORED \n\t" // jump to row storage case
"andb %%bh, %%al \n\t" // set ZF if bh & al == 1.
"jne .SROWSTORED \n\t" // jump to row storage case
" \n\t" " \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
@@ -439,63 +420,51 @@ void bli_sgemm_asm_6x16
".SROWSTORED: \n\t" ".SROWSTORED: \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps (%%rcx), %%ymm0 \n\t" "vfmadd231ps (%%rcx), %%ymm3, %%ymm4 \n\t"
"vfmadd213ps %%ymm4, %%ymm3, %%ymm0 \n\t" "vmovups %%ymm4, (%%rcx) \n\t"
"vmovaps %%ymm0, (%%rcx) \n\t"
"addq %%rdi, %%rcx \n\t" "addq %%rdi, %%rcx \n\t"
"vmovaps (%%rdx), %%ymm1 \n\t" "vfmadd231ps (%%rdx), %%ymm3, %%ymm5 \n\t"
"vfmadd213ps %%ymm5, %%ymm3, %%ymm1 \n\t" "vmovups %%ymm5, (%%rdx) \n\t"
"vmovaps %%ymm1, (%%rdx) \n\t"
"addq %%rdi, %%rdx \n\t" "addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps (%%rcx), %%ymm0 \n\t" "vfmadd231ps (%%rcx), %%ymm3, %%ymm6 \n\t"
"vfmadd213ps %%ymm6, %%ymm3, %%ymm0 \n\t" "vmovups %%ymm6, (%%rcx) \n\t"
"vmovaps %%ymm0, (%%rcx) \n\t"
"addq %%rdi, %%rcx \n\t" "addq %%rdi, %%rcx \n\t"
"vmovaps (%%rdx), %%ymm1 \n\t" "vfmadd231ps (%%rdx), %%ymm3, %%ymm7 \n\t"
"vfmadd213ps %%ymm7, %%ymm3, %%ymm1 \n\t" "vmovups %%ymm7, (%%rdx) \n\t"
"vmovaps %%ymm1, (%%rdx) \n\t"
"addq %%rdi, %%rdx \n\t" "addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps (%%rcx), %%ymm0 \n\t" "vfmadd231ps (%%rcx), %%ymm3, %%ymm8 \n\t"
"vfmadd213ps %%ymm8, %%ymm3, %%ymm0 \n\t" "vmovups %%ymm8, (%%rcx) \n\t"
"vmovaps %%ymm0, (%%rcx) \n\t"
"addq %%rdi, %%rcx \n\t" "addq %%rdi, %%rcx \n\t"
"vmovaps (%%rdx), %%ymm1 \n\t" "vfmadd231ps (%%rdx), %%ymm3, %%ymm9 \n\t"
"vfmadd213ps %%ymm9, %%ymm3, %%ymm1 \n\t" "vmovups %%ymm9, (%%rdx) \n\t"
"vmovaps %%ymm1, (%%rdx) \n\t"
"addq %%rdi, %%rdx \n\t" "addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps (%%rcx), %%ymm0 \n\t" "vfmadd231ps (%%rcx), %%ymm3, %%ymm10 \n\t"
"vfmadd213ps %%ymm10, %%ymm3, %%ymm0 \n\t" "vmovups %%ymm10, (%%rcx) \n\t"
"vmovaps %%ymm0, (%%rcx) \n\t"
"addq %%rdi, %%rcx \n\t" "addq %%rdi, %%rcx \n\t"
"vmovaps (%%rdx), %%ymm1 \n\t" "vfmadd231ps (%%rdx), %%ymm3, %%ymm11 \n\t"
"vfmadd213ps %%ymm11, %%ymm3, %%ymm1 \n\t" "vmovups %%ymm11, (%%rdx) \n\t"
"vmovaps %%ymm1, (%%rdx) \n\t"
"addq %%rdi, %%rdx \n\t" "addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps (%%rcx), %%ymm0 \n\t" "vfmadd231ps (%%rcx), %%ymm3, %%ymm12 \n\t"
"vfmadd213ps %%ymm12, %%ymm3, %%ymm0 \n\t" "vmovups %%ymm12, (%%rcx) \n\t"
"vmovaps %%ymm0, (%%rcx) \n\t"
"addq %%rdi, %%rcx \n\t" "addq %%rdi, %%rcx \n\t"
"vmovaps (%%rdx), %%ymm1 \n\t" "vfmadd231ps (%%rdx), %%ymm3, %%ymm13 \n\t"
"vfmadd213ps %%ymm13, %%ymm3, %%ymm1 \n\t" "vmovups %%ymm13, (%%rdx) \n\t"
"vmovaps %%ymm1, (%%rdx) \n\t"
"addq %%rdi, %%rdx \n\t" "addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps (%%rcx), %%ymm0 \n\t" "vfmadd231ps (%%rcx), %%ymm3, %%ymm14 \n\t"
"vfmadd213ps %%ymm14, %%ymm3, %%ymm0 \n\t" "vmovups %%ymm14, (%%rcx) \n\t"
"vmovaps %%ymm0, (%%rcx) \n\t"
//"addq %%rdi, %%rcx \n\t" //"addq %%rdi, %%rcx \n\t"
"vmovaps (%%rdx), %%ymm1 \n\t" "vfmadd231ps (%%rdx), %%ymm3, %%ymm15 \n\t"
"vfmadd213ps %%ymm15, %%ymm3, %%ymm1 \n\t" "vmovups %%ymm15, (%%rdx) \n\t"
"vmovaps %%ymm1, (%%rdx) \n\t"
//"addq %%rdi, %%rdx \n\t" //"addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
@@ -505,10 +474,9 @@ void bli_sgemm_asm_6x16
" \n\t" " \n\t"
" \n\t" " \n\t"
".SBETAZERO: \n\t" ".SBETAZERO: \n\t"
" \n\t" // check if aligned/row-stored " \n\t"
"andb %%bl, %%bh \n\t" // set ZF if bl & bh == 1. "cmpq $4, %%rsi \n\t" // set ZF if (4*cs_c) == 4.
"andb %%bh, %%al \n\t" // set ZF if bh & al == 1. "jz .SROWSTORBZ \n\t" // jump to row storage case
"jne .SROWSTORBZ \n\t" // jump to row storage case
" \n\t" " \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
@@ -586,38 +554,38 @@ void bli_sgemm_asm_6x16
".SROWSTORBZ: \n\t" ".SROWSTORBZ: \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps %%ymm4, (%%rcx) \n\t" "vmovups %%ymm4, (%%rcx) \n\t"
"addq %%rdi, %%rcx \n\t" "addq %%rdi, %%rcx \n\t"
"vmovaps %%ymm5, (%%rdx) \n\t" "vmovups %%ymm5, (%%rdx) \n\t"
"addq %%rdi, %%rdx \n\t" "addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
"vmovaps %%ymm6, (%%rcx) \n\t" "vmovups %%ymm6, (%%rcx) \n\t"
"addq %%rdi, %%rcx \n\t" "addq %%rdi, %%rcx \n\t"
"vmovaps %%ymm7, (%%rdx) \n\t" "vmovups %%ymm7, (%%rdx) \n\t"
"addq %%rdi, %%rdx \n\t" "addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps %%ymm8, (%%rcx) \n\t" "vmovups %%ymm8, (%%rcx) \n\t"
"addq %%rdi, %%rcx \n\t" "addq %%rdi, %%rcx \n\t"
"vmovaps %%ymm9, (%%rdx) \n\t" "vmovups %%ymm9, (%%rdx) \n\t"
"addq %%rdi, %%rdx \n\t" "addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps %%ymm10, (%%rcx) \n\t" "vmovups %%ymm10, (%%rcx) \n\t"
"addq %%rdi, %%rcx \n\t" "addq %%rdi, %%rcx \n\t"
"vmovaps %%ymm11, (%%rdx) \n\t" "vmovups %%ymm11, (%%rdx) \n\t"
"addq %%rdi, %%rdx \n\t" "addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps %%ymm12, (%%rcx) \n\t" "vmovups %%ymm12, (%%rcx) \n\t"
"addq %%rdi, %%rcx \n\t" "addq %%rdi, %%rcx \n\t"
"vmovaps %%ymm13, (%%rdx) \n\t" "vmovups %%ymm13, (%%rdx) \n\t"
"addq %%rdi, %%rdx \n\t" "addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"
"vmovaps %%ymm14, (%%rcx) \n\t" "vmovups %%ymm14, (%%rcx) \n\t"
//"addq %%rdi, %%rcx \n\t" //"addq %%rdi, %%rcx \n\t"
"vmovaps %%ymm15, (%%rdx) \n\t" "vmovups %%ymm15, (%%rdx) \n\t"
//"addq %%rdi, %%rdx \n\t" //"addq %%rdi, %%rdx \n\t"
" \n\t" " \n\t"
" \n\t" " \n\t"