;++ ; ; Copyright (c) Microsoft Corporation. All rights reserved. ; ; Licensed under the MIT License. ; ; Module Name: ; ; QgemmU8U8KernelAvx2.asm ; ; Abstract: ; ; This module implements the kernels for the quantized integer matrix/matrix ; multiply operation (QGEMM). ; ; This implementation uses AVX2 instructions. ; ;-- .xlist INCLUDE mlasi.inc .list EXTERN MlasMaskMoveTableAvx:NEAR ; ; Stack frame layout for the U8U8 CopyPackA routine. ; GemmU8U8CopyPackAFrame STRUCT PaddedMatrixAData OWORD 4 DUP (?) SavedXmm6 OWORD ? SavedXmm7 OWORD ? SavedXmm8 OWORD ? SavedXmm9 OWORD ? Padding QWORD ? SavedR13 QWORD ? SavedR12 QWORD ? SavedRdi QWORD ? SavedRsi QWORD ? SavedRbx QWORD ? SavedRbp QWORD ? ReturnAddress QWORD ? PreviousP1Home QWORD ? PreviousP2Home QWORD ? PreviousP3Home QWORD ? PreviousP4Home QWORD ? CountK QWORD ? RowSumBuffer QWORD ? GemmU8U8CopyPackAFrame ENDS ; ; Stack frame layout for the U8U8 CopyPackB routine. ; GemmU8U8CopyPackBFrame STRUCT PaddedMatrixBData OWORD 2 DUP (?) SavedRsi QWORD ? SavedRbx QWORD ? SavedRbp QWORD ? ReturnAddress QWORD ? PreviousP1Home QWORD ? PreviousP2Home QWORD ? PreviousP3Home QWORD ? PreviousP4Home QWORD ? CountK QWORD ? ColumnSumBuffer QWORD ? GemmU8U8CopyPackBFrame ENDS ;++ ; ; Routine Description: ; ; This routine copies elements from the source matrix to the destination ; packed buffer. ; ; The kernel expects that elements from matrix A have been zero extended to ; 16-bits and padded to a multiple of 32-bits (two pairs of 16-bit values). ; The kernel can then efficiently broadcast 32-bits from the packed buffer ; and avoid expensive shuffling inside the kernel. ; ; Arguments: ; ; D (rcx) - Supplies the address of the destination packed buffer. ; ; A (rdx) - Supplies the address of the source matrix. ; ; lda (r8) - Supplies the number of elements per row of the source matrix. ; ; CountM (r9) - Supplies the number of rows of the source matrix to copy. ; ; CountK - Supplies the number of columns of the source matrix to copy. ; ; RowSumBuffer - Supplies the address of the buffer to receive the sums of ; the elements along each of the rows. ; ; Return Value: ; ; None. ; ;-- NESTED_ENTRY MlasGemmU8U8CopyPackAAvx2, _TEXT rex_push_reg rbp push_reg rbx push_reg rsi push_reg rdi push_reg r12 push_reg r13 alloc_stack (GemmU8U8CopyPackAFrame.SavedR13) save_xmm128 xmm6,GemmU8U8CopyPackAFrame.SavedXmm6 save_xmm128 xmm7,GemmU8U8CopyPackAFrame.SavedXmm7 save_xmm128 xmm8,GemmU8U8CopyPackAFrame.SavedXmm8 save_xmm128 xmm9,GemmU8U8CopyPackAFrame.SavedXmm9 END_PROLOGUE mov rdi,rcx mov rsi,rdx mov r10,GemmU8U8CopyPackAFrame.CountK[rsp] lea r11,[r10+1] and r11,NOT 1 ; align CountK up to pair count mov r12,GemmU8U8CopyPackAFrame.RowSumBuffer[rsp] vpcmpeqw ymm8,ymm8,ymm8 ; generate word vector [0xFFFF] vpsrlw ymm8,ymm8,15 ; generate word vector [0x0001] ; ; Compute the conditional load/store mask for an unaligned CountK. ; mov eax,r10d and eax,15 ; isolate unaligned count inc eax shr eax,1 ; align unaligned count to pair count neg rax lea rbx,MlasMaskMoveTableAvx+8*4 vmovdqu ymm9,YMMWORD PTR [rbx+rax*4] ; ; Zero initialize the padded stack buffers. ; vpxor xmm0,xmm0,xmm0 vmovdqu YMMWORD PTR GemmU8U8CopyPackAFrame.PaddedMatrixAData[rsp],ymm0 vmovdqu YMMWORD PTR GemmU8U8CopyPackAFrame.PaddedMatrixAData[rsp+32],ymm0 ; ; Process 4 rows of matrix A in a loop. ; ; Zero extend the source bytes to 16-bits and write to the packed buffer. ; ; The packed buffer has the same data ordering as the source bytes, but CountK ; is aligned up to a multiple of 2 to maintain 32-bit alignment. All padding ; bytes are zero filled. ; ; These 16-bit values are also accumulated into an intermediate per-row ; accumulator. CountK cannot be greater than 128 to avoid overflowing these ; signed 16-bit accumulators. ; sub r9,4 jb ProcessRemainingRows ProcessNextRowM4: vpxor xmm0,xmm0,xmm0 ; clear row accumulators vpxor xmm1,xmm1,xmm1 vpxor xmm2,xmm2,xmm2 vpxor xmm3,xmm3,xmm3 mov rdx,rsi mov rcx,rdi lea rsi,[rsi+r8*4] ; advance next matrix A by 4 rows lea rdi,[rdi+r11*8] ; advance next matrix D by 4 rows mov rbx,r10 ; reload columns remaining sub rbx,16 jb ProcessRemainingColumnsM4 ProcessNextColumnLoopM4: lea rax,[rdx+r8*2] ; compute matrix A plus 2 rows vpmovzxbw ymm4,XMMWORD PTR [rdx] vpmovzxbw ymm5,XMMWORD PTR [rdx+r8] vpmovzxbw ymm6,XMMWORD PTR [rax] vpmovzxbw ymm7,XMMWORD PTR [rax+r8] lea rax,[rcx+r11*4] ; compute matrix D plus 2 rows vmovdqu YMMWORD PTR [rcx],ymm4 vmovdqu YMMWORD PTR [rcx+r11*2],ymm5 vmovdqu YMMWORD PTR [rax],ymm6 vmovdqu YMMWORD PTR [rax+r11*2],ymm7 vpaddw ymm0,ymm0,ymm4 ; accumulate per row along columns vpaddw ymm1,ymm1,ymm5 vpaddw ymm2,ymm2,ymm6 vpaddw ymm3,ymm3,ymm7 add rdx,16 ; advance matrix A by 16 bytes add rcx,16*2 ; advance matrix D by 16 words sub rbx,16 ; subtract columns remaining jae ProcessNextColumnLoopM4 ProcessRemainingColumnsM4: add rbx,16 ; correct for over-subtract above jz ReduceRowSumBufferM4 ; ; Copy the unaligned CountK columns to a zero padded stack buffer. ; .errnz GemmU8U8CopyPackAFrame.PaddedMatrixAData mov rbp,rsp ; GemmU8U8CopyPackAFrame.PaddedMatrixAData test bl,8 ; (CountK & 8) != 0? jz CopyRemainingCountKLessThan8M4 lea r13,[rdx+r8*2] ; compute matrix A plus 2 rows mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax mov rax,QWORD PTR [rdx+r8] mov QWORD PTR [rbp+16],rax mov rax,QWORD PTR [r13] mov QWORD PTR [rbp+32],rax mov rax,QWORD PTR [r13+r8] mov QWORD PTR [rbp+48],rax add rdx,8 add rbp,8 ; advance padded buffer destination CopyRemainingCountKLessThan8M4: test bl,4 ; (CountK & 4) != 0? jz CopyRemainingCountKLessThan4M4 lea r13,[rdx+r8*2] ; compute matrix A plus 2 rows mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax mov eax,DWORD PTR [rdx+r8] mov DWORD PTR [rbp+16],eax mov eax,DWORD PTR [r13] mov DWORD PTR [rbp+32],eax mov eax,DWORD PTR [r13+r8] mov DWORD PTR [rbp+48],eax add rdx,4 add rbp,4 ; advance padded buffer destination CopyRemainingCountKLessThan4M4: test bl,2 ; (CountK & 2) != 0? jz CopyRemainingCountKLessThan2M4 lea r13,[rdx+r8*2] ; compute matrix A plus 2 rows movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax movzx eax,WORD PTR [rdx+r8] mov WORD PTR [rbp+16],ax movzx eax,WORD PTR [r13] mov WORD PTR [rbp+32],ax movzx eax,WORD PTR [r13+r8] mov WORD PTR [rbp+48],ax add rdx,2 add rbp,2 ; advance padded buffer destination CopyRemainingCountKLessThan2M4: test bl,1 ; (CountK & 1) != 0? jz ProcessPaddedMatrixADataM4 lea r13,[rdx+r8*2] ; compute matrix A plus 2 rows movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al movzx eax,BYTE PTR [rdx+r8] mov BYTE PTR [rbp+16],al movzx eax,BYTE PTR [r13] mov BYTE PTR [rbp+32],al movzx eax,BYTE PTR [r13+r8] mov BYTE PTR [rbp+48],al ; ; Process the remaining CountK columns using the zero padded stack buffer. ; ProcessPaddedMatrixADataM4: vpmovzxbw ymm4,XMMWORD PTR GemmU8U8CopyPackAFrame.PaddedMatrixAData[rsp] vpmovzxbw ymm5,XMMWORD PTR GemmU8U8CopyPackAFrame.PaddedMatrixAData[rsp+16] vpmovzxbw ymm6,XMMWORD PTR GemmU8U8CopyPackAFrame.PaddedMatrixAData[rsp+32] vpmovzxbw ymm7,XMMWORD PTR GemmU8U8CopyPackAFrame.PaddedMatrixAData[rsp+48] lea rax,[rcx+r11*4] ; compute matrix D plus 2 rows vpmaskmovd YMMWORD PTR [rcx],ymm9,ymm4 vpmaskmovd YMMWORD PTR [rcx+r11*2],ymm9,ymm5 vpmaskmovd YMMWORD PTR [rax],ymm9,ymm6 vpmaskmovd YMMWORD PTR [rax+r11*2],ymm9,ymm7 vpaddw ymm0,ymm0,ymm4 ; accumulate per row along columns vpaddw ymm1,ymm1,ymm5 vpaddw ymm2,ymm2,ymm6 vpaddw ymm3,ymm3,ymm7 ; ; Reduce the sums for the four rows of output. ; ReduceRowSumBufferM4: vpmaddwd ymm0,ymm0,ymm8 ; horizontal word+word=dword per row vpmaddwd ymm1,ymm1,ymm8 vphaddd ymm0,ymm0,ymm1 ; reduce and interleave Sum1/Sum0 vpmaddwd ymm2,ymm2,ymm8 vpmaddwd ymm3,ymm3,ymm8 vphaddd ymm1,ymm2,ymm3 ; reduce and interleave Sum3/Sum2 vphaddd ymm0,ymm0,ymm1 ; reduce and interleave Sum3/Sum2/Sum1/Sum0 vextracti128 xmm1,ymm0,1 ; extract high dwords vpaddd xmm0,xmm0,xmm1 ; reduce low/high dwords vmovdqu XMMWORD PTR [r12],xmm0 add r12,4*4 ; advance row sum buffer by 4 dwords sub r9,4 ; subtract rows remaining jae ProcessNextRowM4 ProcessRemainingRows: add r9,4 ; correct for over-subtract above jz ExitRoutine ; ; Process a single row of matrix A in a loop. ; ProcessNextRowM1: vpxor xmm0,xmm0,xmm0 ; clear row accumulator mov rdx,rsi mov rcx,rdi add rsi,r8 lea rdi,[rdi+r11*2] mov rbx,r10 ; reload columns remaining sub rbx,16 jb ProcessRemainingColumnsM1 ProcessNextColumnLoopM1: vpmovzxbw ymm4,XMMWORD PTR [rdx] vmovdqu YMMWORD PTR [rcx],ymm4 vpaddw ymm0,ymm0,ymm4 ; accumulate per row along columns add rdx,16 ; advance matrix A by 16 bytes add rcx,16*2 ; advance matrix D by 16 words sub rbx,16 ; subtract columns remaining jae ProcessNextColumnLoopM1 ProcessRemainingColumnsM1: add rbx,16 ; correct for over-subtract above jz ReduceRowSumBufferM1 ; ; Copy the unaligned CountK columns to a zero padded stack buffer. ; .errnz GemmU8U8CopyPackAFrame.PaddedMatrixAData mov rbp,rsp ; GemmU8U8CopyPackAFrame.PaddedMatrixAData test bl,8 ; (CountK & 8) != 0? jz CopyRemainingCountKLessThan8M1 mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax add rdx,8 add rbp,8 ; advance padded buffer destination CopyRemainingCountKLessThan8M1: test bl,4 ; (CountK & 4) != 0? jz CopyRemainingCountKLessThan4M1 mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax add rdx,4 add rbp,4 ; advance padded buffer destination CopyRemainingCountKLessThan4M1: test bl,2 ; (CountK & 2) != 0? jz CopyRemainingCountKLessThan2M1 movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax add rdx,2 add rbp,2 ; advance padded buffer destination CopyRemainingCountKLessThan2M1: test bl,1 ; (CountK & 1) != 0? jz ProcessPaddedMatrixADataM1 movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al ; ; Process the remaining CountK columns using the zero padded stack buffer. ; ProcessPaddedMatrixADataM1: vpmovzxbw ymm4,XMMWORD PTR GemmU8U8CopyPackAFrame.PaddedMatrixAData[rsp] vpmaskmovd YMMWORD PTR [rcx],ymm9,ymm4 vpaddw ymm0,ymm0,ymm4 ; accumulate per row along columns ; ; Reduce the sum for the single row of output. ; ReduceRowSumBufferM1: vpmaddwd ymm0,ymm0,ymm8 ; horizontal word+word=dword per row vextracti128 xmm1,ymm0,1 ; extract high dwords vpaddd xmm0,xmm0,xmm1 ; reduction vphaddd xmm0,xmm0,xmm0 vphaddd xmm0,xmm0,xmm0 vmovd DWORD PTR [r12],xmm0 add r12,4 ; advance row sum buffer by 1 dword dec r9 ; decrement rows remaining jnz ProcessNextRowM1 ; ; Restore non-volatile registers and return. ; ExitRoutine: vzeroupper movaps xmm6,GemmU8U8CopyPackAFrame.SavedXmm6[rsp] movaps xmm7,GemmU8U8CopyPackAFrame.SavedXmm7[rsp] movaps xmm8,GemmU8U8CopyPackAFrame.SavedXmm8[rsp] movaps xmm9,GemmU8U8CopyPackAFrame.SavedXmm9[rsp] add rsp,(GemmU8U8CopyPackAFrame.SavedR13) BEGIN_EPILOGUE pop r13 pop r12 pop rdi pop rsi pop rbx pop rbp ret NESTED_END MlasGemmU8U8CopyPackAAvx2, _TEXT ;++ ; ; Routine Description: ; ; This routine copies elements from the source matrix to the destination ; packed buffer. ; ; Arguments: ; ; D (rcx) - Supplies the address of the destination packed buffer. ; ; B (rdx) - Supplies the address of the source matrix. ; ; ldb (r8) - Supplies the number of elements per row of the source matrix. ; ; CountN (r9) - Supplies the number of columns of the source matrix to copy. ; ; CountK - Supplies the number of rows of the source matrix to copy. ; ; ColumnSumBuffer - Supplies the address of the buffer to receive the sums of ; the elements along each of the columns. ; ; Return Value: ; ; None. ; ;-- NESTED_ENTRY MlasGemmU8U8CopyPackBAvx2, _TEXT rex_push_reg rbp push_reg rbx push_reg rsi alloc_stack (GemmU8U8CopyPackBFrame.SavedRsi) END_PROLOGUE mov rsi,rdx mov r10,GemmU8U8CopyPackBFrame.CountK[rsp] mov r11,GemmU8U8CopyPackBFrame.ColumnSumBuffer[rsp] vpcmpeqw ymm5,ymm5,ymm5 ; generate word vector [0xFFFF] vpsrlw ymm5,ymm5,15 ; generate word vector [0x0001] ; ; Zero initialize the padded stack buffers. ; vpxor xmm0,xmm0,xmm0 vmovdqu YMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp],ymm0 ; ; Process 16 columns of matrix B in a loop. ; sub r9,16 jb ProcessRemainingColumns ProcessNextColumnN16: vpxor xmm0,xmm0,xmm0 ; clear column accumulators vpxor xmm1,xmm1,xmm1 mov rdx,rsi add rsi,16 ; advance next matrix B by 16 columns mov rbx,r10 ; reload rows remaining sub rbx,2 jb ProcessRemainingRowsN16 ProcessNextRowLoopN16: vmovdqu xmm2,XMMWORD PTR [rdx] ; load 2 rows vmovdqu xmm3,XMMWORD PTR [rdx+r8] lea rdx,[rdx+r8*2] ; advance matrix B by 2 rows vpunpcklbw xmm4,xmm2,xmm3 ; interleave row data vpunpckhbw xmm3,xmm2,xmm3 vmovdqu XMMWORD PTR [rcx],xmm4 ; store interleaved rows vmovdqu XMMWORD PTR [rcx+16],xmm3 vpmovzxbw ymm4,xmm4 vpmovzxbw ymm3,xmm3 add rcx,32 ; advance matrix D by 32 bytes vpmaddwd ymm4,ymm4,ymm5 ; horizontal word+word=dword per row vpaddd ymm0,ymm0,ymm4 ; accumulate per column vpmaddwd ymm3,ymm3,ymm5 vpaddd ymm1,ymm1,ymm3 sub rbx,2 ; subtract rows remaining jae ProcessNextRowLoopN16 ProcessRemainingRowsN16: add rbx,2 ; correct for over-subtract above jz StoreColumnSumBufferN16 vpmovzxbw ymm4,XMMWORD PTR [rdx] vmovdqu YMMWORD PTR [rcx],ymm4 ; store interleaved rows vextracti128 xmm3,ymm4,1 vpmovzxbw ymm4,xmm4 vpmovzxbw ymm3,xmm3 vpmaddwd ymm4,ymm4,ymm5 ; horizontal word+word=dword per row vpaddd ymm0,ymm0,ymm4 ; accumulate per column vpmaddwd ymm3,ymm3,ymm5 vpaddd ymm1,ymm1,ymm3 add rcx,32 ; advance matrix D by 32 bytes StoreColumnSumBufferN16: vmovdqu YMMWORD PTR [r11],ymm0 vmovdqu YMMWORD PTR [r11+32],ymm1 add r11,16*4 ; advance column sum buffer by 16 dwords sub r9,16 ; subtract columns remaining jae ProcessNextColumnN16 ProcessRemainingColumns: add r9,16 ; correct for over-subtract above jnz ProcessColumnNUnaligned ; ; Restore non-volatile registers and return. ; ExitRoutine: vzeroupper add rsp,(GemmU8U8CopyPackBFrame.SavedRsi) BEGIN_EPILOGUE pop rsi pop rbx pop rbp ret ; ; Process the remaining columns of matrix B. ; ProcessColumnNUnaligned: vpxor xmm0,xmm0,xmm0 ; clear column accumulators vpxor xmm1,xmm1,xmm1 sub r10,2 jb ProcessRemainingRowsNUnaligned ProcessNextRowLoopNUnaligned: mov rdx,rsi .errnz GemmU8U8CopyPackBFrame.PaddedMatrixBData mov rbp,rsp ; GemmU8U8CopyPackBFrame.PaddedMatrixBData test r9b,8 ; (CountN & 8) != 0? jz CopyRemainingCountNLessThan8K2 mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax mov rax,QWORD PTR [rdx+r8] mov QWORD PTR [rbp+16],rax add rdx,8 ; advance matrix B add rbp,8 ; advance padded buffer destination CopyRemainingCountNLessThan8K2: test r9b,4 ; (CountN & 4) != 0? jz CopyRemainingCountNLessThan4K2 mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax mov eax,DWORD PTR [rdx+r8] mov DWORD PTR [rbp+16],eax add rdx,4 ; advance matrix B add rbp,4 ; advance padded buffer destination CopyRemainingCountNLessThan4K2: test r9b,2 ; (CountN & 2) != 0? jz CopyRemainingCountNLessThan2K2 movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax movzx eax,WORD PTR [rdx+r8] mov WORD PTR [rbp+16],ax add rdx,2 ; advance matrix B add rbp,2 ; advance padded buffer destination CopyRemainingCountNLessThan2K2: test r9b,1 ; (CountN & 1) != 0? jz ProcessPaddedMatrixBDataK2 movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al movzx eax,BYTE PTR [rdx+r8] mov BYTE PTR [rbp+16],al ProcessPaddedMatrixBDataK2: vmovdqu xmm2,XMMWORD PTR XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp] vmovdqu xmm3,XMMWORD PTR XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp+16] vpunpcklbw xmm4,xmm2,xmm3 ; interleave row data vpunpckhbw xmm3,xmm2,xmm3 vmovdqu XMMWORD PTR [rcx],xmm4 ; store interleaved rows vmovdqu XMMWORD PTR [rcx+16],xmm3 vpmovzxbw ymm4,xmm4 vpmovzxbw ymm3,xmm3 vpmaddwd ymm4,ymm4,ymm5 ; horizontal word+word=dword per row vpaddd ymm0,ymm0,ymm4 ; accumulate per column vpmaddwd ymm3,ymm3,ymm5 vpaddd ymm1,ymm1,ymm3 lea rsi,[rsi+r8*2] ; advance next matrix B by 2 rows add rcx,32 ; advance matrix D by 32 bytes sub r10,2 ; subtract columns remaining jae ProcessNextRowLoopNUnaligned ProcessRemainingRowsNUnaligned: add r10,2 jz StoreColumnSumBufferNUnaligned mov rdx,rsi .errnz GemmU8U8CopyPackBFrame.PaddedMatrixBData mov rbp,rsp ; GemmU8U8CopyPackBFrame.PaddedMatrixBData test r9b,8 ; (CountN & 8) != 0? jz CopyRemainingCountNLessThan8K1 mov rax,QWORD PTR [rdx] mov QWORD PTR [rbp],rax add rdx,8 ; advance matrix B add rbp,8 ; advance padded buffer destination CopyRemainingCountNLessThan8K1: test r9b,4 ; (CountN & 4) != 0? jz CopyRemainingCountNLessThan4K1 mov eax,DWORD PTR [rdx] mov DWORD PTR [rbp],eax add rdx,4 ; advance matrix B add rbp,4 ; advance padded buffer destination CopyRemainingCountNLessThan4K1: test r9b,2 ; (CountN & 2) != 0? jz CopyRemainingCountNLessThan2K1 movzx eax,WORD PTR [rdx] mov WORD PTR [rbp],ax add rdx,2 ; advance matrix B add rbp,2 ; advance padded buffer destination CopyRemainingCountNLessThan2K1: test r9b,1 ; (CountN & 1) != 0? jz ProcessPaddedMatrixBDataK1 movzx eax,BYTE PTR [rdx] mov BYTE PTR [rbp],al ProcessPaddedMatrixBDataK1: vpmovzxbw ymm4,XMMWORD PTR GemmU8U8CopyPackBFrame.PaddedMatrixBData[rsp] vmovdqu YMMWORD PTR [rcx],ymm4 ; store interleaved rows vextracti128 xmm3,ymm4,1 vpmovzxbw ymm4,xmm4 vpmovzxbw ymm3,xmm3 vpmaddwd ymm4,ymm4,ymm5 ; horizontal word+word=dword per row vpaddd ymm0,ymm0,ymm4 ; accumulate per column vpmaddwd ymm3,ymm3,ymm5 vpaddd ymm1,ymm1,ymm3 StoreColumnSumBufferNUnaligned: vmovdqu YMMWORD PTR [r11],ymm0 vmovdqu YMMWORD PTR [r11+32],ymm1 jmp ExitRoutine NESTED_END MlasGemmU8U8CopyPackBAvx2, _TEXT END