//
//  fdef_mulx.symcryptasm   Assembler code for large integer arithmetic in the default data format
//  using the bmi2 instructions mulx, adcx and adox
//  Expresses asm in a generic enough way to enable generation of MASM and GAS using the
//  symcryptasm_processor.py script and C preprocessor
//
// Copyright (c) Microsoft Corporation. Licensed under the MIT license.
//

#include "symcryptasm_shared.cppasm"

.macro ZEROREG  R
        xor     \R,\R
.endm

.macro ZEROREG_8  R0, R1, R2, R3, R4, R5, R6, R7
    ZEROREG \R0
    ZEROREG \R1
    ZEROREG \R2
    ZEROREG \R3
    ZEROREG \R4
    ZEROREG \R5
    ZEROREG \R6
    ZEROREG \R7
.endm

.macro MULADD18  R0, R1, R2, R3, R4, R5, R6, R7, pD, pA, pB, T0, T1, QH
    // \R0:R[7:1]:D[0] = A[7:0] * B[0] + D[0] + R[7:0]
    // Pre: Cy = Ov = 0
    // Post: Cy = Ov = 0

    xor     \T0, \T0      // clear flags
                        // makes following carry chains architecturally independent of previous ones

    mov     \QH, [\pB]
    adox    \R0, [\pD]

    mulx    \T1, \T0, [\pA + 0 * 8]
    adcx    \R0, \T0
    adox    \R1, \T1

    mov     [\pD], \R0

    mulx    \T1, \T0, [\pA + 1 * 8]
    adcx    \R1, \T0
    adox    \R2, \T1

    mulx    \T1, \T0, [\pA + 2 * 8]
    adcx    \R2, \T0
    adox    \R3, \T1

    mulx    \T1, \T0, [\pA + 3 * 8]
    adcx    \R3, \T0
    adox    \R4, \T1

    mulx    \T1, \T0, [\pA + 4 * 8]
    adcx    \R4, \T0
    adox    \R5, \T1

    mulx    \T1, \T0, [\pA + 5 * 8]
    adcx    \R5, \T0
    adox    \R6, \T1

    mulx    \T1, \T0, [\pA + 6 * 8]
    adcx    \R6, \T0
    adox    \R7, \T1

    mulx    \T1, \T0, [\pA + 7 * 8]
    adcx    \R7, \T0

    mov     \R0, 0
    adox    \R0, \R0

    adcx    \R0, \T1
.endm

.macro MULADD88  R0, R1, R2, R3, R4, R5, R6, R7, pD, pA, pB, T0, T1, QH
    // pre & post: Cy = Ov = 0
    // R[7-0]:D[7-0] = A[7:0] * B[7:0] + R[7:0] + D[7:0]

    MULADD18    \R0, \R1, \R2, \R3, \R4, \R5, \R6, \R7, \pD     , \pA, \pB     , \T0, \T1, \QH
    MULADD18    \R1, \R2, \R3, \R4, \R5, \R6, \R7, \R0, \pD +  8, \pA, \pB +  8, \T0, \T1, \QH
    MULADD18    \R2, \R3, \R4, \R5, \R6, \R7, \R0, \R1, \pD + 16, \pA, \pB + 16, \T0, \T1, \QH
    MULADD18    \R3, \R4, \R5, \R6, \R7, \R0, \R1, \R2, \pD + 24, \pA, \pB + 24, \T0, \T1, \QH
    MULADD18    \R4, \R5, \R6, \R7, \R0, \R1, \R2, \R3, \pD + 32, \pA, \pB + 32, \T0, \T1, \QH
    MULADD18    \R5, \R6, \R7, \R0, \R1, \R2, \R3, \R4, \pD + 40, \pA, \pB + 40, \T0, \T1, \QH
    MULADD18    \R6, \R7, \R0, \R1, \R2, \R3, \R4, \R5, \pD + 48, \pA, \pB + 48, \T0, \T1, \QH
    MULADD18    \R7, \R0, \R1, \R2, \R3, \R4, \R5, \R6, \pD + 56, \pA, \pB + 56, \T0, \T1, \QH
.endm


.macro HALF_SQUARE_NODIAG8  R0, R1, R2, R3, R4, R5, R6, R7, pD, pA, T0, T1, QH
    // pre & post: Cy = Ov = 0
    // R[7-0]:D[7-0] = D[7:0] + (A[0:7]^2 - \sum_{i=0}^7 (A[i] * 2^{64*i}) )/2
    // This is the component of the square that needs to be doubled, and then the diagonals added

    // Note that Dst[0] is not changed by this macro

    mov     \QH, [\pA + 0 * 8]            // \QH = A0
    mov     \R1, [\pD + 1 * 8]
    mov     \R2, [\pD + 2 * 8]
    mov     \R3, [\pD + 3 * 8]
    mov     \R4, [\pD + 4 * 8]
    mov     \R5, [\pD + 5 * 8]
    mov     \R6, [\pD + 6 * 8]
    mov     \R7, [\pD + 7 * 8]
    xor     \R0, \R0

    mulx    \T1, \T0, [\pA + 1 * 8]
    adcx    \R1, \T0
    adox    \R2, \T1

    mulx    \T1, \T0, [\pA + 2 * 8]
    adcx    \R2, \T0
    adox    \R3, \T1

    mulx    \T1, \T0, [\pA + 3 * 8]
    adcx    \R3, \T0
    adox    \R4, \T1

    mulx    \T1, \T0, [\pA + 4 * 8]
    adcx    \R4, \T0
    adox    \R5, \T1

    mulx    \T1, \T0, [\pA + 5 * 8]
    adcx    \R5, \T0
    adox    \R6, \T1

    mulx    \T1, \T0, [\pA + 6 * 8]
    adcx    \R6, \T0
    adox    \R7, \T1

    mulx    \T1, \T0, [\pA + 7 * 8]
    adcx    \R7, \T0
    mov     [\pD + 1 * 8], \R1

    adox    \R0, \R0
    adcx    \R0, \T1
    mov     [\pD + 2 * 8], \R2
    mov     \QH, [\pA + 1 * 8]        // \QH = A1

    //=======
    xor     \T0, \T0      // clear flags
                        // makes following carry chains architecturally independent of previous ones

    mulx    \T1, \T0, [\pA + 2 * 8]
    adcx    \R3, \T0
    adox    \R4, \T1

    mulx    \T1, \T0, [\pA + 3 * 8]
    adcx    \R4, \T0
    adox    \R5, \T1

    mulx    \T1, \T0, [\pA + 4 * 8]
    adcx    \R5, \T0
    adox    \R6, \T1

    mulx    \T1, \T0, [\pA + 5 * 8]
    adcx    \R6, \T0
    adox    \R7, \T1

    mulx    \T1, \T0, [\pA + 6 * 8]
    adcx    \R7, \T0
    adox    \R0, \T1

    mov     \QH, [\pA + 7 * 8]        // \QH = A7
    mov     \R1, 0
    mov     \R2, 0
    mov     [\pD + 3 * 8], \R3

    mulx    \T1, \T0, [\pA + 1 * 8]
    adcx    \R0, \T0
    adox    \R1, \T1                  // doesn't produce Ov as \T1 <= 0xff..fe and \R1=0

    mulx    \T1, \T0, [\pA + 2 * 8]
    adcx    \R1, \T0
    mov     [\pD + 4 * 8], \R4

    adcx    \R2, \T1
    mov     \QH, [\pA + 2 * 8]        // \QH = A2

    //======
    xor     \T0, \T0      // clear flags
                        // makes following carry chains architecturally independent of previous ones

    mulx    \T1, \T0, [\pA + 3 * 8]
    adcx    \R5, \T0
    adox    \R6, \T1

    mulx    \T1, \T0, [\pA + 4 * 8]
    adcx    \R6, \T0
    adox    \R7, \T1

    mulx    \T1, \T0, [\pA + 5 * 8]
    adcx    \R7, \T0
    adox    \R0, \T1

    mulx    \T1, \T0, [\pA + 6 * 8]
    adcx    \R0, \T0
    adox    \R1, \T1

    mov     \QH, [\pA + 4 * 8]        // \QH = A4
    mov     \R3, 0
    mov     \R4, 0

    mulx    \T1, \T0, [\pA + 5 * 8]
    adcx    \R1, \T0
    adox    \R2, \T1

    mulx    \T1,\T0, [\pA + 6 * 8]
    adcx    \R2, \T0
    adox    \R3, \T1                  // doesn't produce Ov as \T1 <= 0xff..fe and \R3=0

    mov     \QH, [\pA + 5 * 8]        // \QH = A5
    mov     [\pD + 5 * 8], \R5

    mulx    \T1, \T0, [\pA + 6 * 8]
    adcx    \R3, \T0
    adcx    \R4, \T1

    mov     \QH, [\pA + 3 * 8]        // \QH = A3
    mov     [\pD + 6 * 8], \R6

    //======
    xor     \T0, \T0      // clear flags
                        // makes following carry chains architecturally independent of previous ones

    mulx    \T1, \T0, [\pA + 4 * 8]
    adcx    \R7, \T0
    adox    \R0, \T1

    mulx    \T1, \T0, [\pA + 5 * 8]
    adcx    \R0, \T0
    adox    \R1, \T1

    mulx    \T1, \T0, [\pA + 6 * 8]
    adcx    \R1, \T0
    adox    \R2, \T1

    mulx    \T1, \T0, [\pA + 7 * 8]
    adcx    \R2, \T0
    adox    \R3, \T1

    mov     \QH, [\pA + 7 * 8]        // \QH = A7
    mov     \R5, 0
    mov     \R6, 0
    mov     [\pD + 7 * 8], \R7

    mulx    \T1, \T0, [\pA + 4 * 8]
    adcx    \R3, \T0
    adox    \R4, \T1

    mulx    \T1, \T0, [\pA + 5 * 8]
    adcx    \R4, \T0
    adox    \R5, \T1                  // doesn't produce Ov as \T1 <= 0xff..fe and \R5=0

    mulx    \T1, \T0, [\pA + 6 * 8]
    adcx    \R5, \T0
    adcx    \R6, \T1

    xor     \R7, \R7
.endm

.macro MONTGOMERY18  R0, R1, R2, R3, R4, R5, R6, R7, modInv, pMod, pMont, T0, T1, QH
    // Mont[0] = (modinv * \R0 mod 2^64)
    // \R0:R[7:1]:<phantom> = Mont[0] * Mod[7:0] + R[7:0]
    // Pre: -
    // Post: -

    mov     \QH, \R0
    imul    \QH, \modInv

    // Rather than add the low half of the first mulx to \R0 we can go ahead and set
    // up the Cy flag appropriately based on \R0 directly (the addition will always
    // result in 0 by construction), so we can have the result while imul is running

    // This has a small but measurable perf improvement on SKLX (~2% improvement for
    // 512b modmul)
    // and it seems unlikely that it can make the performance worse
    // My best guess as to why is that allowing this to execute a few cycles early
    // can reduce port contention when the macro is being speculatively executed
    or      \T0, -1          // Clear Cy and Ov
    adcx    \R0, \T0          // Set Cy when \R0 is non-zero
    mov     \R0, 0
    mov     [\pMont], \QH

    mulx    \T1, \T1, [\pMod + 0 * 8]
    adox    \R1, \T1

    mulx    \T1, \T0, [\pMod + 1 * 8]
    adcx    \R1, \T0
    adox    \R2, \T1

    mulx    \T1, \T0, [\pMod + 2 * 8]
    adcx    \R2, \T0
    adox    \R3, \T1

    mulx    \T1, \T0, [\pMod + 3 * 8]
    adcx    \R3, \T0
    adox    \R4, \T1

    mulx    \T1, \T0, [\pMod + 4 * 8]
    adcx    \R4, \T0
    adox    \R5, \T1

    mulx    \T1, \T0, [\pMod + 5 * 8]
    adcx    \R5, \T0
    adox    \R6, \T1

    mulx    \T1, \T0, [\pMod + 6 * 8]
    adcx    \R6, \T0
    adox    \R7, \T1

    mulx    \T1, \T0, [\pMod + 7 * 8]
    adcx    \R7, \T0

    adcx    \R0, \R0
    adox    \R0, \T1
.endm

.macro SYMCRYPT_SQUARE_DIAG  index, src_reg, dest_reg, T0, T1, T2, T3, QH
    mov     \QH, [\src_reg + 8 * \index]
    mov     \T0, [\dest_reg + 16 * \index]
    mov     \T1, [\dest_reg + 16 * \index + 8]
    mulx    \T3, \T2, \QH
    adcx    \T2, \T0
    adox    \T2, \T0
    adcx    \T3, \T1
    adox    \T3, \T1
    mov     [\dest_reg + 16 * \index], \T2
    mov     [\dest_reg + 16 * \index + 8], \T3
.endm

// VOID
// SYMCRYPT_CALL
// SymCryptFdefRawMulMulx(
//     _In_reads_(nDigits1*SYMCRYPT_FDEF_DIGIT_NUINT32)                PCUINT32    pSrc1,
//                                                                     UINT32      nDigits1,
//     _In_reads_(nDigits2*SYMCRYPT_FDEF_DIGIT_NUINT32)                PCUINT32    pSrc2,
//                                                                     UINT32      nDigits2,
//     _Out_writes_((nDigits1+nDigits2)*SYMCRYPT_FDEF_DIGIT_NUINT32)   PUINT32     pDst )

#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
#define Q13 r15
#define D13 r15d
#define W13 r15w
#define B13 r15b
PLATFORM_NAME_MANGLE(SymCryptFdefRawMulMulx):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefRawMulMulx)
.def PLATFORM_NAME_MANGLE(SymCryptFdefRawMulMulx); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefRawMulMulx)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12
push Q13
.seh_pushreg Q13

.seh_endprologue

mov Q2, QH
mov Q5, [rsp + 104]

        shl     Q4, 6
        mov     [rsp + 72 /*MEMSLOT0*/], Q4
        mov     [rsp + 80 /*MEMSLOT1*/], D2

        // First we wipe nDigits2 of the result (size of in)
        mov     Q6, Q5

        // Wipe destination for nDigit2 blocks
        xorps   xmm0,xmm0               // Zero register for 16-byte wipes
        mov     Q0, Q4

SymCryptFdefRawMulMulxWipeLoop:
        movaps      [Q6],xmm0
        movaps      [Q6+16],xmm0            // Wipe 32 bytes
        movaps      [Q6+32],xmm0            // Wipe 32 bytes
        movaps      [Q6+48],xmm0            // Wipe 32 bytes
        add         Q6, 64
        sub         Q0, 64
        jnz         SymCryptFdefRawMulMulxWipeLoop


SymCryptFdefRawMulxOuterLoop:

        ZEROREG_8   Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13      // Leaves Cy = Ov = 0

SymCryptFdefRawMulMulxInnerLoop:

        // Register allocation in loops:
        // Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13        8-word carry
        // Q0, Q2                                    temps for multiplication
        // Q1, Q3                                    pSrc1, pSrc2 running pointers
        // Q4                                        inner loop counter
        // QH                                        fixed input reg for multiplication
        // Q5                                        Destination running pointer inner loop
        // slot0                                     nDigits2*64
        // slot1                                     outer loop counter

        MULADD88  Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13, Q5, Q1, Q3, Q0, Q2, QH

        add     Q3, 64              // Src2 ptr
        add     Q5, 64

        sub     D4, 64                            // sets Cy = Ov = 0 because 64*nDigits2 < 2^32
        jnz     SymCryptFdefRawMulMulxInnerLoop

        // Write the 8-word carry-out to the destination
        mov     [Q5 + 0*8], Q6
        mov     [Q5 + 1*8], Q7
        mov     [Q5 + 2*8], Q8
        mov     [Q5 + 3*8], Q9
        mov     [Q5 + 4*8], Q10
        mov     [Q5 + 5*8], Q11
        mov     [Q5 + 6*8], Q12
        mov     [Q5 + 7*8], Q13

        // set up for next iteration
        // reload 64*nDigits2
        mov     Q4, [rsp + 72 /*MEMSLOT0*/]

        // reset Q5 & increment
        sub     Q5, Q4
        add     Q5, 64
        // reset Q3
        sub     Q3, Q4

        // update PSrc1
        add     Q1, 64

        // nDigits1 loop counter
        mov     D2, [rsp + 80 /*MEMSLOT1*/]
        sub     D2, 1                              // sets Cy = Ov = 0 because nDigits1 < 2^32 / 64
        mov     [rsp + 80 /*MEMSLOT1*/], D2
        jnz     SymCryptFdefRawMulxOuterLoop


pop Q13
pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12
#undef Q13
#undef D13
#undef W13
#undef B13

// VOID
// SYMCRYPT_CALL
// SymCryptFdefRawSquareMulx(
//     _In_reads_(nDigits*SYMCRYPT_FDEF_DIGIT_NUINT32)         PCUINT32    pSrc,
//                                                             UINT32      nDigits,
//     _Out_writes_(2*nDigits*SYMCRYPT_FDEF_DIGIT_NUINT32)     PUINT32     pDst )

#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
#define Q13 r15
#define D13 r15d
#define W13 r15w
#define B13 r15b
PLATFORM_NAME_MANGLE(SymCryptFdefRawSquareMulx):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefRawSquareMulx)
.def PLATFORM_NAME_MANGLE(SymCryptFdefRawSquareMulx); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefRawSquareMulx)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12
push Q13
.seh_pushreg Q13

.seh_endprologue

mov Q2, QH

        // Q1 = pSrc
        // Q2 = nDigits
        // Q3 = pDst

        // Save parameters for phase 2

        mov     [rsp + 72 /*MEMSLOT0*/], Q1   // save pSrc
        mov     [rsp + 80 /*MEMSLOT1*/], Q2   // save nDigits
        mov     [rsp + 88 /*MEMSLOT2*/], Q3   // save pDst

        shl     Q2, 6       // nDigits * 64 = # bytes in Src to process
        mov     [rsp + 96 /*MEMSLOT3*/], Q2   // save # bytes in Src to process

        // Wipe destination for nDigits blocks
        xor     Q0, Q0
        mov     Q5, Q3
        mov     Q4, Q2

SymCryptFdefRawSquareMulxWipeLoop:
        // we use 8-byte writes as we will be reading this very soon in 8-byte chunks, and this way the store-load
        // forwarding works
        mov     [Q5     ], Q0
        mov     [Q5 +  8], Q0
        mov     [Q5 + 16], Q0
        mov     [Q5 + 24], Q0
        mov     [Q5 + 32], Q0
        mov     [Q5 + 40], Q0
        mov     [Q5 + 48], Q0
        mov     [Q5 + 56], Q0
        add     Q5, 64
        sub     Q4, 64
        jnz     SymCryptFdefRawSquareMulxWipeLoop

        // Cy = Ov = 0 here because the last 'sub Q4,64' yielded 0

SymCryptFdefRawSquareMulxOuterLoop:

        HALF_SQUARE_NODIAG8 Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13, Q3, Q1, Q0, Q4, QH

        sub     Q2, 64
        jz      SymCryptFdefRawSquareMulxPhase2     // end of phase 1

        lea     Q5, [Q1 + 64]
        lea     Q3, [Q3 + 64]

SymCryptFdefRawSquareMulxInnerLoop:
        // Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13        8-word carry
        // Q0, Q4                                    temps for multiplication
        // Q1                                        pSrc running pointer outer loop
        // Q2                                        # bytes left in pSrc to process in the inner loop
        // Q3                                        pDst running pointer inner loop
        // QH                                        fixed input reg for multiplication
        // Q5                                        pSrc running pointer inner loop

        // slot0                                     pSrc (used for final pass)
        // slot1                                     nDigits (used for final pass)
        // slot2                                     pDst (used for final pass)
        // slot3                                     # bytes to process from pSrc in this iteration

        MULADD88    Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13, Q3, Q1, Q5, Q0, Q4, QH

        add     Q3, 64
        add     Q5, 64

        sub     Q2, 64                  // Sets Cy = Ov = 0 because nDigits < 2^32 / bits_per_digit
        jnz     SymCryptFdefRawSquareMulxInnerLoop

        // Write the 8-word carry-out to the destination
        mov     [Q3 + 0*8], Q6
        mov     [Q3 + 1*8], Q7
        mov     [Q3 + 2*8], Q8
        mov     [Q3 + 3*8], Q9
        mov     [Q3 + 4*8], Q10
        mov     [Q3 + 5*8], Q11
        mov     [Q3 + 6*8], Q12
        mov     [Q3 + 7*8], Q13

        mov     Q2, [rsp + 96 /*MEMSLOT3*/]   // restore # bytes in Src to process next

        add     Q1, 64                                  // Shift outer Src pointer by 1 digit
        sub     Q3, Q2                                  // reset output ptr
        add     Q3, 128                                 // Shift output ptr by 2 digits

        sub     Q2, 64                                  // Reduce number of bytes to process by 1 digit
        mov     [rsp + 96 /*MEMSLOT3*/], Q2

        jmp     SymCryptFdefRawSquareMulxOuterLoop


SymCryptFdefRawSquareMulxPhase2:
        // Cy = Ov = 0 because last 'sub Q2, 64' resulted in 0

        // Write the 8-word carry-out to the destination
        mov     [Q3 +  8*8], Q6
        mov     [Q3 +  9*8], Q7
        mov     [Q3 + 10*8], Q8
        mov     [Q3 + 11*8], Q9
        mov     [Q3 + 12*8], Q10
        mov     [Q3 + 13*8], Q11
        mov     [Q3 + 14*8], Q12
        mov     [Q3 + 15*8], Q13

        // Compute diagonals, and add double the result so far

        mov     Q1, [rsp + 72 /*MEMSLOT0*/]
        mov     Q2, [rsp + 80 /*MEMSLOT1*/]
        mov     Q3, [rsp + 88 /*MEMSLOT2*/]

        // We can't keep the carries in Cy and Ov because there is no way to do a loop counter
        // without touching the Ov flag.
        // So we set the Ov carry in Q0, and retain a zero in Q4
        xor     Q0, Q0
        xor     Q4, Q4

SymCryptFdefRawSquareMulxDiagonalsLoop:
        // Cy = carry in
        // Q0 = carry in (1 bit)
        // Ov = 0

        // First word is different to handle the carry
        // SYMCRYPT_SQUARE_DIAG    0, Q1, Q3, Q5, Q6, Q7, Q8, QH
        mov     QH, [Q1]
        mov     Q5, [Q3]
        mov     Q6, [Q3 + 8]
        mulx    Q8, Q7, QH
        adcx    Q7, Q0              // add both carries
        adcx    Q8, Q4              // Q4 = 0 - now Cy = 0 because result of multiply <= ff..fe00..01

        adcx    Q7, Q5
        adox    Q7, Q5
        adcx    Q8, Q6
        adox    Q8, Q6
        mov     [Q3], Q7
        mov     [Q3 + 8], Q8

        SYMCRYPT_SQUARE_DIAG 1, Q1, Q3, Q5, Q6, Q7, Q8, QH
        SYMCRYPT_SQUARE_DIAG 2, Q1, Q3, Q5, Q6, Q7, Q8, QH
        SYMCRYPT_SQUARE_DIAG 3, Q1, Q3, Q5, Q6, Q7, Q8, QH
        SYMCRYPT_SQUARE_DIAG 4, Q1, Q3, Q5, Q6, Q7, Q8, QH
        SYMCRYPT_SQUARE_DIAG 5, Q1, Q3, Q5, Q6, Q7, Q8, QH
        SYMCRYPT_SQUARE_DIAG 6, Q1, Q3, Q5, Q6, Q7, Q8, QH
        SYMCRYPT_SQUARE_DIAG 7, Q1, Q3, Q5, Q6, Q7, Q8, QH

        // Move the Ov flag into Q0
        mov     D0, D4
        adox    D0, D4

        // There is no way to do a loop counter without overwriting the Ov flag
        // Even the 'dec' instruction touches it, and LAHF/SAHF doesn't load/store the Ov flag.
        // We can't push/pop efl in a function body

        lea     Q1, [Q1 + 64]
        lea     Q3, [Q3 + 128]
        dec     Q2
        jnz     SymCryptFdefRawSquareMulxDiagonalsLoop


pop Q13
pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12
#undef Q13
#undef D13
#undef W13
#undef B13

// VOID
// SYMCRYPT_CALL
// SymCryptFdefMontgomeryReduceMulx(
//     _In_                            PCSYMCRYPT_MODULUS      pmMod,
//     _Inout_                         PUINT32                 pSrc,
//     _Out_                           PUINT32                 pDst )
#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
#define Q13 r15
#define D13 r15d
#define W13 r15w
#define B13 r15b
PLATFORM_NAME_MANGLE(SymCryptFdefMontgomeryReduceMulx):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefMontgomeryReduceMulx)
.def PLATFORM_NAME_MANGLE(SymCryptFdefMontgomeryReduceMulx); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefMontgomeryReduceMulx)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12
push Q13
.seh_pushreg Q13

.seh_endprologue

mov Q2, QH

        mov     [rsp + 72 /*MEMSLOT0*/], Q1
        mov     [rsp + 80 /*MEMSLOT1*/], Q2
        mov     [rsp + 88 /*MEMSLOT2*/], Q3

        mov     D0, [Q1 + SymCryptModulusNdigitsOffsetAmd64]
        mov     [rsp + 96 /*MEMSLOT3*/], D0
        // CntOuter = nDigits - using first half of slot3

        xor     D4, D4
        mov     [rsp + 96 /*MEMSLOT3*/ + 4], D4
        // HighCarry = 0 - using second half of slot3

SymCryptFdefMontgomeryReduceMulxOuterLoop:
        // Q1 = pmMod
        // Q2 = pSrc = tmp buffer that we will reduce
        mov     Q6, [Q2 + 0 * 8]
        mov     Q7, [Q2 + 1 * 8]
        mov     Q8, [Q2 + 2 * 8]
        mov     Q9, [Q2 + 3 * 8]
        mov     Q10, [Q2 + 4 * 8]
        mov     Q11, [Q2 + 5 * 8]
        mov     Q12, [Q2 + 6 * 8]
        mov     Q13, [Q2 + 7 * 8]

        mov     Q3, [Q1 + SymCryptModulusInv64OffsetAmd64]      // inv64
        mov     D4, [Q1 + SymCryptModulusNdigitsOffsetAmd64]
        lea     Q1, [Q1 + SymCryptModulusValueOffsetAmd64]      // modulus value

        // Q2 = value to reduce
        // Q6 - Q13 = Q2[0..7]
        // Q1 = modulus value
        // Q3 = modinv

        MONTGOMERY18    Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13,  Q3, Q1, Q2 + (0 * 8), Q0, Q5, QH
        MONTGOMERY18    Q7, Q8, Q9, Q10, Q11, Q12, Q13, Q6,  Q3, Q1, Q2 + (1 * 8), Q0, Q5, QH
        MONTGOMERY18    Q8, Q9, Q10, Q11, Q12, Q13, Q6, Q7,  Q3, Q1, Q2 + (2 * 8), Q0, Q5, QH
        MONTGOMERY18    Q9, Q10, Q11, Q12, Q13, Q6, Q7, Q8,  Q3, Q1, Q2 + (3 * 8), Q0, Q5, QH
        MONTGOMERY18    Q10, Q11, Q12, Q13, Q6, Q7, Q8, Q9,  Q3, Q1, Q2 + (4 * 8), Q0, Q5, QH
        MONTGOMERY18    Q11, Q12, Q13, Q6, Q7, Q8, Q9, Q10,  Q3, Q1, Q2 + (5 * 8), Q0, Q5, QH
        MONTGOMERY18    Q12, Q13, Q6, Q7, Q8, Q9, Q10, Q11,  Q3, Q1, Q2 + (6 * 8), Q0, Q5, QH
        MONTGOMERY18    Q13, Q6, Q7, Q8, Q9, Q10, Q11, Q12,  Q3, Q1, Q2 + (7 * 8), Q0, Q5, QH

        // Q6 - Q13 = carry from multiply-add
        // Q2[0..7] = Montgomery factors

        mov     Q3, Q2         // factor to multiply by
        add     Q1, 64
        add     Q2, 64

        dec     D4
        jz      SymCryptFdefMontgomeryReduceMulxInnerLoopDone

SymCryptFdefMontgomeryReduceMulxInnerLoop:

        // Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13        8-word carry
        // Q0, Q5                                    temps for multiplication
        // Q1                                        running pointer pMod inner loop
        // Q2                                        running pointer pSrc inner loop
        // Q3                                        Montgomery factors for this row
        // D4                                        loop ctr
        // QH                                        fixed input reg for multiplication

        MULADD88    Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13,  Q2, Q1, Q3, Q0, Q5, QH
            // pre & post: Cy = Ov = 0
            // Q13..Q6:Q2[7-0] = R[7-0]:D[7-0] = A[7:0] * B[7:0] + R[7:0] + D[7:0]
            // QH is volatile

        add     Q1, 64
        add     Q2, 64
        dec     D4
        jnz     SymCryptFdefMontgomeryReduceMulxInnerLoop


SymCryptFdefMontgomeryReduceMulxInnerLoopDone:

        // We have an 8-word carry here, which we need to add to the in-memory buffer and retain a carry
        // We also saved a 1-bit carry from the previous outer loop
        mov     D5, [rsp + 96 /*MEMSLOT3*/ + 4]
        // move carry into Cy flag
        neg     D5

        // We do this in separate instructions to help the instruction decoder build up a lead...
        mov     Q0, [Q2 + 0 * 8]
        adc     Q0, Q6
        mov     [Q2 + 0 * 8], Q0

        mov     Q5, [Q2 + 1 * 8]
        adc     Q5, Q7
        mov     [Q2 + 1 * 8], Q5

        mov     Q0, [Q2 + 2 * 8]
        adc     Q0, Q8
        mov     [Q2 + 2 * 8], Q0

        mov     Q5, [Q2 + 3 * 8]
        adc     Q5, Q9
        mov     [Q2 + 3 * 8], Q5

        mov     Q0, [Q2 + 4 * 8]
        adc     Q0, Q10
        mov     [Q2 + 4 * 8], Q0

        mov     Q5, [Q2 + 5 * 8]
        adc     Q5, Q11
        mov     [Q2 + 5 * 8], Q5

        mov     Q0, [Q2 + 6 * 8]
        adc     Q0, Q12
        mov     [Q2 + 6 * 8], Q0

        mov     Q5, [Q2 + 7 * 8]
        adc     Q5, Q13
        mov     [Q2 + 7 * 8], Q5

        adc     D4, D4                // D4 = carry (D4 was previously zero)
        mov     [rsp + 96 /*MEMSLOT3*/ + 4], D4

        mov     Q2, [rsp + 80 /*MEMSLOT1*/]
        add     Q2, 64
        mov     [rsp + 80 /*MEMSLOT1*/], Q2

        mov     Q1, [rsp + 72 /*MEMSLOT0*/]

        mov     D0, [rsp + 96 /*MEMSLOT3*/]
        dec     D0
        mov     [rsp + 96 /*MEMSLOT3*/], D0

        jnz     SymCryptFdefMontgomeryReduceMulxOuterLoop

        // D4 = output carry

        mov     D6, [Q1 + SymCryptModulusNdigitsOffsetAmd64]
        lea     Q1, [Q1 + SymCryptModulusValueOffsetAmd64]                    // modulus value

        mov     Q3, [rsp + 88 /*MEMSLOT2*/]

        // Q2 = result buffer pointer
        // D6 = # digits
        // Q1 = modulus value
        // Q3 = Dst

        // copy these values for the masked copy loop
        mov     D7, D6      // nDigits
        mov     Q8, Q2      // result buffer
        mov     Q9, Q3      // destination pointer

        // pDst = Reduction result - Modulus

SymCryptFdefMontgomeryReduceMulxSubLoop:
        mov     Q0,[Q2 + 0 * 8]
        sbb     Q0,[Q1 + 0 * 8]
        mov     [Q3 + 0 * 8], Q0

        mov     Q5,[Q2 + 1 * 8]
        sbb     Q5,[Q1 + 1 * 8]
        mov     [Q3 + 1 * 8], Q5

        mov     Q0,[Q2 + 2 * 8]
        sbb     Q0,[Q1 + 2 * 8]
        mov     [Q3 + 2 * 8], Q0

        mov     Q5,[Q2 + 3 * 8]
        sbb     Q5,[Q1 + 3 * 8]
        mov     [Q3 + 3 * 8], Q5

        mov     Q0,[Q2 + 4 * 8]
        sbb     Q0,[Q1 + 4 * 8]
        mov     [Q3 + 4 * 8], Q0

        mov     Q5,[Q2 + 5 * 8]
        sbb     Q5,[Q1 + 5 * 8]
        mov     [Q3 + 5 * 8], Q5

        mov     Q0,[Q2 + 6 * 8]
        sbb     Q0,[Q1 + 6 * 8]
        mov     [Q3 + 6 * 8], Q0

        mov     Q5,[Q2 + 7 * 8]
        sbb     Q5,[Q1 + 7 * 8]
        mov     [Q3 + 7 * 8], Q5

        lea     Q2, [Q2 + 64]
        lea     Q1, [Q1 + 64]
        lea     Q3, [Q3 + 64]
        dec     D6
        jnz     SymCryptFdefMontgomeryReduceMulxSubLoop

        // now a masked copy from the reduction buffer to the destination.
        // copy if high carry = 0 and Cy = 1
        sbb     D4, 0
        // D4 = copy mask, ff...ff  if copy, 0 of no copy

        movd    xmm0, D4           // xmm0[0] = mask
        pcmpeqd xmm1, xmm1          // xmm1 = ff...ff
        pshufd  xmm0, xmm0, 0       // xmm0[0..3] = mask
        pxor    xmm1, xmm0          // xmm1 = not Mask

SymCryptFdefMontgomeryReduceMulxMaskedCopyLoop:
        movdqa  xmm2, [Q8 + 0 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q9 + 0 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q9 + 0 * 16], xmm2

        movdqa  xmm2, [Q8 + 1 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q9 + 1 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q9 + 1 * 16], xmm2

        movdqa  xmm2, [Q8 + 2 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q9 + 2 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q9 + 2 * 16], xmm2

        movdqa  xmm2, [Q8 + 3 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q9 + 3 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q9 + 3 * 16], xmm2

        // Move on to the next digit

        add     Q8, 64
        add     Q9, 64
        dec     D7
        jnz     SymCryptFdefMontgomeryReduceMulxMaskedCopyLoop


pop Q13
pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12
#undef Q13
#undef D13
#undef W13
#undef B13

.macro MULADD_LOADSTORE18  pS, pM, pD, QH, Tc, T0, T1
    // \Tc:D[7:0] = (M[7:0] * K) + S[7:0] + \Tc
    //
    // \QH = K
    // \T0, \T1 = scratch
    // \Tc is carry in at start and carry out at end (in [0,2^64-1])

    xor     \T0, \T0 // clear flags

    mulx    \T1, \T0, [\pM + 0 * 8]
    adox    \T0, \Tc
    adcx    \T0, [\pS + 0 * 8]
    mov     [\pD + 0 * 8], \T0

    mulx    \Tc, \T0, [\pM + 1 * 8]
    adox    \T0, \T1
    adcx    \T0, [\pS + 1 * 8]
    mov     [\pD + 1 * 8], \T0

    mulx    \T1, \T0, [\pM + 2 * 8]
    adox    \T0, \Tc
    adcx    \T0, [\pS + 2 * 8]
    mov     [\pD + 2 * 8], \T0

    mulx    \Tc, \T0, [\pM + 3 * 8]
    adox    \T0, \T1
    adcx    \T0, [\pS + 3 * 8]
    mov     [\pD + 3 * 8], \T0

    mulx    \T1, \T0, [\pM + 4 * 8]
    adox    \T0, \Tc
    adcx    \T0, [\pS + 4 * 8]
    mov     [\pD + 4 * 8], \T0

    mulx    \Tc, \T0, [\pM + 5 * 8]
    adox    \T0, \T1
    adcx    \T0, [\pS + 5 * 8]
    mov     [\pD + 5 * 8], \T0

    mulx    \T1, \T0, [\pM + 6 * 8]
    adox    \T0, \Tc
    adcx    \T0, [\pS + 6 * 8]
    mov     [\pD + 6 * 8], \T0

    mulx    \Tc, \T0, [\pM + 7 * 8]
    adox    \T0, \T1
    adcx    \T0, [\pS + 7 * 8]
    mov     [\pD + 7 * 8], \T0

    mov     \T1, 0
    adox    \Tc, \T1 // cannot overflow
    adcx    \Tc, \T1 // cannot overflow
.endm

.macro SHIFTRIGHT2  pD, index, shrVal, shrMask, shlVal, Tc, T0, T1
    // \T0 = D[\index + 1]
    // D[\index + 1] = \Tc:\T0 >> \shrVal (== ((\T0>>\shrVal)&\shrMask) | (\Tc<<\shlVal))
    // \Tc = D[\index]
    // D[\index] = \T0:\Tc >> \shrVal
    //
    // \shrVal in [1,64]
    // \shrMask = (\shrVal==64) ? 0 : -1
    // \shlVal = 64-\shrVal
    //
    // \T0, \T1 = scratch
    // Value of \Tc is bits to shift in the top at start, and bits to shift
    // into next macro invocation at end

    mov     \T0, [\pD + ((\index+1)*8)]
    shrx    \T1, \T0, \shrVal
    shlx    \Tc, \Tc, \shlVal
    and     \T1, \shrMask
    or      \T1, \Tc
    mov     [\pD + ((\index+1)*8)], \T1

    mov     \Tc, [\pD + (\index*8)]
    shrx    \T1, \Tc, \shrVal
    shlx    \T0, \T0, \shlVal
    and     \T1, \shrMask
    or      \T1, \T0
    mov     [\pD + (\index*8)], \T1
.endm

//VOID
//SYMCRYPT_CALL
//SymCryptFdefModDivSmallPow2Mulx(
//    _In_                            PCSYMCRYPT_MODULUS      pmMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   peSrc,
//    _In_range_(1, 64)               UINT32                  exp,
//    _Out_                           PSYMCRYPT_MODELEMENT    peDst );

#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
PLATFORM_NAME_MANGLE(SymCryptFdefModDivSmallPow2Mulx):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModDivSmallPow2Mulx)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModDivSmallPow2Mulx); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModDivSmallPow2Mulx)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8

.seh_endprologue

mov Q2, QH

        mov     Q7, [Q2]
        mov     QH, [Q1 + SymCryptModulusInv64OffsetAmd64]      // inv64

        imul    QH, Q7  // QH = inv64 * peSrc[0]
                        // this is the multiple of the modulus that must be added to peSrc
                        // s.t. the least significant **64** bits are all zero

        // Form bitmask to mask out bottom exp bits from multiplication result
        xor     D0, D0
        mov     Q7, -1
        sub     D0, D3
        shrx    Q7, Q7, Q0      // mask = -1 >> (-exp % 64)
                                //      = 0xffffffffffffffff >> (64-exp)
                                //      = (((UINT65)1)<<exp)-1

        and     QH, Q7          // Extract the least significant exp bits from QH to produce K
                                // K is the minimal multiple of the modulus that when added to peSrc
                                // sets the least significant **exp** bits to zero

        mov     D0, [Q1 + SymCryptModulusNdigitsOffsetAmd64]            // numDigits, used directly as loop counter
        lea     Q5, [Q1 + SymCryptModulusValueOffsetAmd64]              // pointer to modulus value

        xor     Q6, Q6 // zero one word carry

SymCryptFdefModDivSmallPow2MulxMulAddLoop:
        // Set peDst = peSrc + (K * mod)
        // Also has up to 64b of carry out the top which is stored in Q6 at the end of the loop
        //
        // D0   loop ctr (one iteration per digit)
        // QH   K
        // Q2   running pointer peSrc inner loop (we directly modify Q2 as we only need to read peSrc once)
        // Q5   running pointer pMod inner loop
        // Q4   running pointer peDst inner loop (we directly modify Q4 as we read/write peDst in reverse in second loop)
        // Q6   one word carry
        // Q7, Q8 scratch

        MULADD_LOADSTORE18 Q2, Q5, Q4, QH, Q6, Q7, Q8

        // Move on to the next digit
        add     Q2, 64
        add     Q5, 64
        add     Q4, 64

        dec     D0
        jnz     SymCryptFdefModDivSmallPow2MulxMulAddLoop

        mov     D0, [Q1 + SymCryptModulusNdigitsOffsetAmd64]            // numDigits, used directly as loop counter
        mov     D1, 64
        xor     Q7, Q7
        sub     D1, D3  // shlVal (also sets ZF iff exp == 64)
        mov     Q2, -1
        cmovz   Q2, Q7  // shrMask = ((exp == 64) ? 0 : -1)

        sub     Q4, 64  // Q4 points to the most significant digit of peDst

SymCryptFdefModDivSmallPow2MulxShiftRightLoop:
        // Set peDst = peDst >> exp
        // We work backwards (from MSB to LSB) so we avoid having to do special handling of the one word
        // carry out of the top of the multiply add loop
        //
        // D0   loop ctr (one iteration per digit)
        // Q1   shlVal
        // Q2   shrMask
        // Q3   shrVal
        // Q4   running pointer peDst inner loop (decreasing)
        // Q6   one word carry
        // Q7, Q8 scratch

        SHIFTRIGHT2 Q4, 6, Q3, Q2, Q1, Q6, Q7, Q8
        SHIFTRIGHT2 Q4, 4, Q3, Q2, Q1, Q6, Q7, Q8
        SHIFTRIGHT2 Q4, 2, Q3, Q2, Q1, Q6, Q7, Q8
        SHIFTRIGHT2 Q4, 0, Q3, Q2, Q1, Q6, Q7, Q8

        sub     Q4, 64

        dec     D0
        jnz     SymCryptFdefModDivSmallPow2MulxShiftRightLoop


pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8

// --------------------------------
// 256-bit size specific functions
// --------------------------------

//VOID
//SYMCRYPT_CALL
//SymCryptFdefModAddMulx256Asm(
//    _In_                            PCSYMCRYPT_MODULUS      pmMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   peSrc1,
//    _In_                            PCSYMCRYPT_MODELEMENT   peSrc2,
//    _Out_                           PSYMCRYPT_MODELEMENT    peDst,
//    _Out_writes_bytes_( cbScratch ) PBYTE                   pbScratch,
//                                    SIZE_T                  cbScratch )

#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q2 rdx
#define D2 edx
#define W2 dx
#define B2 dl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q5 r10
#define D5 r10d
#define W5 r10w
#define B5 r10b
#define Q6 r11
#define D6 r11d
#define W6 r11w
#define B6 r11b
#define Q7 rsi
#define D7 esi
#define W7 si
#define B7 sil
#define Q8 rdi
#define D8 edi
#define W8 di
#define B8 dil
PLATFORM_NAME_MANGLE(SymCryptFdefModAddMulx256Asm):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModAddMulx256Asm)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModAddMulx256Asm); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModAddMulx256Asm)

.byte 0x40; push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8

.seh_endprologue


        // Q1 = pmMod
        // Q2 = peSrc1
        // Q3 = peSrc2
        // Q4 = peDst

        add     Q1, SymCryptNegDivisorSingleDigitOffsetAmd64

        // compute Src1 + Src2 into (Q0, Q5, Q6, Q2), Cy = addition carry
        // Compute sum + (-Mod) into (Q7, Q8, Q3, Q1), Ov = inverted subtraction borrow

        xor     Q0, Q0 // Clear the flags

        mov     Q0, [Q2 + 0*8]
        adcx    Q0, [Q3 + 0*8]

        mov     Q5, [Q2 + 1*8]
        adcx    Q5, [Q3 + 1*8]

        mov     Q6, [Q2 + 2*8]
        adcx    Q6, [Q3 + 2*8]

        mov     Q2, [Q2 + 3*8]
        adcx    Q2, [Q3 + 3*8]

        mov     Q7, [Q1 + 0*8]
        adox    Q7, Q0

        mov     Q8, [Q1 + 1*8]
        adox    Q8, Q5

        mov     Q3, [Q1 + 2*8]
        adox    Q3, Q6

        mov     Q1, [Q1 + 3*8]
        adox    Q1, Q2

        // Choose between the two
        // addition carry = 1, then subtraction borrow = 1 and we pick the 2nd result.
        // addition carry = 0 and subtraction borrow = 0: pick 2nd result
        // addition carry = 0 and subtraction borrow = 1: pick first result

        cmovc   Q0, Q7
        cmovc   Q5, Q8
        cmovc   Q6, Q3
        cmovc   Q2, Q1

        cmovo   Q0, Q7
        cmovo   Q5, Q8
        cmovo   Q6, Q3
        cmovo   Q2, Q1

        mov     [Q4 + 0*8], Q0
        mov     [Q4 + 1*8], Q5
        mov     [Q4 + 2*8], Q6
        mov     [Q4 + 3*8], Q2


pop Q8
pop Q7
ret
.seh_endproc
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8

.macro MUL_AND_MONTGOMERY_REDUCE14_INTERLEAVE  T0, T1, QH, pA, Aoff, pB, pM, K, R0, R1, R2, R3, R4, R5
    // (\R1, \R2, \R3, \R4, \R5) = A[\Aoff] * (B0..3) + (\R1, \R2, \R3, \R4, \R5)
    // (xx, \R1, \R2, \R3, \R4, \R5, \R0) = \K * (M0..3) + (\R0, \R1, \R2, \R3, \R4, \R5)
    // \K = modInv * \R1
    // Note \K is chosen using pMod's montgomery inverse s.t. xx would get set to 0
    // \pM is a pointer to the bytes of the modulus (i.e. offset into SYMCRYPT_MODULUS structure)
    // \R5 is 0 or 1 at start
    // \R0 is 0 or 1 at end
    // \T0, \T1 = scratch

    mov     \QH, [\pA + \Aoff]
    xor     \T0, \T0      // clear flags
                        // makes following carry chains architecturally independent of previous ones

    mulx    \T1, \T0, [\pB + 0*8]
    adox    \R1, \T0
    adcx    \R2, \T1

    mulx    \T1, \T0, [\pB + 1*8]
    adox    \R2, \T0
    adcx    \R3, \T1

    mulx    \T1, \T0, [\pB + 2*8]
    adox    \R3, \T0
    adcx    \R4, \T1

    mulx    \T1, \T0, [\pB + 3*8]
    adox    \R4, \T0
    mov     \T0, 0
    adcx    \T1, \T0      // \T1 in range [0, 2^64-1]

    adox    \R5, \T0      // \R5 in range [0, 2]
                        // defer adding \T1 to avoid overflowing earlier than necessary (we are bound by adox/adcx throughput)
                        // Multiplication result in \R1, \R2, \R3, \R4, \R5+\T1

    xor     \T0, \T0      // clear flags

    mov     \QH, \K       // now use \K as a temporary while \T1 has the value to add to \R5

    mulx    \K, \T0, [\pM + 0*8]
    adcx    \R0, \T0      // sets \R0 to 0
    adox    \R1, \K

    mulx    \K, \T0, [\pM + 1*8]
    adcx    \R1, \T0
    adox    \R2, \K

    mulx    \K, \T0, [\pM + 2*8]
    adcx    \R2, \T0
    adox    \R3, \K

    mulx    \K, \T0, [\pM + 3*8]
    adcx    \R3, \T0
    adox    \R4, \K       // This may overflow

    adcx    \R4, \R0      // This may overflow
    adox    \R5, \R0      // \R5 in range [0, 3]

    adc     \R5, \T1      // This may overflow

    adc     \R0, \R0

    mov     \K, \R1
    imul    \K, [\pM - SymCryptModulusValueOffsetAmd64 + SymCryptModulusInv64OffsetAmd64]
.endm

//VOID
//SYMCRYPT_CALL
//SymCryptFdefModMulMontgomeryMulx256Asm(
//    _In_                            PCSYMCRYPT_MODULUS      pMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc1,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc2,
//    _Out_                           PSYMCRYPT_MODELEMENT    pDst,
//    _Out_writes_bytes_( cbScratch ) PBYTE                   pbScratch,
//                                    SIZE_T                  cbScratch )

#if defined(SYMCRYPT_MASM)
altentry SymCryptFdefModMulMontgomeryMulx256AsmInternal
#endif

// Note we specify only 4 arguments as we never use arguments 5 and 6 (saves some prolog code in MSFT calling convention)
#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulx256Asm):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulx256Asm)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulx256Asm); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulx256Asm)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12

.seh_endprologue

mov Q2, QH

        // Q1 = pMod
        // Q2 = pSrc1
        // Q3 = pSrc2
        // Q4 = pDst

SymCryptFdefModMulMontgomeryMulx256AsmInternal: .global SymCryptFdefModMulMontgomeryMulx256AsmInternal

        //
        // SamL: We previously did small modular multiplication by doing a full integer multiplication followed by a
        // Montgomery reduction on the full result.
        // I observed that this approach could have an issue with dependency chains in the Montgomery reduction step,
        // if the multiple K we need to add to the result to zero out the bottom 64b is not computed early enough we
        // can be bound by the latency of multiplication rather than the throughput of multiplication and carry chain
        // handling.
        // Additionally, I wanted the new 384-bit specific code to follow a similar pattern to the new 256-bit code, and
        // producing a full 384bx384b->768b multiplication would mean we run out of registers and have to spill in the
        // body of the 384-bit function.
        //
        // The approach I have taken is to interleave the multiplication with the Montgomery reduction, so we only have
        // to keep n+2 words in registers for accumulators in the inner processing. Additionally, for all but the
        // Montgomery reduction of the final least-significant word, we have a word*limb multiplication (4 or 6 mulx)
        // between issuing the computation of K and using the result.
        //

        xor     Q11, Q11  // clear flags
        mov     QH, [Q2]

        mulx    Q7, Q6,  [Q3 + 0*8]

        mulx    Q8, Q0,  [Q3 + 1*8]
        adc     Q7, Q0        // Note: this is intentionally adc as, if we issue an add here it is
                              // liable to contend with a multiplication uop on the critical path
                              // (adc issues on different ports to mulx/imul so does not contend)

        mulx    Q9, Q0,  [Q3 + 2*8]
        adc     Q8, Q0

        mulx    Q10, Q0, [Q3 + 3*8]
        adc     Q9, Q0

        adc     Q10, Q11      // Multiplication result in Q6, Q7, Q8, Q9, Q10

        // Compute K for reducing Multiplication result
        mov     Q12, Q6
        imul    Q12, [Q1 + SymCryptModulusInv64OffsetAmd64]
        add     Q1, SymCryptModulusValueOffsetAmd64

        MUL_AND_MONTGOMERY_REDUCE14_INTERLEAVE Q0, Q5, QH, Q2,  8, Q3, Q1, Q12, Q6, Q7, Q8, Q9, Q10, Q11

        MUL_AND_MONTGOMERY_REDUCE14_INTERLEAVE Q0, Q5, QH, Q2, 16, Q3, Q1, Q12, Q7, Q8, Q9, Q10, Q11, Q6

        MUL_AND_MONTGOMERY_REDUCE14_INTERLEAVE Q0, Q5, QH, Q2, 24, Q3, Q1, Q12, Q8, Q9, Q10, Q11, Q6, Q7

        xor     Q0, Q0      // clear flags
        mov     QH, Q12

        mulx    Q5, Q0, [Q1 + 0*8]
        adcx    Q9, Q0      // Set Cy when Q9 is non-zero
        adox    Q10, Q5

        mulx    Q5, Q0, [Q1 + 1*8]
        adcx    Q10, Q0
        adox    Q11, Q5

        mulx    Q5, Q0, [Q1 + 2*8]
        adcx    Q11, Q0
        adox    Q6, Q5

        mulx    Q5, Q0, [Q1 + 3*8]
        adcx    Q6, Q0
        adox    Q7, Q5      // This may overflow

        mov     Q0, 0
        adcx    Q7, Q0      // This may overflow
        adox    Q8, Q0

        adc     Q8, Q0

        // Montgomery-reduced multiplication value in (Q10, Q11, Q6, Q7, Q8), and it is less than 2*Modulus
        // Compute value + (-Mod) into (Q0, Q2, Q3, Q5), Cy = inverted subtraction borrow

        add     Q1, SymCryptNegDivisorSingleDigitOffsetAmd64 - SymCryptModulusValueOffsetAmd64

        mov     Q0, [Q1 + 0*8]
        add     Q0, Q10
        mov     Q2, [Q1 + 1*8]
        adc     Q2, Q11
        mov     Q3, [Q1 + 2*8]
        adc     Q3, Q6
        mov     Q5, [Q1 + 3*8]
        adc     Q5, Q7

        // Choose between the two
        // addition carry = 1, then subtraction borrow = 1 and we pick the 2nd result.
        // addition carry = 0 and subtraction borrow = 0: pick 2nd result
        // addition carry = 0 and subtraction borrow = 1: pick first result

        adc     Q8, Q8  // This is non-zero iff
                        //      the reduced value did not fit in 256-bits; or
                        //      the subtraction by Mod did not borrow
                        // adc sets the ZF appropriately

        cmovnz  Q10, Q0
        cmovnz  Q11, Q2
        cmovnz  Q6, Q3
        cmovnz  Q7, Q5

        mov     [Q4 + 0*8], Q10
        mov     [Q4 + 1*8], Q11
        mov     [Q4 + 2*8], Q6
        mov     [Q4 + 3*8], Q7


pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12

//VOID
//SYMCRYPT_CALL
//SymCryptFdefModSquareMontgomeryMulx256Asm(
//    _In_                            PCSYMCRYPT_MODULUS      pMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc,
//    _Out_                           PSYMCRYPT_MODELEMENT    pDst,
//    _Out_writes_bytes_( cbScratch ) PBYTE                   pbScratch,
//                                    SIZE_T                  cbScratch )

// Note we specify 4 arguments to keep prolog of this and SymCryptFdefModMulMontgomeryMulx256Asm the same
#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulx256Asm):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulx256Asm)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulx256Asm); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulx256Asm)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12

.seh_endprologue

mov Q2, QH

        // Q1 = pMod
        // Q2 = pSrc
        // Q3 = pDst

        mov     Q4, Q3
        mov     Q3, Q2

        // Normal code doesn't jump from the body of one function to the body of another function.
        // Here we have ensured that our stack frames are identical, so it is safe.
        // We just have to convince the other system components that this works...

        // Use conditional jump so that stack unwinder doesn't think it is an epilogue
        test    rsp,rsp
        jne     SymCryptFdefModMulMontgomeryMulx256AsmInternal       // jumps always

        int     3       // Dummy instruction because the debugger seems to have an off-by-one
                        // error and still see the (wrong) epilogue when on the JNE instruction
                        // Best guess: the debugger starts the stack trace *after* the current instruction


pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12

// --------------------------------
// 384-bit size specific functions
// --------------------------------

//VOID
//SYMCRYPT_CALL
//SymCryptFdefModAddMulx384(
//    _In_                            PCSYMCRYPT_MODULUS      pmMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   peSrc1,
//    _In_                            PCSYMCRYPT_MODELEMENT   peSrc2,
//    _Out_                           PSYMCRYPT_MODELEMENT    peDst,
//    _Out_writes_bytes_( cbScratch ) PBYTE                   pbScratch,
//                                    SIZE_T                  cbScratch )

#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q2 rdx
#define D2 edx
#define W2 dx
#define B2 dl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q5 r10
#define D5 r10d
#define W5 r10w
#define B5 r10b
#define Q6 r11
#define D6 r11d
#define W6 r11w
#define B6 r11b
#define Q7 rsi
#define D7 esi
#define W7 si
#define B7 sil
#define Q8 rdi
#define D8 edi
#define W8 di
#define B8 dil
#define Q9 rbp
#define D9 ebp
#define W9 bp
#define B9 bpl
#define Q10 rbx
#define D10 ebx
#define W10 bx
#define B10 bl
#define Q11 r12
#define D11 r12d
#define W11 r12w
#define B11 r12b
#define Q12 r13
#define D12 r13d
#define W12 r13w
#define B12 r13b
PLATFORM_NAME_MANGLE(SymCryptFdefModAddMulx384Asm):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModAddMulx384Asm)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModAddMulx384Asm); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModAddMulx384Asm)

.byte 0x40; push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12

.seh_endprologue


        // Q1 = pmMod
        // Q2 = peSrc1
        // Q3 = peSrc2
        // Q4 = peDst

        add     Q1, SymCryptNegDivisorSingleDigitOffsetAmd64

        // compute Src1 + Src2 into (Q0, Q5, Q6, Q7, Q8, Q2), Cy = addition carry
        // Compute sum + (-Mod) into (Q9, Q10, Q11, Q12, Q3, Q1), Ov = inverted subtraction borrow

        xor     Q0, Q0  // clear flags

        mov     Q0, [Q2 + 0*8]
        adcx    Q0, [Q3 + 0*8]

        mov     Q5, [Q2 + 1*8]
        adcx    Q5, [Q3 + 1*8]

        mov     Q6, [Q2 + 2*8]
        adcx    Q6, [Q3 + 2*8]

        mov     Q7, [Q2 + 3*8]
        adcx    Q7, [Q3 + 3*8]

        mov     Q8, [Q2 + 4*8]
        adcx    Q8, [Q3 + 4*8]

        mov     Q2, [Q2 + 5*8]
        adcx    Q2, [Q3 + 5*8]

        mov     Q9,  [Q1 + 0*8]
        adox    Q9,  Q0

        mov     Q10, [Q1 + 1*8]
        adox    Q10, Q5

        mov     Q11, [Q1 + 2*8]
        adox    Q11, Q6

        mov     Q12, [Q1 + 3*8]
        adox    Q12, Q7

        mov     Q3,  [Q1 + 4*8]
        adox    Q3,  Q8

        mov     Q1,  [Q1 + 5*8]
        adox    Q1,  Q2

        // Choose between the two
        // addition carry = 1, then subtraction borrow = 1 and we pick the 2nd result.
        // addition carry = 0 and subtraction borrow = 0: pick 2nd result
        // addition carry = 0 and subtraction borrow = 1: pick first result

        cmovc   Q0, Q9
        cmovc   Q5, Q10
        cmovc   Q6, Q11
        cmovc   Q7, Q12
        cmovc   Q8, Q3
        cmovc   Q2, Q1

        cmovo   Q0, Q9
        cmovo   Q5, Q10
        cmovo   Q6, Q11
        cmovo   Q7, Q12
        cmovo   Q8, Q3
        cmovo   Q2, Q1

        mov     [Q4 + 0*8], Q0
        mov     [Q4 + 1*8], Q5
        mov     [Q4 + 2*8], Q6
        mov     [Q4 + 3*8], Q7
        mov     [Q4 + 4*8], Q8
        mov     [Q4 + 5*8], Q2


pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
ret
.seh_endproc
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12

.macro MUL16_P384  T0, T1, QH, pA, Aoff, pB, R0, R1, R2, R3, R4, R5, R6, R7
    // (\R0, \R1, \R2, \R3, \R4, \R5, \R6, \R7) = A[\Aoff] * (B0..5) + (\R0, \R1, \R2, \R3, \R4, \R5, \R6)
    // \R6 in [0, 2] at start
    // \R7 in [0, 1] at end
    // \T0, \T1 = scratch

    mov     \QH, [\pA + \Aoff]
    xor     \R7, \R7      // clear flags
                        // makes following carry chains architecturally independent of previous ones

    mulx    \T1, \T0, [\pB + 0*8]
    adcx    \R0, \T0
    adox    \R1, \T1

    mulx    \T1, \T0, [\pB + 1*8]
    adcx    \R1, \T0
    adox    \R2, \T1

    mulx    \T1, \T0, [\pB + 2*8]
    adcx    \R2, \T0
    adox    \R3, \T1

    mulx    \T1, \T0, [\pB + 3*8]
    adcx    \R3, \T0
    adox    \R4, \T1

    mulx    \T1, \T0, [\pB + 4*8]
    adcx    \R4, \T0
    adox    \R5, \T1

    mulx    \T1, \T0, [\pB + 5*8]
    adcx    \R5, \T0
    adox    \R6, \R7      // \R6 in [0, 3]


    adc     \R6, \T1      // overflow possible
    adc     \R7, \R7
.endm

.macro MONT16_P384  T0, T1, QH, pM, N4, R0, R1, R2, R3, R4, R5, R6, R7
    // K = modInv * \R0
    // (\R3, \N4) = \R3 - (K-\N4)
    // (\R0, \R1, \R2, \R3, \N4) = K * (M0..2) + (\R0, \R1, \R2, \R3, \N4)
    // (\R6, \R7) = K + (\R6, \R7)
    //
    // Note K is chosen using pMod's montgomery inverse s.t. \R0 is set to 0
    // \pM is a pointer to the bytes of the modulus (i.e. offset into SYMCRYPT_MODULUS structure)
    // \N4 is 0 or -1 at start and end
    // \R7 in [0, 1] at start and in [0, 2] at end
    // \R0 is 0 at end
    // \T0, \T1 = scratch
    //
    // P384 has a modulus with a form which makes it possible to perform Montgomery reduction
    // in an interesting way. The modulus is:
    // 2^384 - 2^128 - 2^96 + 2^32 - 1
    // This means when we want to add a multiple of the Modulus in reduction, we can consider
    // not performing the full multiplication, but taking shortcuts.
    // It is hard to get much benefit from this, as doing the full multiplication of
    // 1 digit * 6 digits takes approximately 6 cycles with mulx, adox and adcx
    //
    // The approach taken here notes that K*M will always add 2^384 * K to the value to reduce
    // with a subtraction of a value < 2^256.
    // On entry to this macro we have \N4 representing a deferred borrow (value 0 or -1) from the
    // 4th word of the last macro (now aligned to the 3rd word of this macro), on exit we set \N4
    // to be the borrow to defer to the next round.
    //
    //            ____ ____ ____ ____ ____ ____
    //           | M5 | M4 | M3 | M2 | M1 | M0 |
    //           | ff | ff | ff | fe | f0 | 0f | (P384 modulus if you sign extend each nibble to 32b)
    //           |____|____|____|____|____|____|
    //  ____ ____ ____ ____ ____ ____ ____ ____
    // |    |    |\  /|\  /|    |    |    |    |
    // | \R7 | \R6 | \R5 | \R4 | \R3 | \R2 | \R1 | \R0 |
    // |____|____|/__\|/__\|____|____|____|____|
    //                                      | x modInv (0x0000000100000001)
    //                      ____           _v__
    //                     |    |         |    |
    //                   + |N4in|         |  K |
    //                     |____|         |____|
    //       ____           ____
    //      |    |         |    |
    //    + |  K |       - |  K | This row is equivalent to multiplying K by (M3..5)
    //      |____|     ____|____| once the borrow is propagated
    //                |    |
    //                | \N4 |
    //                |____|     ____ ____ ____
    //                          |M2*K|M1*K|M0*K|
    //                        + | lo | lo | lo |
    //                      ____|____|____|____|
    //                     |M2*K|M1*K|M0*K|
    //                   + | hi | hi | hi |
    //                     |____|____|____|
    //
    // It is tempting to avoid the multiplication of K by M2, and subtract 2*K from \R2 instead, but the
    // deferred subtraction (N3) then can be in the range [0,-2], and it is no longer possible to do the
    // subtraction using the borrow flag. It ends up being more expensive than just doing another multiplication
    // in my testing.
    // It is also tempting to compute K using a shift and add, rather than imul, but experimentally, this
    // performs worse.
    //

    mov     \QH, \R0
    imul    \QH, [\pM - SymCryptModulusValueOffsetAmd64 + SymCryptModulusInv64OffsetAmd64]

    add     \N4, -1
    sbb     \R3, \QH
    sbb     \N4, \N4      // sets \N4 to 0 if no borrow, or -1 if there is a borrow

    xor     \T0, \T0      // clear flags

    mulx    \T1, \T0, [\pM + 0*8]
    adcx    \R0, \T0      // sets \R0 to 0
    adox    \R1, \T1

    mulx    \T1, \T0, [\pM + 1*8]
    adcx    \R1, \T0
    adox    \R2, \T1

    mulx    \T1, \T0, [\pM + 2*8]
    adcx    \R2, \T0
    adox    \R3, \T1

    adcx    \R3, \R0
    adox    \N4, \R0

    adc     \N4, \R0      // A carry into \N4 can occur iff the previous subtraction borrowed
                        // The value of \N4 is guaranteed to be 0 or -1 here

    add     \R6, \QH
    adc     \R7, \R0

.endm

#if defined(SYMCRYPT_MASM)
altentry SymCryptFdefModMulMontgomeryMulxP384AsmInternal
#endif

//VOID
//SYMCRYPT_CALL
//SymCryptFdefModMulMontgomeryMulxP384Asm(
//    _In_                            PCSYMCRYPT_MODULUS      pMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc1,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc2,
//    _Out_                           PSYMCRYPT_MODELEMENT    pDst,
//    _Out_writes_bytes_( cbScratch ) PBYTE                   pbScratch,
//                                    SIZE_T                  cbScratch )

// Note we specify only 4 arguments as we never use arguments 5 and 6 (saves some prolog code in MSFT calling convention)
#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
#define Q13 r15
#define D13 r15d
#define W13 r15w
#define B13 r15b
PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulxP384Asm):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulxP384Asm)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulxP384Asm); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulxP384Asm)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12
push Q13
.seh_pushreg Q13

.seh_endprologue

mov Q2, QH

        // Q1 = pMod
        // Q2 = pSrc1
        // Q3 = pSrc2
        // Q4 = pDst

SymCryptFdefModMulMontgomeryMulxP384AsmInternal: .global SymCryptFdefModMulMontgomeryMulxP384AsmInternal

        mov     [rsp + 72 /*MEMSLOT0*/], Q4   // save pDst

        // SamL: In optimizing for P384 specifically, we need to keep an extra register
        // free for deferring a borrow (N4 in the MONT16_P384 macro). This means that it
        // is not possible to do the INTERLEAVE approach taken in the other small Mulx
        // multiplication variants (and described in SymCryptFdefModMulMontgomeryMulx256Asm),
        // as there is too much register pressure.
        // Although we reintroduce possible dependency chain issues with the generation and
        // use of K for the Montgomery reduction, we generate fewer total micro-ops and the
        // overall performance is a few % better at the time of writing.

        xor     Q13, Q13  // clear flags
        mov     QH, [Q2]

        mulx    Q7, Q6,  [Q3 + 0*8]

        mulx    Q8, Q0,  [Q3 + 1*8]
        adc     Q7, Q0        // Note: this is intentionally adc as, if we issue an add here it is
                              // liable to contend with a multiplication uop on the critical path
                              // (adc issues on different ports to mulx/imul so does not contend)

        mulx    Q9, Q0,  [Q3 + 2*8]
        adc     Q8, Q0

        mulx    Q10, Q0, [Q3 + 3*8]
        adc     Q9, Q0

        mulx    Q11, Q0, [Q3 + 4*8]
        adc     Q10, Q0

        mulx    Q12, Q0, [Q3 + 5*8]
        adc     Q11, Q0

        adc     Q12, Q13        // Multiplication result in Q6, Q7, Q8, Q9, Q10, Q11, Q12
        add     Q1, SymCryptModulusValueOffsetAmd64
        xor     Q4, Q4

        MONT16_P384     Q0, Q5, QH, Q1, Q4,     Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13

        MUL16_P384      Q0, Q5, QH, Q2,  8, Q3, Q7, Q8, Q9, Q10, Q11, Q12, Q13, Q6
        MONT16_P384     Q0, Q5, QH, Q1, Q4,     Q7, Q8, Q9, Q10, Q11, Q12, Q13, Q6

        MUL16_P384      Q0, Q5, QH, Q2, 16, Q3, Q8, Q9, Q10, Q11, Q12, Q13, Q6, Q7
        MONT16_P384     Q0, Q5, QH, Q1, Q4,     Q8, Q9, Q10, Q11, Q12, Q13, Q6, Q7

        MUL16_P384      Q0, Q5, QH, Q2, 24, Q3, Q9, Q10, Q11, Q12, Q13, Q6, Q7, Q8
        MONT16_P384     Q0, Q5, QH, Q1, Q4,     Q9, Q10, Q11, Q12, Q13, Q6, Q7, Q8

        MUL16_P384      Q0, Q5, QH, Q2, 32, Q3, Q10, Q11, Q12, Q13, Q6, Q7, Q8, Q9
        MONT16_P384     Q0, Q5, QH, Q1, Q4,     Q10, Q11, Q12, Q13, Q6, Q7, Q8, Q9

        MUL16_P384      Q0, Q5, QH, Q2, 40, Q3, Q11, Q12, Q13, Q6, Q7, Q8, Q9, Q10
        MONT16_P384     Q0, Q5, QH, Q1, Q4,     Q11, Q12, Q13, Q6, Q7, Q8, Q9, Q10

        xor     Q0, Q0

        adox    Q7, Q4  // Q4 is 0 or -1
                        // Adding it to Q7..Q10 sign extends it correctly; a carry represents an inverted borrow
        adox    Q8, Q4
        adox    Q9, Q4
        adox    Q10, Q4

        // Montgomery-reduced multiplication value in (Q12, Q13, Q6, Q7, Q8, Q9, Q10), and it is less than 2*Modulus
        // Compute value + (-Mod) into (Q2, Q3, Q5, Q0, Q1, Q11), Cy = inverted subtraction borrow
        // (-Mod) is (0xffffffff00000001, 0x00000000ffffffff, 1, 0, 0, 0)

        mov     Q2,  [Q1 + SymCryptNegDivisorSingleDigitOffsetAmd64 - SymCryptModulusValueOffsetAmd64 + 0*8]
        add     Q2,  Q12
        mov     Q3,  [Q1 + 0*8] // Modulus word 0 is equal to Negated Modulus word 1
        adc     Q3,  Q13
        mov     Q5,  1  // Don't need to load 1 and 0 values from memory - we statically know these values for P384
        adc     Q5,  Q6
        mov     Q1,  Q0 // Q0 and Q11 are already zero, set Q1 to zero too
        adc     Q0,  Q7
        adc     Q1,  Q8
        adc     Q11, Q9

        // Choose between the two
        // addition carry = 1, then subtraction borrow = 1 and we pick the 2nd result.
        // addition carry = 0 and subtraction borrow = 0: pick 2nd result
        // addition carry = 0 and subtraction borrow = 1: pick first result

        adc     Q10, Q10        // This is non-zero iff
                                //      the reduced value did not fit in 384-bits; or
                                //      the subtraction by Mod did not borrow
                                // adc sets the ZF appropriately

        cmovnz  Q12, Q2
        cmovnz  Q13, Q3
        cmovnz  Q6,  Q5
        cmovnz  Q7,  Q0
        cmovnz  Q8,  Q1
        cmovnz  Q9,  Q11

        mov     Q4, [rsp + 72 /*MEMSLOT0*/]   // restore pDst

        mov     [Q4 + 0*8], Q12
        mov     [Q4 + 1*8], Q13
        mov     [Q4 + 2*8], Q6
        mov     [Q4 + 3*8], Q7
        mov     [Q4 + 4*8], Q8
        mov     [Q4 + 5*8], Q9


pop Q13
pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12
#undef Q13
#undef D13
#undef W13
#undef B13

//VOID
//SYMCRYPT_CALL
//SymCryptFdefModSquareMontgomeryMulxP384Asm(
//    _In_                            PCSYMCRYPT_MODULUS      pMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc,
//    _Out_                           PSYMCRYPT_MODELEMENT    pDst,
//    _Out_writes_bytes_( cbScratch ) PBYTE                   pbScratch,
//                                    SIZE_T                  cbScratch )

// Note we specify 4 arguments to keep prolog of this and SymCryptFdefModMulMontgomeryMulx384Asm the same
#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
#define Q13 r15
#define D13 r15d
#define W13 r15w
#define B13 r15b
PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulxP384Asm):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulxP384Asm)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulxP384Asm); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulxP384Asm)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12
push Q13
.seh_pushreg Q13

.seh_endprologue

mov Q2, QH

        // Q1 = pMod
        // Q2 = pSrc
        // Q3 = pDst

        mov     Q4, Q3
        mov     Q3, Q2

        // Normal code doesn't jump from the body of one function to the body of another function.
        // Here we have ensured that our stack frames are identical, so it is safe.
        // We just have to convince the other system components that this works...

        // Use conditional jump so that stack unwinder doesn't think it is an epilogue
        test    rsp,rsp
        jne     SymCryptFdefModMulMontgomeryMulxP384AsmInternal       // jumps always

        int     3       // Dummy instruction because the debugger seems to have an off-by-one
                        // error and still see the (wrong) epilogue when on the JNE instruction
                        // Best guess: the debugger starts the stack trace *after* the current instruction


pop Q13
pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12
#undef Q13
#undef D13
#undef W13
#undef B13


// --------------------------------
// 1024-bit size specific functions
// --------------------------------

//VOID
//SYMCRYPT_CALL
//SymCryptFdefRawMul(
//    _In_reads_(nWords1)             PCUINT32    pSrc1,
//    _In_reads_(nWords2)             PCUINT32    pSrc2,
//                                    UINT32      nDigits,
//    _Out_writes_(nWords1 + nWords2) PUINT32     pDst )

#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
PLATFORM_NAME_MANGLE(SymCryptFdefRawMulMulx1024):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefRawMulMulx1024)
.def PLATFORM_NAME_MANGLE(SymCryptFdefRawMulMulx1024); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefRawMulMulx1024)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12

.seh_endprologue

mov Q2, QH

        // First we wipe nDigits of the result (size of in)
        // Q1 = pSrc1
        // Q2 = pSrc2
        // Q3 = nDigits
        // Q4 = pDst

        // Wipe destination for nDigit2 blocks
        xorps       xmm0,xmm0               // Zero register for 16-byte wipes

        movaps      [Q4],xmm0
        movaps      [Q4+16],xmm0            // Wipe 32 bytes
        movaps      [Q4+32],xmm0            // Wipe 32 bytes
        movaps      [Q4+48],xmm0            // Wipe 32 bytes

        movaps      [Q4+64],xmm0
        movaps      [Q4+80],xmm0            // Wipe 32 bytes
        movaps      [Q4+96],xmm0            // Wipe 32 bytes
        movaps      [Q4+112],xmm0           // Wipe 32 bytes

        // Digit 1 from src2

        ZEROREG_8   Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12      // Leaves Cy = Ov = 0

        MULADD88  Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q4, Q1, Q2, Q0, Q3, QH

        add     Q2, 64              // Src2 ptr
        add     Q4, 64
        xor     Q0, Q0              // sets Cy = Ov = 0

        MULADD88  Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q4, Q1, Q2, Q0, Q3, QH

        add     Q4, 64

        // Write the 8-word carry-out to the destination
        mov     [Q4 + 0*8], Q5
        mov     [Q4 + 1*8], Q6
        mov     [Q4 + 2*8], Q7
        mov     [Q4 + 3*8], Q8
        mov     [Q4 + 4*8], Q9
        mov     [Q4 + 5*8], Q10
        mov     [Q4 + 6*8], Q11
        mov     [Q4 + 7*8], Q12

        // Digit 2 from src2

        // set up

        // Mov Q4 one digit back
        sub     Q4, 64

        // reload pSrc2
        sub     Q2, 64

        // update PSrc1
        add     Q1, 64

        ZEROREG_8   Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12      // Leaves Cy = Ov = 0

        MULADD88  Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q4, Q1, Q2, Q0, Q3, QH

        add     Q2, 64              // Src2 ptr
        add     Q4, 64
        xor     Q0, Q0              // sets Cy = Ov = 0

        MULADD88  Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q4, Q1, Q2, Q0, Q3, QH

        add     Q4, 64

        // Write the 8-word carry-out to the destination
        mov     [Q4 + 0*8], Q5
        mov     [Q4 + 1*8], Q6
        mov     [Q4 + 2*8], Q7
        mov     [Q4 + 3*8], Q8
        mov     [Q4 + 4*8], Q9
        mov     [Q4 + 5*8], Q10
        mov     [Q4 + 6*8], Q11
        mov     [Q4 + 7*8], Q12


pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12

// VOID
// SYMCRYPT_CALL
// SymCryptFdefRawSquareMulx1024(
//     _In_reads_(nDigits*SYMCRYPT_FDEF_DIGIT_NUINT32)         PCUINT32    pSrc,
//                                                             UINT32      nDigits,
//     _Out_writes_(2*nDigits*SYMCRYPT_FDEF_DIGIT_NUINT32)     PUINT32     pDst )

#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
PLATFORM_NAME_MANGLE(SymCryptFdefRawSquareMulx1024):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefRawSquareMulx1024)
.def PLATFORM_NAME_MANGLE(SymCryptFdefRawSquareMulx1024); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefRawSquareMulx1024)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12

.seh_endprologue

mov Q2, QH

        // Wipe 128 bytes of destination
        xorps       xmm0,xmm0               // Zero register for 16-byte wipes

        movaps      [Q3],xmm0
        movaps      [Q3+16],xmm0
        movaps      [Q3+32],xmm0
        movaps      [Q3+48],xmm0

        movaps      [Q3+64],xmm0
        movaps      [Q3+80],xmm0
        movaps      [Q3+96],xmm0
        movaps      [Q3+112],xmm0

        xor     Q0, Q0                      // Sets Cy = Ov = 0

        HALF_SQUARE_NODIAG8 Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12,  Q3, Q1, Q0, Q2, QH

        lea     Q4, [Q1 + 64]               // Q4 = pSrc + 64
        lea     Q3, [Q3 + 64]               // Q3 = pDst + 64

        // Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12        8-word carry
        // Q0, Q2                                   temps for multiplication
        // Q1                                       pSrc (constant)
        // Q4                                       pSrc + 64 (constant)
        // Q3                                       pDst running pointer
        // QH                                       fixed input reg for multiplication

        MULADD88    Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q3, Q1, Q4, Q0, Q2, QH

        add     Q3, 64                      // Q3 = pDst + 128

        // Write the 8-word carry-out to the destination
        mov     [Q3 + 0*8], Q5
        mov     [Q3 + 1*8], Q6
        mov     [Q3 + 2*8], Q7
        mov     [Q3 + 3*8], Q8
        mov     [Q3 + 4*8], Q9
        mov     [Q3 + 5*8], Q10
        mov     [Q3 + 6*8], Q11
        mov     [Q3 + 7*8], Q12

        // Q3 which is the destination pointer is shifted here by 2 digits

        xor     Q0, Q0                        // Sets Cy = Ov = 0

        HALF_SQUARE_NODIAG8 Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12,  Q3, Q4, Q0, Q2, QH

        // Write the 8-word carry-out to the destination
        mov     [Q3 +  8*8], Q5
        mov     [Q3 +  9*8], Q6
        mov     [Q3 + 10*8], Q7
        mov     [Q3 + 11*8], Q8
        mov     [Q3 + 12*8], Q9
        mov     [Q3 + 13*8], Q10
        mov     [Q3 + 14*8], Q11
        mov     [Q3 + 15*8], Q12

        // Compute diagonals, and add double the result so far

        sub     Q3, 128         // Q3 = pDst - sets Cy = Ov = 0

        SYMCRYPT_SQUARE_DIAG 0, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 1, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 2, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 3, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 4, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 5, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 6, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 7, Q1, Q3, Q0, Q2, Q4, Q5, QH

        SYMCRYPT_SQUARE_DIAG 8, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 9, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 10, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 11, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 12, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 13, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 14, Q1, Q3, Q0, Q2, Q4, Q5, QH
        SYMCRYPT_SQUARE_DIAG 15, Q1, Q3, Q0, Q2, Q4, Q5, QH


pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12

// VOID
// SYMCRYPT_CALL
// SymCryptFdefMontgomeryReduceMulx1024(
//     _In_                            PCSYMCRYPT_MODULUS      pmMod,
//     _Inout_                         PUINT32                 pSrc,
//     _Out_                           PUINT32                 pDst )
#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
#define Q13 r15
#define D13 r15d
#define W13 r15w
#define B13 r15b
PLATFORM_NAME_MANGLE(SymCryptFdefMontgomeryReduceMulx1024):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefMontgomeryReduceMulx1024)
.def PLATFORM_NAME_MANGLE(SymCryptFdefMontgomeryReduceMulx1024); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefMontgomeryReduceMulx1024)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12
push Q13
.seh_pushreg Q13

.seh_endprologue

mov Q2, QH

        mov     [rsp + 72 /*MEMSLOT0*/], Q3

        mov     D0, 2
        mov     [rsp + 80 /*MEMSLOT1*/], D0
        // CntOuter = nDigits - using first half of slot3

        xor     D4, D4
        lea     Q1, [Q1 + SymCryptModulusValueOffsetAmd64]                      // modulus value

SymCryptFdefMontgomeryReduceMulx1024OuterLoop:
        // Q1 = pmMod
        // Q2 = pSrc = tmp buffer that we will reduce
        mov     Q6, [Q2 + 0 * 8]
        mov     Q7, [Q2 + 1 * 8]
        mov     Q8, [Q2 + 2 * 8]
        mov     Q9, [Q2 + 3 * 8]
        mov     Q10, [Q2 + 4 * 8]
        mov     Q11, [Q2 + 5 * 8]
        mov     Q12, [Q2 + 6 * 8]
        mov     Q13, [Q2 + 7 * 8]

        mov     Q3, [Q1 - SymCryptModulusValueOffsetAmd64 + SymCryptModulusInv64OffsetAmd64]    // inv64

        // Q2 = value to reduce
        // Q6 - Q13 = Q2[0..7]
        // Q1 = modulus value
        // Q3 = modinv

        MONTGOMERY18    Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13,  Q3, Q1, Q2 + (0 * 8), Q0, Q5, QH
        MONTGOMERY18    Q7, Q8, Q9, Q10, Q11, Q12, Q13, Q6,  Q3, Q1, Q2 + (1 * 8), Q0, Q5, QH
        MONTGOMERY18    Q8, Q9, Q10, Q11, Q12, Q13, Q6, Q7,  Q3, Q1, Q2 + (2 * 8), Q0, Q5, QH
        MONTGOMERY18    Q9, Q10, Q11, Q12, Q13, Q6, Q7, Q8,  Q3, Q1, Q2 + (3 * 8), Q0, Q5, QH
        MONTGOMERY18    Q10, Q11, Q12, Q13, Q6, Q7, Q8, Q9,  Q3, Q1, Q2 + (4 * 8), Q0, Q5, QH
        MONTGOMERY18    Q11, Q12, Q13, Q6, Q7, Q8, Q9, Q10,  Q3, Q1, Q2 + (5 * 8), Q0, Q5, QH
        MONTGOMERY18    Q12, Q13, Q6, Q7, Q8, Q9, Q10, Q11,  Q3, Q1, Q2 + (6 * 8), Q0, Q5, QH
        MONTGOMERY18    Q13, Q6, Q7, Q8, Q9, Q10, Q11, Q12,  Q3, Q1, Q2 + (7 * 8), Q0, Q5, QH

        // Q6 - Q13 = carry from multiply-add
        // Q2[0..7] = Montgomery factors

        mov     Q3, Q2         // factor to multiply by
        add     Q1, 64
        add     Q2, 64

        // Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13        8-word carry
        // Q0, Q5                                    temps for multiplication
        // Q1                                        running pointer pMod inner loop
        // Q2                                        running pointer pSrc inner loop
        // Q3                                        Montgomery factors for this row
        // D4                                        loop ctr
        // QH                                        fixed input reg for multiplication

        MULADD88    Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13,  Q2, Q1, Q3, Q0, Q5, QH
            // pre & post: Cy = Ov = 0
            // Q13..Q6:Q2[7-0] = R[7-0]:D[7-0] = A[7:0] * B[7:0] + R[7:0] + D[7:0]
            // QH is volatile

        add     Q1, 64
        add     Q2, 64

        // We have an 8-word carry here, which we need to add to the in-memory buffer and retain a carry
        // We also saved a 1-bit carry from the previous outer loop in D4
        // move carry into Cy flag
        neg     D4
        mov     D4, 0

        // We do this in separate instructions to help the instruction decoder build up a lead...
        mov     Q0, [Q2 + 0 * 8]
        adc     Q0, Q6
        mov     [Q2 + 0 * 8], Q0

        mov     Q5, [Q2 + 1 * 8]
        adc     Q5, Q7
        mov     [Q2 + 1 * 8], Q5

        mov     Q0, [Q2 + 2 * 8]
        adc     Q0, Q8
        mov     [Q2 + 2 * 8], Q0

        mov     Q5, [Q2 + 3 * 8]
        adc     Q5, Q9
        mov     [Q2 + 3 * 8], Q5

        mov     Q0, [Q2 + 4 * 8]
        adc     Q0, Q10
        mov     [Q2 + 4 * 8], Q0

        mov     Q5, [Q2 + 5 * 8]
        adc     Q5, Q11
        mov     [Q2 + 5 * 8], Q5

        mov     Q0, [Q2 + 6 * 8]
        adc     Q0, Q12
        mov     [Q2 + 6 * 8], Q0

        mov     Q5, [Q2 + 7 * 8]
        adc     Q5, Q13
        mov     [Q2 + 7 * 8], Q5

        adc     D4, D4                  // D4 = carry (D4 was previously zero)

        sub     Q2, 64                  // Q2 = tmp buffer that we will reduce (64B are now zeroed)
        sub     Q1, 128                 // Q1 = modulus value

        mov     D0, [rsp + 80 /*MEMSLOT1*/]
        sub     D0, 1
        mov     [rsp + 80 /*MEMSLOT1*/], D0

        jnz     SymCryptFdefMontgomeryReduceMulx1024OuterLoop

        // D4 = output carry

        mov     Q3, [rsp + 72 /*MEMSLOT0*/]

        // Q2 = result buffer pointer
        // Q1 = modulus value
        // Q3 = Dst

        // pDst = Reduction result - Modulus

        mov     Q0,[Q2 + 0 * 8]
        sbb     Q0,[Q1 + 0 * 8]
        mov     [Q3 + 0 * 8], Q0

        mov     Q5,[Q2 + 1 * 8]
        sbb     Q5,[Q1 + 1 * 8]
        mov     [Q3 + 1 * 8], Q5

        mov     Q0,[Q2 + 2 * 8]
        sbb     Q0,[Q1 + 2 * 8]
        mov     [Q3 + 2 * 8], Q0

        mov     Q5,[Q2 + 3 * 8]
        sbb     Q5,[Q1 + 3 * 8]
        mov     [Q3 + 3 * 8], Q5

        mov     Q0,[Q2 + 4 * 8]
        sbb     Q0,[Q1 + 4 * 8]
        mov     [Q3 + 4 * 8], Q0

        mov     Q5,[Q2 + 5 * 8]
        sbb     Q5,[Q1 + 5 * 8]
        mov     [Q3 + 5 * 8], Q5

        mov     Q0,[Q2 + 6 * 8]
        sbb     Q0,[Q1 + 6 * 8]
        mov     [Q3 + 6 * 8], Q0

        mov     Q5,[Q2 + 7 * 8]
        sbb     Q5,[Q1 + 7 * 8]
        mov     [Q3 + 7 * 8], Q5

        mov     Q0,[Q2 + 8 * 8]
        sbb     Q0,[Q1 + 8 * 8]
        mov     [Q3 + 8 * 8], Q0

        mov     Q5,[Q2 + 9 * 8]
        sbb     Q5,[Q1 + 9 * 8]
        mov     [Q3 + 9 * 8], Q5

        mov     Q0,[Q2 + 10 * 8]
        sbb     Q0,[Q1 + 10 * 8]
        mov     [Q3 + 10 * 8], Q0

        mov     Q5,[Q2 + 11 * 8]
        sbb     Q5,[Q1 + 11 * 8]
        mov     [Q3 + 11 * 8], Q5

        mov     Q0,[Q2 + 12 * 8]
        sbb     Q0,[Q1 + 12 * 8]
        mov     [Q3 + 12 * 8], Q0

        mov     Q5,[Q2 + 13 * 8]
        sbb     Q5,[Q1 + 13 * 8]
        mov     [Q3 + 13 * 8], Q5

        mov     Q0,[Q2 + 14 * 8]
        sbb     Q0,[Q1 + 14 * 8]
        mov     [Q3 + 14 * 8], Q0

        mov     Q5,[Q2 + 15 * 8]
        sbb     Q5,[Q1 + 15 * 8]
        mov     [Q3 + 15 * 8], Q5

        // now a masked copy from the reduction buffer to the destination.
        // copy if high carry = 0 and Cy = 1
        sbb     D4, 0
        // D4 = copy mask, ff...ff  if copy, 0 of no copy

        movd    xmm0, D4            // xmm0[0] = mask
        pcmpeqd xmm1, xmm1          // xmm1 = ff...ff
        pshufd  xmm0, xmm0, 0       // xmm0[0..3] = mask
        pxor    xmm1, xmm0          // xmm1 = not Mask


        movdqa  xmm2, [Q2 + 0 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q3 + 0 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q3 + 0 * 16], xmm2

        movdqa  xmm2, [Q2 + 1 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q3 + 1 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q3 + 1 * 16], xmm2

        movdqa  xmm2, [Q2 + 2 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q3 + 2 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q3 + 2 * 16], xmm2

        movdqa  xmm2, [Q2 + 3 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q3 + 3 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q3 + 3 * 16], xmm2

        movdqa  xmm2, [Q2 + 4 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q3 + 4 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q3 + 4 * 16], xmm2

        movdqa  xmm2, [Q2 + 5 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q3 + 5 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q3 + 5 * 16], xmm2

        movdqa  xmm2, [Q2 + 6 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q3 + 6 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q3 + 6 * 16], xmm2

        movdqa  xmm2, [Q2 + 7 * 16]    // xmm2 = pSrc[0]
        movdqa  xmm3, [Q3 + 7 * 16]    // xmm3 = pDst[0]
        pand    xmm2, xmm0
        pand    xmm3, xmm1
        por     xmm2, xmm3
        movdqa  [Q3 + 7 * 16], xmm2


pop Q13
pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12
#undef Q13
#undef D13
#undef W13
#undef B13

#if 0
//////////////////////////////////////////////////////////////
//
// UNUSED ASSEMBLY FROM HERE
// Kept in case we need it later, but removed at compile time
// to avoid inclusion in produced binaries (the linker struggles
// to remove unused assembly functions)
//
//////////////////////////////////////////////////////////////

#if defined(SYMCRYPT_MASM)
altentry SymCryptFdefModMulMontgomeryMulxP256AsmInternal
#endif

.macro MUL_AND_MONTGOMERY_REDUCE14_INTERLEAVE_P256  T0, T1, QH, pA, Aoff, pB, pM, R0, R1, R2, R3, R4, R5
    // (\R1, \R2, \R3, \R4, \R5) = A[\Aoff] * (B0..3) + (\R1, \R2, \R3, \R4, \R5)
    // (xx, \R1, \R2, \R3, \R4, \R5, \R0) = \R0 * (M0..3) + (\R0, \R1, \R2, \R3, \R4, \R5)
    // \R1 = K = modInv * \R1, as modInv == 1
    // Note K is chosen using pMod's montgomery inverse s.t. xx would get set to 0
    // \pM is a pointer to the bytes of the modulus (i.e. offset into SYMCRYPT_MODULUS structure)
    // \R5 is 0 or 1 at start
    // \R0 is 0 or 1 at end
    // \T0, \T1 = scratch

    mov     \QH, [\pA + \Aoff]
    xor     \T0, \T0      // clear flags
                        // makes following carry chains architecturally independent of previous ones

    mulx    \T1, \T0, [\pB + 0*8]
    adox    \R1, \T0
    adcx    \R2, \T1

    mulx    \T1, \T0, [\pB + 1*8]
    adox    \R2, \T0
    adcx    \R3, \T1

    mulx    \T1, \T0, [\pB + 2*8]
    adox    \R3, \T0
    adcx    \R4, \T1

    mulx    \T1, \T0, [\pB + 3*8]
    adox    \R4, \T0
    mov     \T0, 0
    adcx    \T1, \T0      // \T1 in range [0, 2^64-1]

    adox    \R5, \T0      // \R5 in range [0, 2]
                        // defer adding \T1 to avoid overflowing earlier than necessary (we are bound by adox/adcx throughput)
                        // Multiplication result in \R1, \R2, \R3, \R4, \R5+\T1

    xor     \T0, \T0      // clear flags

    mov     \QH, \R0      // now use \R0 as a temporary while \T1 has the value to add to \R5

    adox    \R1, \R0

    mulx    \R0, \T0, [\pM + 1*8]
    adcx    \R1, \T0
    adox    \R2, \R0

    mov     \T0, 0       // For P256, 3rd digit of the modulus is 0
    adcx    \R2, \T0
    adox    \R3, \T0

    mulx    \R0, \T0, [\pM + 3*8]
    adcx    \R3, \T0
    adox    \R4, \R0      // This may overflow

    mov     \R0, 0
    adcx    \R4, \R0      // This may overflow
    adox    \R5, \R0      // \R5 in range [0, 3]

    adcx    \R5, \T1      // This may overflow

    adcx    \R0, \R0
.endm

//VOID
//SYMCRYPT_CALL
//SymCryptFdefModMulMontgomeryMulxP256Asm(
//    _In_                            PCSYMCRYPT_MODULUS      pMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc,
//    _Out_                           PSYMCRYPT_MODELEMENT    pDst,
//    _Out_writes_bytes_( cbScratch ) PBYTE                   pbScratch,
//                                    SIZE_T                  cbScratch )

// Note we specify only 4 arguments as we never use arguments 5 and 6 (saves some prolog code in MSFT calling convention)
#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
#define Q13 r15
#define D13 r15d
#define W13 r15w
#define B13 r15b
PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulxP256Asm):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulxP256Asm)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulxP256Asm); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulxP256Asm)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12
push Q13
.seh_pushreg Q13

.seh_endprologue

mov Q2, QH

        // Q1 = pMod
        // Q2 = pSrc1
        // Q3 = pSrc2
        // Q4 = pDst

SymCryptFdefModMulMontgomeryMulxP256AsmInternal: .global SymCryptFdefModMulMontgomeryMulxP256AsmInternal

        // See SymCryptFdefModMulMontgomeryMulx256Asm for the rationale behind this function!

        xor     Q11, Q11  // clear flags
        mov     QH, [Q2]

        mulx    Q7, Q6,  [Q3 + 0*8]

        mulx    Q8, Q0,  [Q3 + 1*8]
        adc     Q7, Q0

        mulx    Q9, Q0,  [Q3 + 2*8]
        adc     Q8, Q0

        mulx    Q10, Q0, [Q3 + 3*8]
        adc     Q9, Q0

        adc     Q10, Q11      // Multiplication result in Q6, Q7, Q8, Q9, Q10

        add     Q1, SymCryptModulusValueOffsetAmd64

        MUL_AND_MONTGOMERY_REDUCE14_INTERLEAVE_P256 Q0, Q5, QH, Q2,  8, Q3, Q1, Q6, Q7, Q8, Q9, Q10, Q11

        MUL_AND_MONTGOMERY_REDUCE14_INTERLEAVE_P256 Q0, Q5, QH, Q2, 16, Q3, Q1, Q7, Q8, Q9, Q10, Q11, Q6

        MUL_AND_MONTGOMERY_REDUCE14_INTERLEAVE_P256 Q0, Q5, QH, Q2, 24, Q3, Q1, Q8, Q9, Q10, Q11, Q6, Q7

        xor     Q0, Q0      // clear flags
        mov     QH, Q9

        adox    Q10, Q9

        mulx    Q5, Q0, [Q1 + 1*8]
        adcx    Q10, Q0
        adox    Q11, Q5

        mov     Q0, 0       // For P256, 3rd digit of the modulus is 0
        adcx    Q11, Q0
        adox    Q6, Q0

        mulx    Q5, Q0, [Q1 + 3*8]
        adcx    Q6, Q0
        adox    Q7, Q5      // This may overflow

        mov     Q0, 0
        adcx    Q7, Q0      // This may overflow
        adox    Q8, Q0

        adcx    Q8, Q0

        // Montgomery-reduced multiplication value in (Q10, Q11, Q6, Q7, Q8), and it is less than 2*Modulus
        // Compute value + (-Mod) into (Q0, Q2, Q3, Q5), Cy = inverted subtraction borrow

        add     Q1, SymCryptNegDivisorSingleDigitOffsetAmd64 - SymCryptModulusValueOffsetAmd64

        mov     Q0, 1
        add     Q0, Q10
        mov     Q2, [Q1 + 1*8]
        adc     Q2, Q11
        mov     Q3, [Q1 + 2*8]
        adc     Q3, Q6
        mov     Q5, [Q1 + 3*8]
        adc     Q5, Q7

        // Choose between the two
        // addition carry = 1, then subtraction borrow = 1 and we pick the 2nd result.
        // addition carry = 0 and subtraction borrow = 0: pick 2nd result
        // addition carry = 0 and subtraction borrow = 1: pick first result

        adc     Q8, Q8  // This is non-zero iff
                        //      the reduced value did not fit in 256-bits; or
                        //      the subtraction by Mod did not borrow
                        // adc sets the ZF appropriately

        cmovnz  Q10, Q0
        cmovnz  Q11, Q2
        cmovnz  Q6, Q3
        cmovnz  Q7, Q5

        mov     [Q4 + 0*8], Q10
        mov     [Q4 + 1*8], Q11
        mov     [Q4 + 2*8], Q6
        mov     [Q4 + 3*8], Q7


pop Q13
pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12
#undef Q13
#undef D13
#undef W13
#undef B13


//VOID
//SYMCRYPT_CALL
//SymCryptFdefModSquareMontgomeryMulxP256Asm(
//    _In_                            PCSYMCRYPT_MODULUS      pMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc,
//    _Out_                           PSYMCRYPT_MODELEMENT    pDst,
//    _Out_writes_bytes_( cbScratch ) PBYTE                   pbScratch,
//                                    SIZE_T                  cbScratch )

// Note we specify 4 arguments to keep prolog of this and SymCryptFdefModMulMontgomeryMulxP256Asm the same
#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
#define Q13 r15
#define D13 r15d
#define W13 r15w
#define B13 r15b
PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulxP256Asm):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulxP256Asm)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulxP256Asm); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulxP256Asm)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12
push Q13
.seh_pushreg Q13

.seh_endprologue

mov Q2, QH

        // Q1 = pMod
        // Q2 = pSrc
        // Q3 = pDst

        mov     Q4, Q3
        mov     Q3, Q2

        // Normal code doesn't jump from the body of one function to the body of another function.
        // Here we have ensured that our stack frames are identical, so it is safe.
        // We just have to convince the other system components that this works...

        // Use conditional jump so that stack unwinder doesn't think it is an epilogue
        test    rsp,rsp
        jne     SymCryptFdefModMulMontgomeryMulxP256AsmInternal       // jumps always

        int     3       // Dummy instruction because the debugger seems to have an off-by-one
                        // error and still see the (wrong) epilogue when on the JNE instruction
                        // Best guess: the debugger starts the stack trace *after* the current instruction


pop Q13
pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12
#undef Q13
#undef D13
#undef W13
#undef B13


.macro MUL_AND_MONTGOMERY_REDUCE16_INTERLEAVE  T0, T1, QH, pA, Aoff, pB, pM, K, R0, R1, R2, R3, R4, R5, R6, R7
    // (\R1, \R2, \R3, \R4, \R5, \R6, \R7) = A[\Aoff] * (B0..5) + (\R1, \R2, \R3, \R4, \R5, \R6, \R7)
    // (xx, \R1, \R2, \R3, \R4, \R5, \R6, \R7, \R0) = \K * (M0..5) + (\R0, \R1, \R2, \R3, \R4, \R5, \R6, \R7)
    // \K = modInv * \R1
    // Note \K is chosen using pMod's montgomery inverse s.t. xx would get set to 0
    // \pM is a pointer to the bytes of the modulus (i.e. offset into SYMCRYPT_MODULUS structure)
    // \R7 is 0 or 1 at start
    // \R0 is 0 or 1 at end
    // \T0, \T1 = scratch

    mov     \QH, [\pA + \Aoff]
    xor     \T0, \T0      // clear flags
                        // makes following carry chains architecturally independent of previous ones

    mulx    \T1, \T0, [\pB + 0*8]
    adox    \R1, \T0
    adcx    \R2, \T1

    mulx    \T1, \T0, [\pB + 1*8]
    adox    \R2, \T0
    adcx    \R3, \T1

    mulx    \T1, \T0, [\pB + 2*8]
    adox    \R3, \T0
    adcx    \R4, \T1

    mulx    \T1, \T0, [\pB + 3*8]
    adox    \R4, \T0
    adcx    \R5, \T1

    mulx    \T1, \T0, [\pB + 4*8]
    adox    \R5, \T0
    adcx    \R6, \T1

    mulx    \T1, \T0, [\pB + 5*8]
    adox    \R6, \T0
    mov     \T0, 0
    adcx    \T1, \T0      // \T1 in range [0, 2^64-1]
    adox    \R7, \T0      // \R7 in range [0, 2]
                        // defer adding \T1 to avoid overflowing earlier than necessary (we are bound by adox/adcx throughput)
                        // Multiplication result in \R1, \R2, \R3, \R4, \R5, \R6, \R7+\T1

    xor     \T0, \T0      // clear flags

    mov     \QH, \K       // now use \K as a temporary while \T1 has the value to add to \R7

    mulx    \K, \T0, [\pM + 0*8]
    adcx    \R0, \T0
    adox    \R1, \K

    mulx    \K, \T0, [\pM + 1*8]
    adcx    \R1, \T0
    adox    \R2, \K

    mulx    \K, \T0, [\pM + 2*8]
    adcx    \R2, \T0
    adox    \R3, \K

    mulx    \K, \T0, [\pM + 3*8]
    adcx    \R3, \T0
    adox    \R4, \K

    mulx    \K, \T0, [\pM + 4*8]
    adcx    \R4, \T0
    adox    \R5, \K

    mulx    \K, \T0, [\pM + 5*8]
    adcx    \R5, \T0
    adox    \R6, \K       // This may overflow

    mov     \R0, 0
    adcx    \R6, \R0      // This may overflow
    adox    \R7, \R0      // \R7 in range [0,3]

    adc     \R7, \T1      // This may overflow

    adc     \R0, \R0

    mov     \K, \R1
    imul    \K, [\pM - SymCryptModulusValueOffsetAmd64 + SymCryptModulusInv64OffsetAmd64]
.endm

#if defined(SYMCRYPT_MASM)
altentry SymCryptFdefModMulMontgomeryMulx384AsmInternal
#endif

//VOID
//SYMCRYPT_CALL
//SymCryptFdefModMulMontgomeryMulx384Asm(
//    _In_                            PCSYMCRYPT_MODULUS      pMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc1,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc2,
//    _Out_                           PSYMCRYPT_MODELEMENT    pDst,
//    _Out_writes_bytes_( cbScratch ) PBYTE                   pbScratch,
//                                    SIZE_T                  cbScratch )

// Note we specify only 4 arguments as we never use arguments 5 and 6 (saves some prolog code in MSFT calling convention)
#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
#define Q13 r15
#define D13 r15d
#define W13 r15w
#define B13 r15b
PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulx384Asm):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulx384Asm)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulx384Asm); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModMulMontgomeryMulx384Asm)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12
push Q13
.seh_pushreg Q13

.seh_endprologue

mov Q2, QH

        // Q1 = pMod
        // Q2 = pSrc1
        // Q3 = pSrc2
        // Q4 = pDst

SymCryptFdefModMulMontgomeryMulx384AsmInternal: .global SymCryptFdefModMulMontgomeryMulx384AsmInternal

        mov     [rsp + 72 /*MEMSLOT0*/], Q4   // save pDst

        // See SymCryptFdefModMulMontgomeryMulx256Asm for the rationale behind this function!

        xor     Q13, Q13  // clear flags
        mov     QH, [Q2]

        mulx    Q7, Q6,  [Q3 + 0*8]

        mulx    Q8, Q0,  [Q3 + 1*8]
        adc     Q7, Q0

        mulx    Q9, Q0,  [Q3 + 2*8]
        adc     Q8, Q0

        mulx    Q10, Q0, [Q3 + 3*8]
        adc     Q9, Q0

        mulx    Q11, Q0, [Q3 + 4*8]
        adc     Q10, Q0

        mulx    Q12, Q0, [Q3 + 5*8]
        adc     Q11, Q0

        adc     Q12, Q13        // Multiplication result in Q6, Q7, Q8, Q9, Q10, Q11, Q12

        // Compute K for reducing Multiplication result
        mov     Q4, Q6
        imul    Q4, [Q1 + SymCryptModulusInv64OffsetAmd64]
        add     Q1, SymCryptModulusValueOffsetAmd64

        MUL_AND_MONTGOMERY_REDUCE16_INTERLEAVE Q0, Q5, QH, Q2,  8, Q3, Q1, Q4, Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13

        MUL_AND_MONTGOMERY_REDUCE16_INTERLEAVE Q0, Q5, QH, Q2, 16, Q3, Q1, Q4, Q7, Q8, Q9, Q10, Q11, Q12, Q13, Q6

        MUL_AND_MONTGOMERY_REDUCE16_INTERLEAVE Q0, Q5, QH, Q2, 24, Q3, Q1, Q4, Q8, Q9, Q10, Q11, Q12, Q13, Q6, Q7

        MUL_AND_MONTGOMERY_REDUCE16_INTERLEAVE Q0, Q5, QH, Q2, 32, Q3, Q1, Q4, Q9, Q10, Q11, Q12, Q13, Q6, Q7, Q8

        MUL_AND_MONTGOMERY_REDUCE16_INTERLEAVE Q0, Q5, QH, Q2, 40, Q3, Q1, Q4, Q10, Q11, Q12, Q13, Q6, Q7, Q8, Q9

        xor     Q0, Q0          // clear flags
        mov     QH, Q4

        mulx    Q5, Q0, [Q1 + 0*8]
        adcx    Q11, Q0         // Set Cy when Q11 is non-zero
        adox    Q12, Q5

        mulx    Q5, Q0, [Q1 + 1*8]
        adcx    Q12, Q0
        adox    Q13, Q5

        mulx    Q5, Q0, [Q1 + 2*8]
        adcx    Q13, Q0
        adox    Q6, Q5

        mulx    Q5, Q0, [Q1 + 3*8]
        adcx    Q6, Q0
        adox    Q7, Q5

        mulx    Q5, Q0, [Q1 + 4*8]
        adcx    Q7, Q0
        adox    Q8, Q5

        mulx    Q5, Q0, [Q1 + 5*8]
        adcx    Q8, Q0
        adox    Q9, Q5          // This may overflow

        mov     Q0, 0
        adcx    Q9, Q0          // This may overflow
        adox    Q10, Q0

        adcx    Q10, Q0

        // Montgomery-reduced multiplication value in (Q12, Q13, Q6, Q7, Q8, Q9, Q10), and it is less than 2*Modulus
        // Compute value + (-Mod) into (Q0, Q2, Q3, Q5, Q11, Q1), Cy = inverted subtraction borrow

        add     Q1, SymCryptNegDivisorSingleDigitOffsetAmd64 - SymCryptModulusValueOffsetAmd64

        mov     Q0,  [Q1 + 0*8]
        add     Q0,  Q12
        mov     Q2,  [Q1 + 1*8]
        adc     Q2,  Q13
        mov     Q3,  [Q1 + 2*8]
        adc     Q3,  Q6
        mov     Q5,  [Q1 + 3*8]
        adc     Q5,  Q7
        mov     Q11, [Q1 + 4*8]
        adc     Q11, Q8
        mov     Q1,  [Q1 + 5*8]
        adc     Q1,  Q9

        // Choose between the two
        // addition carry = 1, then subtraction borrow = 1 and we pick the 2nd result.
        // addition carry = 0 and subtraction borrow = 0: pick 2nd result
        // addition carry = 0 and subtraction borrow = 1: pick first result

        adc     Q10, Q10        // This is non-zero iff
                                //      the reduced value did not fit in 384-bits; or
                                //      the subtraction by Mod did not borrow
                                // adc sets the ZF appropriately

        cmovnz  Q12, Q0
        cmovnz  Q13, Q2
        cmovnz  Q6,  Q3
        cmovnz  Q7,  Q5
        cmovnz  Q8,  Q11
        cmovnz  Q9,  Q1

        mov     Q4, [rsp + 72 /*MEMSLOT0*/]   // restore pDst

        mov     [Q4 + 0*8], Q12
        mov     [Q4 + 1*8], Q13
        mov     [Q4 + 2*8], Q6
        mov     [Q4 + 3*8], Q7
        mov     [Q4 + 4*8], Q8
        mov     [Q4 + 5*8], Q9


pop Q13
pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12
#undef Q13
#undef D13
#undef W13
#undef B13

//VOID
//SYMCRYPT_CALL
//SymCryptFdefModSquareMontgomeryMulx384Asm(
//    _In_                            PCSYMCRYPT_MODULUS      pMod,
//    _In_                            PCSYMCRYPT_MODELEMENT   pSrc,
//    _Out_                           PSYMCRYPT_MODELEMENT    pDst,
//    _Out_writes_bytes_( cbScratch ) PBYTE                   pbScratch,
//                                    SIZE_T                  cbScratch )

// Note we specify 4 arguments to keep prolog of this and SymCryptFdefModMulMontgomeryMulx384Asm the same
#define QH rdx
#define DH edx
#define WH dx
#define BH dl
#define Q0 rax
#define D0 eax
#define W0 ax
#define B0 al
#define Q1 rcx
#define D1 ecx
#define W1 cx
#define B1 cl
#define Q3 r8
#define D3 r8d
#define W3 r8w
#define B3 r8b
#define Q4 r9
#define D4 r9d
#define W4 r9w
#define B4 r9b
#define Q2 r10
#define D2 r10d
#define W2 r10w
#define B2 r10b
#define Q5 r11
#define D5 r11d
#define W5 r11w
#define B5 r11b
#define Q6 rsi
#define D6 esi
#define W6 si
#define B6 sil
#define Q7 rdi
#define D7 edi
#define W7 di
#define B7 dil
#define Q8 rbp
#define D8 ebp
#define W8 bp
#define B8 bpl
#define Q9 rbx
#define D9 ebx
#define W9 bx
#define B9 bl
#define Q10 r12
#define D10 r12d
#define W10 r12w
#define B10 r12b
#define Q11 r13
#define D11 r13d
#define W11 r13w
#define B11 r13b
#define Q12 r14
#define D12 r14d
#define W12 r14w
#define B12 r14b
#define Q13 r15
#define D13 r15d
#define W13 r15w
#define B13 r15b
PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulx384Asm):
.globl PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulx384Asm)
.def PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulx384Asm); .scl 2; .type 32; .endef
.seh_proc PLATFORM_NAME_MANGLE(SymCryptFdefModSquareMontgomeryMulx384Asm)

.byte 0x40; push Q6
.seh_pushreg Q6
push Q7
.seh_pushreg Q7
push Q8
.seh_pushreg Q8
push Q9
.seh_pushreg Q9
push Q10
.seh_pushreg Q10
push Q11
.seh_pushreg Q11
push Q12
.seh_pushreg Q12
push Q13
.seh_pushreg Q13

.seh_endprologue

mov Q2, QH

        // Q1 = pMod
        // Q2 = pSrc
        // Q3 = pDst

        mov     Q4, Q3
        mov     Q3, Q2

        // Normal code doesn't jump from the body of one function to the body of another function.
        // Here we have ensured that our stack frames are identical, so it is safe.
        // We just have to convince the other system components that this works...

        // Use conditional jump so that stack unwinder doesn't think it is an epilogue
        test    rsp,rsp
        jne     SymCryptFdefModMulMontgomeryMulx384AsmInternal       // jumps always

        int     3       // Dummy instruction because the debugger seems to have an off-by-one
                        // error and still see the (wrong) epilogue when on the JNE instruction
                        // Best guess: the debugger starts the stack trace *after* the current instruction


pop Q13
pop Q12
pop Q11
pop Q10
pop Q9
pop Q8
pop Q7
pop Q6
ret
.seh_endproc
#undef QH
#undef DH
#undef WH
#undef BH
#undef Q0
#undef D0
#undef W0
#undef B0
#undef Q1
#undef D1
#undef W1
#undef B1
#undef Q3
#undef D3
#undef W3
#undef B3
#undef Q4
#undef D4
#undef W4
#undef B4
#undef Q2
#undef D2
#undef W2
#undef B2
#undef Q5
#undef D5
#undef W5
#undef B5
#undef Q6
#undef D6
#undef W6
#undef B6
#undef Q7
#undef D7
#undef W7
#undef B7
#undef Q8
#undef D8
#undef W8
#undef B8
#undef Q9
#undef D9
#undef W9
#undef B9
#undef Q10
#undef D10
#undef W10
#undef B10
#undef Q11
#undef D11
#undef W11
#undef B11
#undef Q12
#undef D12
#undef W12
#undef B12
#undef Q13
#undef D13
#undef W13
#undef B13

#endif

FILE_END()
