dnl Intel Pentium-II mpn_divrem_1 -- mpn by limb division.
dnl Copyright 1999-2002 Free Software Foundation, Inc.
dnl This file is part of the GNU MP Library.
dnl
dnl The GNU MP Library is free software; you can redistribute it and/or modify
dnl it under the terms of either:
dnl
dnl * the GNU Lesser General Public License as published by the Free
dnl Software Foundation; either version 3 of the License, or (at your
dnl option) any later version.
dnl
dnl or
dnl
dnl * the GNU General Public License as published by the Free Software
dnl Foundation; either version 2 of the License, or (at your option) any
dnl later version.
dnl
dnl or both in parallel, as here.
dnl
dnl The GNU MP Library is distributed in the hope that it will be useful, but
dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
dnl for more details.
dnl
dnl You should have received copies of the GNU General Public License and the
dnl GNU Lesser General Public License along with the GNU MP Library. If not,
dnl see https://www.gnu.org/licenses/.
include(`../config.m4')
C P6MMX: 25.0 cycles/limb integer part, 17.5 cycles/limb fraction part.
C mp_limb_t mpn_divrem_1 (mp_ptr dst, mp_size_t xsize,
C mp_srcptr src, mp_size_t size,
C mp_limb_t divisor);
C mp_limb_t mpn_divrem_1c (mp_ptr dst, mp_size_t xsize,
C mp_srcptr src, mp_size_t size,
C mp_limb_t divisor, mp_limb_t carry);
C mp_limb_t mpn_preinv_divrem_1 (mp_ptr dst, mp_size_t xsize,
C mp_srcptr src, mp_size_t size,
C mp_limb_t divisor, mp_limb_t inverse,
C unsigned shift);
C
C This code is a lightly reworked version of mpn/x86/k7/mmx/divrem_1.asm,
C see that file for some comments. It's possible what's here can be improved.
dnl MUL_THRESHOLD is the value of xsize+size at which the multiply by
dnl inverse method is used, rather than plain "divl"s. Minimum value 1.
dnl
dnl The different speeds of the integer and fraction parts means that using
dnl xsize+size isn't quite right. The threshold wants to be a bit higher
dnl for the integer part and a bit lower for the fraction part. (Or what's
dnl really wanted is to speed up the integer part!)
dnl
dnl The threshold is set to make the integer part right. At 4 limbs the
dnl div and mul are about the same there, but on the fractional part the
dnl mul is much faster.
deflit(MUL_THRESHOLD, 4)
defframe(PARAM_PREINV_SHIFT, 28) dnl mpn_preinv_divrem_1
defframe(PARAM_PREINV_INVERSE, 24) dnl mpn_preinv_divrem_1
defframe(PARAM_CARRY, 24) dnl mpn_divrem_1c
defframe(PARAM_DIVISOR,20)
defframe(PARAM_SIZE, 16)
defframe(PARAM_SRC, 12)
defframe(PARAM_XSIZE, 8)
defframe(PARAM_DST, 4)
defframe(SAVE_EBX, -4)
defframe(SAVE_ESI, -8)
defframe(SAVE_EDI, -12)
defframe(SAVE_EBP, -16)
defframe(VAR_NORM, -20)
defframe(VAR_INVERSE, -24)
defframe(VAR_SRC, -28)
defframe(VAR_DST, -32)
defframe(VAR_DST_STOP,-36)
deflit(STACK_SPACE, 36)
TEXT
ALIGN(16)
PROLOGUE(mpn_preinv_divrem_1)
deflit(`FRAME',0)
movl PARAM_XSIZE, %ecx
subl $STACK_SPACE, %esp FRAME_subl_esp(STACK_SPACE)
movl %esi, SAVE_ESI
movl PARAM_SRC, %esi
movl %ebx, SAVE_EBX
movl PARAM_SIZE, %ebx
movl %ebp, SAVE_EBP
movl PARAM_DIVISOR, %ebp
movl %edi, SAVE_EDI
movl PARAM_DST, %edx
movl -4(%esi,%ebx,4), %eax C src high limb
xorl %edi, %edi C initial carry (if can't skip a div)
C
leal 8(%edx,%ecx,4), %edx C &dst[xsize+2]
xor %ecx, %ecx
movl %edx, VAR_DST_STOP C &dst[xsize+2]
cmpl %ebp, %eax C high cmp divisor
cmovc( %eax, %edi) C high is carry if high<divisor
cmovnc( %eax, %ecx) C 0 if skip div, src high if not
C (the latter in case src==dst)
movl %ecx, -12(%edx,%ebx,4) C dst high limb
sbbl $0, %ebx C skip one division if high<divisor
movl PARAM_PREINV_SHIFT, %ecx
leal -8(%edx,%ebx,4), %edx C &dst[xsize+size]
movl $32, %eax
movl %edx, VAR_DST C &dst[xsize+size]
shll %cl, %ebp C d normalized
subl %ecx, %eax
movl %ecx, VAR_NORM
movd %eax, %mm7 C rshift
movl PARAM_PREINV_INVERSE, %eax
jmp L(start_preinv)
EPILOGUE()
ALIGN(16)
PROLOGUE(mpn_divrem_1c)
deflit(`FRAME',0)
movl PARAM_CARRY, %edx
movl PARAM_SIZE, %ecx
subl $STACK_SPACE, %esp
deflit(`FRAME',STACK_SPACE)
movl %ebx, SAVE_EBX
movl PARAM_XSIZE, %ebx
movl %edi, SAVE_EDI
movl PARAM_DST, %edi
movl %ebp, SAVE_EBP
movl PARAM_DIVISOR, %ebp
movl %esi, SAVE_ESI
movl PARAM_SRC, %esi
leal -4(%edi,%ebx,4), %edi
jmp L(start_1c)
EPILOGUE()
C offset 0x31, close enough to aligned
PROLOGUE(mpn_divrem_1)
deflit(`FRAME',0)
movl PARAM_SIZE, %ecx
movl $0, %edx C initial carry (if can't skip a div)
subl $STACK_SPACE, %esp
deflit(`FRAME',STACK_SPACE)
movl %ebp, SAVE_EBP
movl PARAM_DIVISOR, %ebp
movl %ebx, SAVE_EBX
movl PARAM_XSIZE, %ebx
movl %esi, SAVE_ESI
movl PARAM_SRC, %esi
orl %ecx, %ecx C size
movl %edi, SAVE_EDI
movl PARAM_DST, %edi
leal -4(%edi,%ebx,4), %edi C &dst[xsize-1]
jz L(no_skip_div) C if size==0
movl -4(%esi,%ecx,4), %eax C src high limb
xorl %esi, %esi
cmpl %ebp, %eax C high cmp divisor
cmovc( %eax, %edx) C high is carry if high<divisor
cmovnc( %eax, %esi) C 0 if skip div, src high if not
C (the latter in case src==dst)
movl %esi, (%edi,%ecx,4) C dst high limb
sbbl $0, %ecx C size-1 if high<divisor
movl PARAM_SRC, %esi C reload
L(no_skip_div):
L(start_1c):
C eax
C ebx xsize
C ecx size
C edx carry
C esi src
C edi &dst[xsize-1]
C ebp divisor
leal (%ebx,%ecx), %eax C size+xsize
cmpl $MUL_THRESHOLD, %eax
jae L(mul_by_inverse)
orl %ecx, %ecx
jz L(divide_no_integer)
L(divide_integer):
C eax scratch (quotient)
C ebx xsize
C ecx counter
C edx scratch (remainder)
C esi src
C edi &dst[xsize-1]
C ebp divisor
movl -4(%esi,%ecx,4), %eax
divl %ebp
movl %eax, (%edi,%ecx,4)
decl %ecx
jnz L(divide_integer)
L(divide_no_integer):
movl PARAM_DST, %edi
orl %ebx, %ebx
jnz L(divide_fraction)
L(divide_done):
movl SAVE_ESI, %esi
movl SAVE_EDI, %edi
movl SAVE_EBX, %ebx
movl %edx, %eax
movl SAVE_EBP, %ebp
addl $STACK_SPACE, %esp
ret
L(divide_fraction):
C eax scratch (quotient)
C ebx counter
C ecx
C edx scratch (remainder)
C esi
C edi dst
C ebp divisor
movl $0, %eax
divl %ebp
movl %eax, -4(%edi,%ebx,4)
decl %ebx
jnz L(divide_fraction)
jmp L(divide_done)
C -----------------------------------------------------------------------------
L(mul_by_inverse):
C eax
C ebx xsize
C ecx size
C edx carry
C esi src
C edi &dst[xsize-1]
C ebp divisor
leal 12(%edi), %ebx C &dst[xsize+2], loop dst stop
movl %ebx, VAR_DST_STOP
leal 4(%edi,%ecx,4), %edi C &dst[xsize+size]
movl %edi, VAR_DST
movl %ecx, %ebx C size
bsrl %ebp, %ecx C 31-l
movl %edx, %edi C carry
leal 1(%ecx), %eax C 32-l
xorl $31, %ecx C l
movl %ecx, VAR_NORM
movl $-1, %edx
shll %cl, %ebp C d normalized
movd %eax, %mm7
movl $-1, %eax
subl %ebp, %edx C (b-d)-1 giving edx:eax = b*(b-d)-1
divl %ebp C floor (b*(b-d)-1) / d
L(start_preinv):
C eax inverse
C ebx size
C ecx shift
C edx
C esi src
C edi carry
C ebp divisor
C
C mm7 rshift
movl %eax, VAR_INVERSE
orl %ebx, %ebx C size
leal -12(%esi,%ebx,4), %eax C &src[size-3]
movl %eax, VAR_SRC
jz L(start_zero)
movl 8(%eax), %esi C src high limb
cmpl $1, %ebx
jz L(start_one)
L(start_two_or_more):
movl 4(%eax), %edx C src second highest limb
shldl( %cl, %esi, %edi) C n2 = carry,high << l
shldl( %cl, %edx, %esi) C n10 = high,second << l
cmpl $2, %ebx
je L(integer_two_left)
jmp L(integer_top)
L(start_one):
shldl( %cl, %esi, %edi) C n2 = carry,high << l
shll %cl, %esi C n10 = high << l
jmp L(integer_one_left)
L(start_zero):
C Can be here with xsize==0 if mpn_preinv_divrem_1 had size==1 and
C skipped a division.
shll %cl, %edi C n2 = carry << l
movl %edi, %eax C return value for zero_done
cmpl $0, PARAM_XSIZE
je L(zero_done)
jmp L(fraction_some)
C -----------------------------------------------------------------------------
C
C This loop runs at about 25 cycles, which is probably sub-optimal, and
C certainly more than the dependent chain would suggest. A better loop, or
C a better rough analysis of what's possible, would be welcomed.
C
C In the current implementation, the following successively dependent
C micro-ops seem to exist.
C
C uops
C n2+n1 1 (addl)
C mul 5
C q1+1 3 (addl/adcl)
C mul 5
C sub 3 (subl/sbbl)
C addback 2 (cmov)
C ---
C 19
C
C Lack of registers hinders explicit scheduling and it might be that the
C normal out of order execution isn't able to hide enough under the mul
C latencies.
C
C Using sarl/negl to pick out n1 for the n2+n1 stage is a touch faster than
C cmov (and takes one uop off the dependent chain). A sarl/andl/addl
C combination was tried for the addback (despite the fact it would lengthen
C the dependent chain) but found to be no faster.
ALIGN(16)
L(integer_top):
C eax scratch
C ebx scratch (nadj, q1)
C ecx scratch (src, dst)
C edx scratch
C esi n10
C edi n2
C ebp d
C
C mm0 scratch (src qword)
C mm7 rshift for normalization
movl %esi, %eax
movl %ebp, %ebx
sarl $31, %eax C -n1
movl VAR_SRC, %ecx
andl %eax, %ebx C -n1 & d
negl %eax C n1
addl %esi, %ebx C nadj = n10 + (-n1 & d), ignoring overflow
addl %edi, %eax C n2+n1
movq (%ecx), %mm0 C next src limb and the one below it
mull VAR_INVERSE C m*(n2+n1)
subl $4, %ecx
movl %ecx, VAR_SRC
C
C
addl %ebx, %eax C m*(n2+n1) + nadj, low giving carry flag
movl %ebp, %eax C d
leal 1(%edi), %ebx C n2+1
adcl %edx, %ebx C 1 + high(n2<<32 + m*(n2+n1) + nadj) = q1+1
jz L(q1_ff)
mull %ebx C (q1+1)*d
movl VAR_DST, %ecx
psrlq %mm7, %mm0
C
C
C
subl %eax, %esi
movl VAR_DST_STOP, %eax
sbbl %edx, %edi C n - (q1+1)*d
movl %esi, %edi C remainder -> n2
leal (%ebp,%esi), %edx
cmovc( %edx, %edi) C n - q1*d if underflow from using q1+1
movd %mm0, %esi
sbbl $0, %ebx C q
subl $4, %ecx
movl %ebx, (%ecx)
cmpl %eax, %ecx
movl %ecx, VAR_DST
jne L(integer_top)
L(integer_loop_done):
C -----------------------------------------------------------------------------
C
C Here, and in integer_one_left below, an sbbl $0 is used rather than a jz
C q1_ff special case. This make the code a bit smaller and simpler, and
C costs only 2 cycles (each).
L(integer_two_left):
C eax scratch
C ebx scratch (nadj, q1)
C ecx scratch (src, dst)
C edx scratch
C esi n10
C edi n2
C ebp divisor
C
C mm7 rshift
movl %esi, %eax
movl %ebp, %ebx
sarl $31, %eax C -n1
movl PARAM_SRC, %ecx
andl %eax, %ebx C -n1 & d
negl %eax C n1
addl %esi, %ebx C nadj = n10 + (-n1 & d), ignoring overflow
addl %edi, %eax C n2+n1
mull VAR_INVERSE C m*(n2+n1)
movd (%ecx), %mm0 C src low limb
movl VAR_DST_STOP, %ecx
C
C
addl %ebx, %eax C m*(n2+n1) + nadj, low giving carry flag
leal 1(%edi), %ebx C n2+1
movl %ebp, %eax C d
adcl %edx, %ebx C 1 + high(n2<<32 + m*(n2+n1) + nadj) = q1+1
sbbl $0, %ebx
mull %ebx C (q1+1)*d
psllq $32, %mm0
psrlq %mm7, %mm0
C
C
subl %eax, %esi
sbbl %edx, %edi C n - (q1+1)*d
movl %esi, %edi C remainder -> n2
leal (%ebp,%esi), %edx
cmovc( %edx, %edi) C n - q1*d if underflow from using q1+1
movd %mm0, %esi
sbbl $0, %ebx C q
movl %ebx, -4(%ecx)
C -----------------------------------------------------------------------------
L(integer_one_left):
C eax scratch
C ebx scratch (nadj, q1)
C ecx scratch (dst)
C edx scratch
C esi n10
C edi n2
C ebp divisor
C
C mm7 rshift
movl %esi, %eax
movl %ebp, %ebx
sarl $31, %eax C -n1
movl VAR_DST_STOP, %ecx
andl %eax, %ebx C -n1 & d
negl %eax C n1
addl %esi, %ebx C nadj = n10 + (-n1 & d), ignoring overflow
addl %edi, %eax C n2+n1
mull VAR_INVERSE C m*(n2+n1)
C
C
C
addl %ebx, %eax C m*(n2+n1) + nadj, low giving carry flag
leal 1(%edi), %ebx C n2+1
movl %ebp, %eax C d
C
adcl %edx, %ebx C 1 + high(n2<<32 + m*(n2+n1) + nadj) = q1+1
sbbl $0, %ebx C q1 if q1+1 overflowed
mull %ebx
C
C
C
C
subl %eax, %esi
movl PARAM_XSIZE, %eax
sbbl %edx, %edi C n - (q1+1)*d
movl %esi, %edi C remainder -> n2
leal (%ebp,%esi), %edx
cmovc( %edx, %edi) C n - q1*d if underflow from using q1+1
sbbl $0, %ebx C q
movl %ebx, -8(%ecx)
subl $8, %ecx
orl %eax, %eax C xsize
jnz L(fraction_some)
movl %edi, %eax
L(fraction_done):
movl VAR_NORM, %ecx
L(zero_done):
movl SAVE_EBP, %ebp
movl SAVE_EDI, %edi
movl SAVE_ESI, %esi
movl SAVE_EBX, %ebx
addl $STACK_SPACE, %esp
shrl %cl, %eax
emms
ret
C -----------------------------------------------------------------------------
C
C Special case for q1=0xFFFFFFFF, giving q=0xFFFFFFFF meaning the low dword
C of q*d is simply -d and the remainder n-q*d = n10+d
L(q1_ff):
C eax (divisor)
C ebx (q1+1 == 0)
C ecx
C edx
C esi n10
C edi n2
C ebp divisor
movl VAR_DST, %ecx
movl VAR_DST_STOP, %edx
subl $4, %ecx
movl %ecx, VAR_DST
psrlq %mm7, %mm0
leal (%ebp,%esi), %edi C n-q*d remainder -> next n2
movl $-1, (%ecx)
movd %mm0, %esi C next n10
cmpl %ecx, %edx
jne L(integer_top)
jmp L(integer_loop_done)
C -----------------------------------------------------------------------------
C
C In the current implementation, the following successively dependent
C micro-ops seem to exist.
C
C uops
C mul 5
C q1+1 1 (addl)
C mul 5
C sub 3 (negl/sbbl)
C addback 2 (cmov)
C ---
C 16
C
C The loop in fact runs at about 17.5 cycles. Using a sarl/andl/addl for
C the addback was found to be a touch slower.
ALIGN(16)
L(fraction_some):
C eax
C ebx
C ecx
C edx
C esi
C edi carry
C ebp divisor
movl PARAM_DST, %esi
movl VAR_DST_STOP, %ecx C &dst[xsize+2]
movl %edi, %eax
subl $8, %ecx C &dst[xsize]
ALIGN(16)
L(fraction_top):
C eax n2, then scratch
C ebx scratch (nadj, q1)
C ecx dst, decrementing
C edx scratch
C esi dst stop point
C edi n2
C ebp divisor
mull VAR_INVERSE C m*n2
movl %ebp, %eax C d
subl $4, %ecx C dst
leal 1(%edi), %ebx
C
C
C
addl %edx, %ebx C 1 + high(n2<<32 + m*n2) = q1+1
mull %ebx C (q1+1)*d
C
C
C
C
negl %eax C low of n - (q1+1)*d
sbbl %edx, %edi C high of n - (q1+1)*d, caring only about carry
leal (%ebp,%eax), %edx
cmovc( %edx, %eax) C n - q1*d if underflow from using q1+1
sbbl $0, %ebx C q
movl %eax, %edi C remainder->n2
cmpl %esi, %ecx
movl %ebx, (%ecx) C previous q
jne L(fraction_top)
jmp L(fraction_done)
EPILOGUE()