dnl  AMD K7 mpn_addlsh1_n -- rp[] = up[] + (vp[] << 1)

dnl  Copyright 2011 Free Software Foundation, Inc.

dnl  Contributed to the GNU project by Torbjorn Granlund and Marco Bodrato.

dnl  This file is part of the GNU MP Library.
dnl
dnl  The GNU MP Library is free software; you can redistribute it and/or modify
dnl  it under the terms of either:
dnl
dnl    * the GNU Lesser General Public License as published by the Free
dnl      Software Foundation; either version 3 of the License, or (at your
dnl      option) any later version.
dnl
dnl  or
dnl
dnl    * the GNU General Public License as published by the Free Software
dnl      Foundation; either version 2 of the License, or (at your option) any
dnl      later version.
dnl
dnl  or both in parallel, as here.
dnl
dnl  The GNU MP Library is distributed in the hope that it will be useful, but
dnl  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
dnl  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
dnl  for more details.
dnl
dnl  You should have received copies of the GNU General Public License and the
dnl  GNU Lesser General Public License along with the GNU MP Library.  If not,
dnl  see https://www.gnu.org/licenses/.

include(`../config.m4')

C This is an attempt at an addlsh1_n for x86-32, not relying on sse2 insns.
C The innerloop is 2*3-way unrolled, which is best we can do with the available
C registers.  It seems tricky to use the same structure for rsblsh1_n, since we
C cannot feed carry between operations there.

C			    cycles/limb
C P5
C P6 model 0-8,10-12
C P6 model 9  (Banias)
C P6 model 13 (Dothan)		 5.4	(worse than add_n + lshift)
C P4 model 0  (Willamette)
C P4 model 1  (?)
C P4 model 2  (Northwood)
C P4 model 3  (Prescott)
C P4 model 4  (Nocona)
C Intel Atom			 6
C AMD K6			 ?
C AMD K7			 2.5
C AMD K8

C This is a basic addlsh1_n for k7, atom, and perhaps some other x86-32
C processors.  It uses 2*3-way unrolling, for good reasons.  Unfortunately,
C that means we need an initial magic multiply.
C
C It is not clear how to do sublsh1_n or rsblsh1_n using the same pattern.  We
C cannot do rsblsh1_n since we feed carry from the shift blocks to the
C add/subtract blocks, which is right for addition but reversed for
C subtraction.  We could perhaps do sublsh1_n, with some extra move insns,
C without losing any time, since we're not issue limited but carry recurrency
C latency.
C
C Breaking carry recurrency might be a good idea.  We would then need separate
C registers for the shift carry and add/subtract carry, which in turn would
C force is to 2*2-way unrolling.

defframe(PARAM_SIZE,	16)
defframe(PARAM_DBLD,	12)
defframe(PARAM_SRC,	 8)
defframe(PARAM_DST,	 4)

dnl  re-use parameter space
define(VAR_COUNT,`PARAM_DST')
define(VAR_TMP,`PARAM_DBLD')

ASM_START()
	TEXT
	ALIGN(8)
PROLOGUE(mpn_addlsh1_n)
deflit(`FRAME',0)

define(`rp',  `%edi')
define(`up',  `%esi')
define(`vp',  `%ebp')

	mov	$0x2aaaaaab, %eax

	push	%ebx			FRAME_pushl()
	mov	PARAM_SIZE, %ebx	C size

	push	rp			FRAME_pushl()
	mov	PARAM_DST, rp

	mul	%ebx

	push	up			FRAME_pushl()
	mov	PARAM_SRC, up

	not	%edx			C count = -(size\8)-1
	mov	%edx, VAR_COUNT

	push	vp			FRAME_pushl()
	mov	PARAM_DBLD, vp

	lea	3(%edx,%edx,2), %ecx	C count*3+3 = -(size\6)*3
	xor	%edx, %edx
	lea	(%ebx,%ecx,2), %ebx	C size + (count*3+3)*2 = size % 6
	or	%ebx, %ebx
	jz	L(exact)

L(oop):
ifdef(`CPU_P6',`
	shr	%edx ')			C restore 2nd saved carry bit
	mov	(vp), %eax
	adc	%eax, %eax
	rcr	%edx			C restore 1st saved carry bit
	lea	4(vp), vp
	adc	(up), %eax
	lea	4(up), up
	adc	%edx, %edx		C save a carry bit in edx
ifdef(`CPU_P6',`
	adc	%edx, %edx ')		C save another carry bit in edx
	dec	%ebx
	mov	%eax, (rp)
	lea	4(rp), rp
	jnz	L(oop)
	mov	vp, VAR_TMP
L(exact):
	incl	VAR_COUNT
	jz	L(end)

	ALIGN(16)
L(top):
ifdef(`CPU_P6',`
	shr	%edx ')			C restore 2nd saved carry bit
	mov	(vp), %eax
	adc	%eax, %eax
	mov	4(vp), %ebx
	adc	%ebx, %ebx
	mov	8(vp), %ecx
	adc	%ecx, %ecx

	rcr	%edx			C restore 1st saved carry bit

	adc	(up), %eax
	mov	%eax, (rp)
	adc	4(up), %ebx
	mov	%ebx, 4(rp)
	adc	8(up), %ecx
	mov	%ecx, 8(rp)

	mov	12(vp), %eax
	adc	%eax, %eax
	mov	16(vp), %ebx
	adc	%ebx, %ebx
	mov	20(vp), %ecx
	adc	%ecx, %ecx

	lea	24(vp), vp
	adc	%edx, %edx		C save a carry bit in edx

	adc	12(up), %eax
	mov	%eax, 12(rp)
	adc	16(up), %ebx
	mov	%ebx, 16(rp)
	adc	20(up), %ecx

	lea	24(up), up

ifdef(`CPU_P6',`
	adc	%edx, %edx ')		C save another carry bit in edx
	mov	%ecx, 20(rp)
	incl	VAR_COUNT
	lea	24(rp), rp
	jne	L(top)

L(end):
	pop	vp			FRAME_popl()
	pop	up			FRAME_popl()

ifdef(`CPU_P6',`
	xor	%eax, %eax
	shr	$1, %edx
	adc	%edx, %eax
',`
	adc	$0, %edx
	mov	%edx, %eax
')
	pop	rp			FRAME_popl()
	pop	%ebx			FRAME_popl()
	ret
EPILOGUE()
ASM_END()