/* $NetBSD: memset_neon.S,v 1.1 2012/12/13 01:41:59 matt Exp $ */
/*-
* Copyright (c) 2012 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Matt Thomas of 3am Software Foundry.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <machine/asm.h>
#include "assym.h"
/*
* memset: Sets a block of memory to the specified value
* Using NEON instructions
*
* On entry:
* r0 - dest address
* r1 - byte to write
* r2 - number of bytes to write
*
* On exit:
* r0 - dest address
*/
/* LINTSTUB: Func: void *memset(void *, int, size_t) */
ENTRY(memset)
and r3, r1, #0xff /* We deal with bytes */
mov r1, r2
mov ip, r0 /* r0 needs to stay the same */
vdup.8 q0, r3 /* move fill to SIMD */
/* we no longer need to keep the fill value in a ARM register */
/* Ok first we will dword align the address */
ands r2, ip, #7 /* grab the bottom three bits */
beq .Lmemset_dwordaligned /* The addr is dword aligned */
bic ip, ip, #7 /* clear bottom three bits of addr */
vldr d7, [ip] /* load from memory */
add r1, r1, r2 /* add "pre-fill" to length */
lsl r2, r2, #3 /* byte to no-fill bit count */
#ifdef __ARMEB__
neg r1, r1 /* start from the MSB */
#endif
vmov s4, r2 /* move to SIMD d2 */
vmvn.u64 d3, #0 /* set all ones */
vshl.u64 d3, d3, d2 /* create a no-fill mask */
vmvn.u64 d2, d3 /* invert mask for a fill-mask */
vand d7, d7, d3 /* mask out fill bits */
vand d2, d0, d2 /* mask out no-fill bits */
vorr d7, d2, d7 /* merge fill and memory */
cmp r1, #8 /* Do we have less than 8 bytes */
movlt r2, #0 /* indicate this is the last word */
blt .Lmemset_lessthaneight_noload
vstmia ip!, {d7} /* write back to memory */
subs r1, r1, #8 /* and remove 8 bytes from the length */
RETc(eq)
/* We are now doubleword aligned */
.Lmemset_dwordaligned:
vmov q1, q0 /* put fill in q1 (d2-d3) */
vmov q2, q0 /* put fill in q2 (d4-d5) */
vmov q3, q0 /* put fill in q3 (d6-d7) */
and r2, ip, #63 /* check for 64-byte alignment */
beq .Lmemset_8dwordaligned
/*
* Let's align to a 64-byte boundary so that stores don't cross
* cacheline boundaries. We also know we have at least 128-bytes to
* copy so we don't have to worry about the length at the moment.
*/
rsb r2, r2, #64 /* how many bytes until 64 bytes */
cmp r1, r2 /* compare against length */
andlt r2, r1, #0x38 /* if < len, use trunc(len, 8) */
subs r1, r1, r2 /* subtract from len */
add pc, pc, r2 /* and jump to it */
nop
RETc(eq); b .Lmemset_lessthaneight
vstmia ip!, {d0}; b .Lmemset_8dwordaligned
vstmia ip!, {d0-d1}; b .Lmemset_8dwordaligned
vstmia ip!, {d0-d2}; b .Lmemset_8dwordaligned
vstmia ip!, {d0-d3}; b .Lmemset_8dwordaligned
vstmia ip!, {d0-d4}; b .Lmemset_8dwordaligned
vstmia ip!, {d0-d5}; b .Lmemset_8dwordaligned
vstmia ip!, {d0-d6}
.Lmemset_8dwordaligned:
vmov d0, d1 /* restore in case of unaligned start */
cmp r1, #8 /* do we have less than 8 bytes */
movlt r2, #0 /* indicate last word */
blt .Lmemset_lessthaneight
cmp r1, #512
blt .Lmemset_sub512
/* Do 512 bytes at a time */
mov r2, #512
.Lmemset_512:
vstmia ip!, {d0-d7}
vstmia ip!, {d0-d7}
vstmia ip!, {d0-d7}
vstmia ip!, {d0-d7}
vstmia ip!, {d0-d7}
vstmia ip!, {d0-d7}
vstmia ip!, {d0-d7}
vstmia ip!, {d0-d7}
.Lmemset_0:
subs r1, r1, r2
RETc(eq) /* return if done */
cmp r1, #512
bge .Lmemset_512
/*
* We have less than 512 bytes left, but since the sequence above
* store 64 bytes at a time, we determine the number of instructions
* we need to store the remainder (if >= 64 bytes) and execute that
* many vstmia.
*/
.Lmemset_sub512:
lsr r2, r1, #6 /* divide by 64 */
lslne r4, r2, #2 /* multiply by 4 */
addne r4, r4, #1f + 8 - .Lmemset_0
/* add the # of bytes between */
1: subne pc, r4 /* and go */
/*
* We have less than 64 bytes to copy on a 8dword aligned address
*/
and r2, r1, #56 /* get # of full dwords */
ands r1, r1, #7 /* get # of extra bytes */
beq .Lmemset_finalstore
/*
* The last word is a partial fill so load its value and update it
* to include the fill value.
*/
.Lmemset_lessthaneight:
vldr d7, [ip, r2] /* load the last partial dword */
.Lmemset_lessthaneight_noload:
lsl r1, r1, #3 /* byte to fill bit count */
#ifdef __ARMEB__
neg r1, r1 /* start from the MSB */
#endif
vmov s4, r1 /* move to SIMD d2 */
vmvn.u64 d3, #0 /* set all ones */
vshl.u64 d3, d3, d2 /* create a no-fill mask */
vmvn.u64 d2, d3 /* invert mask */
vand d7, d7, d2 /* keep no-fill bits */
vand d2, d0, d3 /* mask out no-fill bits */
vorr d7, d2, d7 /* merge fill and no-fill */
vmov q1, q0 /* restore d2 & d3 */
add r2, r2, #8 /* compensate for the partial dword */
.Lmemset_finalstore:
add pc, pc, r2 /* and jump to it */
nop
vstr d7, [ip]; RET
vstmia ip, {d6-d7}; RET
vstmia ip, {d5-d7}; RET
vstmia ip, {d4-d7}; RET
vstmia ip, {d3-d7}; RET
vstmia ip, {d2-d7}; RET
vstmia ip, {d1-d7}; RET
vstmia ip, {d0-d7}; RET
END(memset)