/* $NetBSD: atomic_op_asm.h,v 1.6 2021/07/29 10:29:05 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Matt Thomas of 3am Software Foundry.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _ATOMIC_OP_ASM_H_
#define _ATOMIC_OP_ASM_H_
#include <machine/asm.h>
#define ATOMIC_OP8(OP, INSN) \
ENTRY_NP(_atomic_##OP##_8) ;\
mov x4, x0 ;\
1: ldxrb w0, [x4] /* load old value */ ;\
INSN w2, w0, w1 /* calculate new value */ ;\
stxrb w3, w2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
ret /* return old value */ ;\
END(_atomic_##OP##_8)
#define SYNC_FETCH_OP8(OP, INSN) \
ENTRY_NP(__sync_fetch_and_##OP##_1) ;\
mov x4, x0 ;\
dmb ish ;\
1: ldxrb w0, [x4] /* load old value */ ;\
INSN w2, w0, w1 /* calculate new value */ ;\
stxrb w3, w2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
dmb ish ;\
ret /* return old value */ ;\
END(__sync_fetch_and_##OP##_1)
#define ATOMIC_OP8_NV(OP, INSN) \
ENTRY_NP(_atomic_##OP##_8_nv) ;\
mov x4, x0 /* need x0 for return value */ ;\
1: ldxrb w0, [x4] /* load old value */ ;\
INSN w0, w0, w1 /* calc new (return) value */ ;\
stxrb w3, w0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
ret /* return new value */ ;\
END(_atomic_##OP##_8_nv)
#define SYNC_OP8_FETCH(OP, INSN) \
ENTRY_NP(__sync_##OP##_and_fetch_1) ;\
mov x4, x0 /* need x0 for return value */ ;\
dmb ish ;\
1: ldxrb w0, [x4] /* load old value */ ;\
INSN w0, w0, w1 /* calc new (return) value */ ;\
stxrb w3, w0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
dmb ish ;\
ret /* return new value */ ;\
END(__sync_##OP##_and_fetch_1)
#define ATOMIC_OP16(OP, INSN) \
ENTRY_NP(_atomic_##OP##_16) ;\
mov x4, x0 ;\
1: ldxrh w0, [x4] /* load old value */ ;\
INSN w2, w0, w1 /* calculate new value */ ;\
stxrh w3, w2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
ret /* return old value */ ;\
END(_atomic_##OP##_16)
#define SYNC_FETCH_OP16(OP, INSN) \
ENTRY_NP(__sync_fetch_and_##OP##_2) ;\
mov x4, x0 ;\
dmb ish ;\
1: ldxrh w0, [x4] /* load old value */ ;\
INSN w2, w0, w1 /* calculate new value */ ;\
stxrh w3, w2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
dmb ish ;\
ret /* return old value */ ;\
END(__sync_fetch_and_##OP##_2)
#define ATOMIC_OP16_NV(OP, INSN) \
ENTRY_NP(_atomic_##OP##_16_nv) ;\
mov x4, x0 /* need x0 for return value */ ;\
1: ldxrh w0, [x4] /* load old value */ ;\
INSN w0, w0, w1 /* calc new (return) value */ ;\
stxrh w3, w0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
ret /* return new value */ ;\
END(_atomic_##OP##_16_nv)
#define SYNC_OP16_FETCH(OP, INSN) \
ENTRY_NP(__sync__##OP##_and_fetch_2) ;\
mov x4, x0 /* need x0 for return value */ ;\
dmb ish ;\
1: ldxrh w0, [x4] /* load old value */ ;\
INSN w0, w0, w1 /* calc new (return) value */ ;\
stxrh w3, w0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
dmb ish ;\
ret /* return new value */ ;\
END(__sync__##OP##_and_fetch_2)
#define ATOMIC_OP32(OP, INSN) \
ENTRY_NP(_atomic_##OP##_32) ;\
mov x4, x0 ;\
1: ldxr w0, [x4] /* load old value */ ;\
INSN w2, w0, w1 /* calculate new value */ ;\
stxr w3, w2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
ret /* return old value */ ;\
END(_atomic_##OP##_32)
#define SYNC_FETCH_OP32(OP, INSN) \
ENTRY_NP(__sync_fetch_and_##OP##_4) ;\
mov x4, x0 ;\
dmb ish ;\
1: ldxr w0, [x4] /* load old value */ ;\
INSN w2, w0, w1 /* calculate new value */ ;\
stxr w3, w2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
dmb ish ;\
ret /* return old value */ ;\
END(__sync_fetch_and_##OP##_4)
#define ATOMIC_OP32_NV(OP, INSN) \
ENTRY_NP(_atomic_##OP##_32_nv) ;\
mov x4, x0 /* need x0 for return value */ ;\
1: ldxr w0, [x4] /* load old value */ ;\
INSN w0, w0, w1 /* calc new (return) value */ ;\
stxr w3, w0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again? */ ;\
ret /* return new value */ ;\
END(_atomic_##OP##_32_nv)
#define SYNC_OP32_FETCH(OP, INSN) \
ENTRY_NP(__sync__##OP##_and_fetch_4) ;\
mov x4, x0 /* need x0 for return value */ ;\
dmb ish ;\
1: ldxr w0, [x4] /* load old value */ ;\
INSN w0, w0, w1 /* calc new (return) value */ ;\
stxr w3, w0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again? */ ;\
dmb ish ;\
ret /* return new value */ ;\
END(__sync__##OP##_and_fetch_4)
#define ATOMIC_OP64(OP, INSN) \
ENTRY_NP(_atomic_##OP##_64) ;\
mov x4, x0 ;\
1: ldxr x0, [x4] /* load old value */ ;\
INSN x2, x0, x1 /* calculate new value */ ;\
stxr w3, x2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
ret /* return old value */ ;\
END(_atomic_##OP##_64)
#define SYNC_FETCH_OP64(OP, INSN) \
ENTRY_NP(__sync_fetch_and_##OP##_8) ;\
mov x4, x0 ;\
dmb ish ;\
1: ldxr x0, [x4] /* load old value */ ;\
INSN x2, x0, x1 /* calculate new value */ ;\
stxr w3, x2, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again */ ;\
dmb ish ;\
ret /* return old value */ ;\
END(__sync_fetch_and_##OP##_8)
#define ATOMIC_OP64_NV(OP, INSN) \
ENTRY_NP(_atomic_##OP##_64_nv) ;\
mov x4, x0 /* need x0 for return value */ ;\
1: ldxr x0, [x4] /* load old value */ ;\
INSN x0, x0, x1 /* calc new (return) value */ ;\
stxr w3, x0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again? */ ;\
ret /* return new value */ ;\
END(_atomic_##OP##_64_nv)
#define SYNC_OP64_FETCH(OP, INSN) \
ENTRY_NP(__sync_##OP##_and_fetch_8) ;\
mov x4, x0 /* need x0 for return value */ ;\
dmb ish ;\
1: ldxr x0, [x4] /* load old value */ ;\
INSN x0, x0, x1 /* calc new (return) value */ ;\
stxr w3, x0, [x4] /* try to store */ ;\
cbnz w3, 1b /* succeed? no, try again? */ ;\
dmb ish ;\
ret /* return new value */ ;\
END(__sync_##OP##_and_fetch_8)
#if defined(_KERNEL)
#define ATOMIC_OP_ALIAS(a,s) STRONG_ALIAS(a,s)
#else /* _KERNEL */
#define ATOMIC_OP_ALIAS(a,s) WEAK_ALIAS(a,s)
#endif /* _KERNEL */
#endif /* _ATOMIC_OP_ASM_H_ */