1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/os/kernelhwsrv/kernel/eka/common/arm/atomics.cia Fri Jun 15 03:10:57 2012 +0200
1.3 @@ -0,0 +1,540 @@
1.4 +// Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
1.5 +// All rights reserved.
1.6 +// This component and the accompanying materials are made available
1.7 +// under the terms of the License "Eclipse Public License v1.0"
1.8 +// which accompanies this distribution, and is available
1.9 +// at the URL "http://www.eclipse.org/legal/epl-v10.html".
1.10 +//
1.11 +// Initial Contributors:
1.12 +// Nokia Corporation - initial contribution.
1.13 +//
1.14 +// Contributors:
1.15 +//
1.16 +// Description:
1.17 +// e32\common\arm\atomics.cia
1.18 +//
1.19 +//
1.20 +
1.21 +
1.22 +#include <cpudefs.h>
1.23 +#include <e32def.h>
1.24 +//#include <e32atomics.h>
1.25 +
1.26 +#if defined(__KERNEL_MODE__)
1.27 +#include "nk_cpu.h"
1.28 +#elif defined(__ATOMIC_USE_FAST_EXEC__) || defined(__ATOMIC64_USE_FAST_EXEC__) || defined(__ATOMIC64_USE_SLOW_EXEC__)
1.29 +#include <u32exec.h>
1.30 +#endif
1.31 +
1.32 +#define __concat__(a,b) a##b
1.33 +#define __concat3__(a,b,c) a##b##c
1.34 +#define __concat5__(a,b,c,d,e) a##b##c##d##e
1.35 +#define __fname__(type,order,size) __concat5__(__e32_atomic_,type,_,order,size)
1.36 +// __e32_atomic_##type##_##order##size
1.37 +
1.38 +#undef __BARRIERS_NEEDED__
1.39 +#undef __AVOID_READ_SIDE_EFFECTS__
1.40 +#ifdef __SMP__
1.41 +#define __BARRIERS_NEEDED__
1.42 +#else
1.43 +#ifdef __KERNEL_MODE__
1.44 +// On non-SMP use interrupt disabling even on V6 and V6K just in case someone
1.45 +// has used the atomic operations on I/O addresses.
1.46 +#define __AVOID_READ_SIDE_EFFECTS__
1.47 +#endif
1.48 +#endif
1.49 +
1.50 +#ifdef __BARRIERS_NEEDED__
1.51 +#define __LOCAL_DATA_MEMORY_BARRIER__(reg) __DATA_MEMORY_BARRIER__(reg)
1.52 +#define __LOCAL_DATA_MEMORY_BARRIER_Z__(reg) __DATA_MEMORY_BARRIER_Z__(reg)
1.53 +#define __LOCAL_DATA_SYNC_BARRIER__(reg) __DATA_SYNC_BARRIER__(reg)
1.54 +#define __LOCAL_DATA_SYNC_BARRIER_Z__(reg) __DATA_SYNC_BARRIER_Z__(reg)
1.55 +#define __LOCAL_INST_SYNC_BARRIER__(reg) __INST_SYNC_BARRIER__(reg)
1.56 +#define __LOCAL_INST_SYNC_BARRIER_Z__(reg) __INST_SYNC_BARRIER_Z__(reg)
1.57 +#else // __BARRIERS_NEEDED__
1.58 +#define __LOCAL_DATA_MEMORY_BARRIER__(reg)
1.59 +#define __LOCAL_DATA_MEMORY_BARRIER_Z__(reg)
1.60 +#define __LOCAL_DATA_SYNC_BARRIER__(reg)
1.61 +#define __LOCAL_DATA_SYNC_BARRIER_Z__(reg)
1.62 +#define __LOCAL_INST_SYNC_BARRIER__(reg)
1.63 +#define __LOCAL_INST_SYNC_BARRIER_Z__(reg)
1.64 +#endif // __BARRIERS_NEEDED__
1.65 +
1.66 +#ifdef __CPU_ARM_HAS_CPS
1.67 +#define __DISABLE_INTERRUPTS__(keep,temp) asm("mrs "#keep ", cpsr"); CPSIDAIF
1.68 +#define __RESTORE_INTERRUPTS__(keep) asm("msr cpsr_c, "#keep ) // flags preserved
1.69 +#else
1.70 +#define __DISABLE_INTERRUPTS__(keep,temp) asm("mrs "#keep ", cpsr"); asm("orr "#temp ", "#keep ", #0xc0" ); asm("msr cpsr, "#temp )
1.71 +#define __RESTORE_INTERRUPTS__(keep) asm("msr cpsr_c, "#keep ) // flags preserved
1.72 +#endif
1.73 +
1.74 +/******************************************************************************
1.75 + * Barriers
1.76 + ******************************************************************************/
1.77 +
1.78 +extern "C" EXPORT_C __NAKED__ void __e32_memory_barrier()
1.79 + {
1.80 + __LOCAL_DATA_MEMORY_BARRIER_Z__(r0);
1.81 + __JUMP(,lr);
1.82 + }
1.83 +
1.84 +/** Barrier guaranteeing completion as well as ordering
1.85 +
1.86 +*/
1.87 +#if defined(__KERNEL_MODE__) || defined(__CPU_ARM_SUPPORTS_USER_MODE_BARRIERS)
1.88 +extern "C" EXPORT_C __NAKED__ void __e32_io_completion_barrier()
1.89 + {
1.90 + __DATA_SYNC_BARRIER_Z__(r0);
1.91 + __JUMP(,lr);
1.92 + }
1.93 +#else
1.94 +extern "C" EXPORT_C __NAKED__ void __e32_io_completion_barrier()
1.95 + {
1.96 + asm("mov r0, sp ");
1.97 + asm("mov r1, #0 ");
1.98 + SLOW_EXEC2(EExecIMBRange);
1.99 + }
1.100 +#endif
1.101 +
1.102 +
1.103 +/******************************************************************************
1.104 + * Miscellaneous utility functions
1.105 + ******************************************************************************/
1.106 +
1.107 +/** Find the most significant 1 in a 32 bit word
1.108 +
1.109 + @param v The word to be scanned
1.110 + @return The bit number of the most significant 1 if v != 0
1.111 + -1 if v == 0
1.112 +*/
1.113 +extern "C" EXPORT_C __NAKED__ TInt __e32_find_ms1_32(TUint32 /*v*/)
1.114 + {
1.115 +#ifdef __CPU_ARM_HAS_CLZ
1.116 + CLZ( 1,0); // r1=31-MSB(r0), 32 if r0=0
1.117 + asm("rsb r0, r1, #31 "); // r0=MSB(r0), -1 if r0=0
1.118 +#else
1.119 + asm("movs r1, r0 ");
1.120 + asm("beq 0f ");
1.121 + asm("mov r0, #31 ");
1.122 + asm("cmp r1, #0x00010000 ");
1.123 + asm("movcc r1, r1, lsl #16 ");
1.124 + asm("subcc r0, r0, #16 ");
1.125 + asm("cmp r1, #0x01000000 ");
1.126 + asm("movcc r1, r1, lsl #8 ");
1.127 + asm("subcc r0, r0, #8 ");
1.128 + asm("cmp r1, #0x10000000 ");
1.129 + asm("movcc r1, r1, lsl #4 ");
1.130 + asm("subcc r0, r0, #4 ");
1.131 + asm("cmp r1, #0x40000000 ");
1.132 + asm("movcc r1, r1, lsl #2 ");
1.133 + asm("subcc r0, r0, #2 ");
1.134 + asm("cmp r1, #0x80000000 ");
1.135 + asm("subcc r0, r0, #1 ");
1.136 + __JUMP(, lr);
1.137 + asm("0: ");
1.138 + asm("mvn r0, #0 "); // if input zero, return -1
1.139 +#endif
1.140 + __JUMP(, lr);
1.141 + }
1.142 +
1.143 +
1.144 +/** Find the least significant 1 in a 32 bit word
1.145 +
1.146 + @param v The word to be scanned
1.147 + @return The bit number of the least significant 1 if v != 0
1.148 + -1 if v == 0
1.149 +*/
1.150 +extern "C" EXPORT_C __NAKED__ TInt __e32_find_ls1_32(TUint32 /*v*/)
1.151 + {
1.152 +#ifdef __CPU_ARM_HAS_CLZ
1.153 + asm("subs r1, r0, #1 "); // r1 = arg - 1
1.154 + asm("eorcs r0, r0, r1 "); // if arg=0, leave alone else mask upper bits
1.155 + CLZ( 1,0); // r1=31-MSB(r0), 32 if r0=0
1.156 + asm("rsb r0, r1, #31 "); // r0=MSB(r0), -1 if r0=0
1.157 +#else
1.158 + asm("movs r1, r0 ");
1.159 + asm("beq 0f ");
1.160 + asm("mov r0, #0 ");
1.161 + asm("movs r2, r1, lsl #16 ");
1.162 + asm("movne r1, r2 ");
1.163 + asm("addeq r0, r0, #16 ");
1.164 + asm("movs r2, r1, lsl #8 ");
1.165 + asm("movne r1, r2 ");
1.166 + asm("addeq r0, r0, #8 ");
1.167 + asm("movs r2, r1, lsl #4 ");
1.168 + asm("movne r1, r2 ");
1.169 + asm("addeq r0, r0, #4 ");
1.170 + asm("movs r2, r1, lsl #2 ");
1.171 + asm("movne r1, r2 ");
1.172 + asm("addeq r0, r0, #2 ");
1.173 + asm("movs r2, r1, lsl #1 ");
1.174 + asm("addeq r0, r0, #1 ");
1.175 + __JUMP(, lr);
1.176 + asm("0: ");
1.177 + asm("mvn r0, #0 "); // if input zero, return -1
1.178 +#endif
1.179 + __JUMP(, lr);
1.180 + }
1.181 +
1.182 +
1.183 +/** Count the number of 1's in a 32 bit word
1.184 +
1.185 + @param v The word to be scanned
1.186 + @return The number of 1's
1.187 +*/
1.188 +extern "C" EXPORT_C __NAKED__ TInt __e32_bit_count_32(TUint32 /*v*/)
1.189 + {
1.190 + asm("mov r2, #0x0f "); // r2=0x0000000f
1.191 + asm("orr r2, r2, r2, lsl #8 "); // r2=0x00000f0f
1.192 + asm("orr r2, r2, r2, lsl #16 "); // r2=0x0f0f0f0f
1.193 + asm("eor r3, r2, r2, lsl #2 "); // r3=0x33333333
1.194 + asm("eor ip, r3, r3, lsl #1 "); // ip=0x55555555
1.195 + asm("bic r1, r0, ip "); // r1=odd bits of input
1.196 + asm("and r0, r0, ip "); // r0=even bits of input
1.197 + asm("add r0, r0, r1, lsr #1 "); // r0[2n:2n+1] = in[2n]+in[2n+1], 0<=n<=15
1.198 + asm("bic r1, r0, r3 "); // r1 = r0[4n+2:4n+3] for 0<=n<=7, other bits 0
1.199 + asm("and r0, r0, r3 "); // r0 = r0[4n:4n+1] for 0<=n<=7, other bits 0
1.200 + asm("add r0, r0, r1, lsr #2 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3], 0<=n<=7
1.201 + asm("add r0, r0, r0, lsr #4 "); // r0[8n:8n+3]=in[8n]+in[8n+1]+...+in[8n+7], 0<=n<=3
1.202 + asm("and r0, r0, r2 "); // make sure other bits of r0 are zero
1.203 + asm("add r0, r0, r0, lsr #8 "); // r0[16n:16n+7]=in[16n]+in[16n+1]+...+in[16n+15], n=0,1
1.204 + asm("add r0, r0, r0, lsr #16 "); // r0[0:7]=SUM{ in[n] : 0<=n<=31 }
1.205 + asm("and r0, r0, #0xff "); // mask other unwanted bits
1.206 + __JUMP(, lr);
1.207 + }
1.208 +
1.209 +
1.210 +/** Find the most significant 1 in a 64 bit word
1.211 +
1.212 + @param v The word to be scanned
1.213 + @return The bit number of the most significant 1 if v != 0
1.214 + -1 if v == 0
1.215 +*/
1.216 +extern "C" EXPORT_C __NAKED__ TInt __e32_find_ms1_64(TUint64 /*v*/)
1.217 + {
1.218 + /* On entry argument in R1:R0 */
1.219 +#ifdef __CPU_ARM_HAS_CLZ
1.220 + CLZ( 2,1); // r2=31-MSB(r1), 32 if r1=0
1.221 + asm("subs r2, r2, #32 "); // r2=-1-MSB(r1), 0 if r1=0
1.222 + CLZcc(CC_EQ,2,0); // if r1=0, r2=31-MSB(r0), 32 if r0=0
1.223 + asm("rsb r0, r2, #31 "); // if r1!=0, r0=32+MSB(r1) else if r0!=0 r0=MSB(r0) else r0=-1
1.224 +#else
1.225 + asm("cmp r1, #1 "); // r1>=1 ?
1.226 + asm("movcs r0, #63 "); // if so r0=63
1.227 + asm("movccs r1, r0 "); // else r1=r0, test for zero (C unaffected)
1.228 + asm("beq 0f ");
1.229 + asm("movcc r0, #31 "); // if r1=0 and r0!=0, r1=original r0 and r0=31
1.230 + asm("cmp r1, #0x00010000 ");
1.231 + asm("movcc r1, r1, lsl #16 ");
1.232 + asm("subcc r0, r0, #16 ");
1.233 + asm("cmp r1, #0x01000000 ");
1.234 + asm("movcc r1, r1, lsl #8 ");
1.235 + asm("subcc r0, r0, #8 ");
1.236 + asm("cmp r1, #0x10000000 ");
1.237 + asm("movcc r1, r1, lsl #4 ");
1.238 + asm("subcc r0, r0, #4 ");
1.239 + asm("cmp r1, #0x40000000 ");
1.240 + asm("movcc r1, r1, lsl #2 ");
1.241 + asm("subcc r0, r0, #2 ");
1.242 + asm("cmp r1, #0x80000000 ");
1.243 + asm("subcc r0, r0, #1 ");
1.244 + __JUMP(, lr);
1.245 + asm("0: ");
1.246 + asm("mvn r0, #0 "); // if input zero, return -1
1.247 +#endif
1.248 + __JUMP(, lr);
1.249 + }
1.250 +
1.251 +
1.252 +/** Find the least significant 1 in a 64 bit word
1.253 +
1.254 + @param v The word to be scanned
1.255 + @return The bit number of the least significant 1 if v != 0
1.256 + -1 if v == 0
1.257 +*/
1.258 +extern "C" EXPORT_C __NAKED__ TInt __e32_find_ls1_64(TUint64 /*v*/)
1.259 + {
1.260 + /* On entry argument in R1:R0 */
1.261 +#ifdef __CPU_ARM_HAS_CLZ
1.262 + asm("subs r2, r0, #1 ");
1.263 + asm("sbcs r3, r1, #0 "); // r3:r2 = arg - 1
1.264 + asm("eorcs r0, r0, r2 "); // if arg=0 leave alone else mask upper bits
1.265 + asm("eorcs r1, r1, r3 ");
1.266 + CLZ( 2,1); // r2=31-MSB(r1), 32 if r1=0
1.267 + asm("subs r2, r2, #32 "); // r2=-1-MSB(r1), 0 if r1=0
1.268 + CLZcc(CC_EQ,2,0); // if r1=0, r2=31-MSB(r0), 32 if r0=0
1.269 + asm("rsb r0, r2, #31 "); // if r1!=0, r0=32+MSB(r1) else if r0!=0 r0=MSB(r0) else r0=-1
1.270 +#else
1.271 + asm("cmp r0, #1 "); // LSW(arg) >= 1?
1.272 + asm("movcs r1, r0 "); // if so r1=r0
1.273 + asm("movcs r0, #32 "); // and r0=32
1.274 + asm("movcc r0, #0 "); // else r0=0
1.275 + asm("cmpcc r1, #1 "); // and test if MSW(arg) >= 1
1.276 + asm("bcc 0f "); // if not, return -1
1.277 + asm("movs r2, r1, lsl #16 ");
1.278 + asm("movne r1, r2 ");
1.279 + asm("addeq r0, r0, #16 ");
1.280 + asm("movs r2, r1, lsl #8 ");
1.281 + asm("movne r1, r2 ");
1.282 + asm("addeq r0, r0, #8 ");
1.283 + asm("movs r2, r1, lsl #4 ");
1.284 + asm("movne r1, r2 ");
1.285 + asm("addeq r0, r0, #4 ");
1.286 + asm("movs r2, r1, lsl #2 ");
1.287 + asm("movne r1, r2 ");
1.288 + asm("addeq r0, r0, #2 ");
1.289 + asm("movs r2, r1, lsl #1 ");
1.290 + asm("addeq r0, r0, #1 ");
1.291 + __JUMP(, lr);
1.292 + asm("0: ");
1.293 + asm("mvn r0, #0 "); // if input zero, return -1
1.294 +#endif
1.295 + __JUMP(, lr);
1.296 + }
1.297 +
1.298 +
1.299 +/** Count the number of 1's in a 64 bit word
1.300 +
1.301 + @param v The word to be scanned
1.302 + @return The number of 1's
1.303 +*/
1.304 +extern "C" EXPORT_C __NAKED__ TInt __e32_bit_count_64(TUint64 /*v*/)
1.305 + {
1.306 + /* On entry argument in R1:R0 */
1.307 + asm("str r4, [sp, #-4]! ");
1.308 + asm("mov r2, #0x0f "); // r2=0x0000000f
1.309 + asm("orr r2, r2, r2, lsl #8 "); // r2=0x00000f0f
1.310 + asm("orr r2, r2, r2, lsl #16 "); // r2=0x0f0f0f0f
1.311 + asm("eor r3, r2, r2, lsl #2 "); // r3=0x33333333
1.312 + asm("eor ip, r3, r3, lsl #1 "); // ip=0x55555555
1.313 +
1.314 + asm("bic r4, r0, ip "); // r4=odd bits of input LSW
1.315 + asm("and r0, r0, ip "); // r0=even bits of input LSW
1.316 + asm("add r0, r0, r4, lsr #1 "); // r0[2n:2n+1] = in[2n]+in[2n+1], 0<=n<=15
1.317 + asm("bic r4, r0, r3 "); // r4 = r0[4n+2:4n+3] for 0<=n<=7, other bits 0
1.318 + asm("and r0, r0, r3 "); // r0 = r0[4n:4n+1] for 0<=n<=7, other bits 0
1.319 + asm("add r0, r0, r4, lsr #2 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3], 0<=n<=7
1.320 +
1.321 + asm("bic r4, r1, ip "); // r4=odd bits of input MSW
1.322 + asm("and r1, r1, ip "); // r1=even bits of input MSW
1.323 + asm("add r1, r1, r4, lsr #1 "); // r1[2n:2n+1] = in[2n+32]+in[2n+33], 0<=n<=15
1.324 + asm("bic r4, r1, r3 "); // r4 = r1[4n+34:4n+35] for 0<=n<=7, other bits 0
1.325 + asm("and r1, r1, r3 "); // r1 = r1[4n+32:4n+33] for 0<=n<=7, other bits 0
1.326 + asm("add r1, r1, r4, lsr #2 "); // r1 bits 4n:4n+3 = in[4n+32]+in[4n+33]+in[4n+34]+in[4n+35], 0<=n<=7
1.327 + asm("ldr r4, [sp], #4 ");
1.328 +
1.329 + asm("add r0, r0, r1 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3]+in[4n+32]+in[4n+33]+in[4n+34]+in[4n+35], 0<=n<=7
1.330 + asm("bic r1, r0, r2 "); // odd nibbles only
1.331 + asm("and r0, r0, r2 "); // even nibbles only
1.332 + asm("add r0, r0, r1, lsr #4 "); // r0[8n:8n+7]=bit count of byte n of MSW + bit count of byte n of LSW
1.333 + asm("add r0, r0, r0, lsr #8 "); // r0[16n:16n+7]=bit count of hword n of MSW + bit count of hword n of LSW
1.334 + asm("add r0, r0, r0, lsr #16 "); // r0[0:7]=total bit count
1.335 + asm("and r0, r0, #0xff "); // mask other unwanted bits
1.336 + __JUMP(, lr);
1.337 + }
1.338 +
1.339 +
1.340 +
1.341 +/******************************************************************************
1.342 + * 64 bit operations
1.343 + ******************************************************************************/
1.344 +#define __DATA_SIZE__ 64
1.345 +#if defined(__CPU_ARM_HAS_LDREX_STREX_V6K) && !defined(__AVOID_READ_SIDE_EFFECTS__)
1.346 +
1.347 +// Include LDREXD/STREXD-based 64 bit operations
1.348 +#define __OP_LOAD__
1.349 +#include "atomic_64_v6k.h"
1.350 +#define __OP_STORE__
1.351 +#include "atomic_64_v6k.h"
1.352 +#define __OP_SWP__
1.353 +#include "atomic_64_v6k.h"
1.354 +#define __OP_CAS__
1.355 +#include "atomic_64_v6k.h"
1.356 +#define __OP_ADD__
1.357 +#include "atomic_64_v6k.h"
1.358 +#define __OP_AND__
1.359 +#include "atomic_64_v6k.h"
1.360 +#define __OP_IOR__
1.361 +#include "atomic_64_v6k.h"
1.362 +#define __OP_XOR__
1.363 +#include "atomic_64_v6k.h"
1.364 +#define __OP_AXO__
1.365 +#include "atomic_64_v6k.h"
1.366 +#define __OP_TAU__
1.367 +#include "atomic_64_v6k.h"
1.368 +#define __OP_TAS__
1.369 +#include "atomic_64_v6k.h"
1.370 +
1.371 +#else
1.372 +#ifdef __KERNEL_MODE__
1.373 +
1.374 +// Include interrupt-disabling 64 bit operations
1.375 +#define __OP_LOAD__
1.376 +#include "atomic_64_v6_v5.h"
1.377 +#define __OP_STORE__
1.378 +#include "atomic_64_v6_v5.h"
1.379 +#define __OP_SWP__
1.380 +#include "atomic_64_v6_v5.h"
1.381 +#define __OP_CAS__
1.382 +#include "atomic_64_v6_v5.h"
1.383 +#define __OP_ADD__
1.384 +#include "atomic_64_v6_v5.h"
1.385 +#define __OP_AND__
1.386 +#include "atomic_64_v6_v5.h"
1.387 +#define __OP_IOR__
1.388 +#include "atomic_64_v6_v5.h"
1.389 +#define __OP_XOR__
1.390 +#include "atomic_64_v6_v5.h"
1.391 +#define __OP_AXO__
1.392 +#include "atomic_64_v6_v5.h"
1.393 +#define __OP_TAU__
1.394 +#include "atomic_64_v6_v5.h"
1.395 +#define __OP_TAS__
1.396 +#include "atomic_64_v6_v5.h"
1.397 +
1.398 +#else
1.399 +
1.400 +// Include 64 bit operations using Exec calls
1.401 +#define __OP_LOAD__
1.402 +#include "atomic_64_v6_v5.h"
1.403 +#define __OP_STORE__
1.404 +#include "atomic_64_v6_v5.h"
1.405 +#define __OP_SWP__
1.406 +#include "atomic_64_exec.h"
1.407 +#define __OP_CAS__
1.408 +#include "atomic_64_exec.h"
1.409 +#define __OP_ADD__
1.410 +#include "atomic_64_exec.h"
1.411 +#define __OP_AND__
1.412 +#include "atomic_64_exec.h"
1.413 +#define __OP_IOR__
1.414 +#include "atomic_64_exec.h"
1.415 +#define __OP_XOR__
1.416 +#include "atomic_64_exec.h"
1.417 +#define __OP_AXO__
1.418 +#include "atomic_64_exec.h"
1.419 +#define __OP_TAU__
1.420 +#include "atomic_64_exec.h"
1.421 +#define __OP_TAS__
1.422 +#include "atomic_64_exec.h"
1.423 +
1.424 +#endif
1.425 +#endif
1.426 +#undef __DATA_SIZE__
1.427 +
1.428 +/******************************************************************************
1.429 + * 8,16,32 bit load/store operations
1.430 + ******************************************************************************/
1.431 +
1.432 +#define __DATA_SIZE__ 8
1.433 +#define __OP_LOAD__
1.434 +#include "atomic_32_v6.h"
1.435 +#define __OP_STORE__
1.436 +#include "atomic_32_v6.h"
1.437 +#undef __DATA_SIZE__
1.438 +
1.439 +#define __DATA_SIZE__ 16
1.440 +#define __OP_LOAD__
1.441 +#include "atomic_32_v6.h"
1.442 +#define __OP_STORE__
1.443 +#include "atomic_32_v6.h"
1.444 +#undef __DATA_SIZE__
1.445 +
1.446 +#define __DATA_SIZE__ 32
1.447 +#define __OP_LOAD__
1.448 +#include "atomic_32_v6.h"
1.449 +#define __OP_STORE__
1.450 +#include "atomic_32_v6.h"
1.451 +#undef __DATA_SIZE__
1.452 +
1.453 +/******************************************************************************
1.454 + * 8,16,32 bit RMW operations
1.455 + ******************************************************************************/
1.456 +
1.457 +#if defined(__CPU_ARM_HAS_LDREX_STREX_V6K) && !defined(__AVOID_READ_SIDE_EFFECTS__)
1.458 +// V6K - Use variants of LDREX/STREX for everything
1.459 +#define __ATOMIC_8_IMPL__ "atomic_32_v6.h"
1.460 +#define __ATOMIC_16_IMPL__ "atomic_32_v6.h"
1.461 +#define __ATOMIC_32_IMPL__ "atomic_32_v6.h"
1.462 +#elif defined(__CPU_ARM_HAS_LDREX_STREX) && !defined(__AVOID_READ_SIDE_EFFECTS__)
1.463 +// V6 - Use LDREX/STREX for 32 bit operations
1.464 +// Use LDREX/STREX with shifts/rotates for 8/16 bit operations
1.465 +#define __ATOMIC_8_IMPL__ "atomic_8_16_v6.h"
1.466 +#define __ATOMIC_16_IMPL__ "atomic_8_16_v6.h"
1.467 +#define __ATOMIC_32_IMPL__ "atomic_32_v6.h"
1.468 +#else
1.469 +// V5 - Use interrupt disabling kernel side, Exec calls user side
1.470 +#ifdef __KERNEL_MODE__
1.471 +#define __ATOMIC_8_IMPL__ "atomic_8_16_32_irq.h"
1.472 +#define __ATOMIC_16_IMPL__ "atomic_8_16_32_irq.h"
1.473 +#define __ATOMIC_32_IMPL__ "atomic_8_16_32_irq.h"
1.474 +#else
1.475 +#define __ATOMIC_8_IMPL__ "atomic_8_16_32_exec.h"
1.476 +#define __ATOMIC_16_IMPL__ "atomic_8_16_32_exec.h"
1.477 +#define __ATOMIC_32_IMPL__ "atomic_8_16_32_exec.h"
1.478 +#endif
1.479 +#endif
1.480 +
1.481 +#define __DATA_SIZE__ 8
1.482 +#define __OP_SWP__
1.483 +#include __ATOMIC_8_IMPL__
1.484 +#define __OP_CAS__
1.485 +#include __ATOMIC_8_IMPL__
1.486 +#define __OP_ADD__
1.487 +#include __ATOMIC_8_IMPL__
1.488 +#define __OP_AND__
1.489 +#include __ATOMIC_8_IMPL__
1.490 +#define __OP_IOR__
1.491 +#include __ATOMIC_8_IMPL__
1.492 +#define __OP_XOR__
1.493 +#include __ATOMIC_8_IMPL__
1.494 +#define __OP_AXO__
1.495 +#include __ATOMIC_8_IMPL__
1.496 +#define __OP_TAU__
1.497 +#include __ATOMIC_8_IMPL__
1.498 +#define __OP_TAS__
1.499 +#include __ATOMIC_8_IMPL__
1.500 +#undef __DATA_SIZE__
1.501 +
1.502 +#define __DATA_SIZE__ 16
1.503 +#define __OP_SWP__
1.504 +#include __ATOMIC_16_IMPL__
1.505 +#define __OP_CAS__
1.506 +#include __ATOMIC_16_IMPL__
1.507 +#define __OP_ADD__
1.508 +#include __ATOMIC_16_IMPL__
1.509 +#define __OP_AND__
1.510 +#include __ATOMIC_16_IMPL__
1.511 +#define __OP_IOR__
1.512 +#include __ATOMIC_16_IMPL__
1.513 +#define __OP_XOR__
1.514 +#include __ATOMIC_16_IMPL__
1.515 +#define __OP_AXO__
1.516 +#include __ATOMIC_16_IMPL__
1.517 +#define __OP_TAU__
1.518 +#include __ATOMIC_16_IMPL__
1.519 +#define __OP_TAS__
1.520 +#include __ATOMIC_16_IMPL__
1.521 +#undef __DATA_SIZE__
1.522 +
1.523 +#define __DATA_SIZE__ 32
1.524 +#define __OP_SWP__
1.525 +#include __ATOMIC_32_IMPL__
1.526 +#define __OP_CAS__
1.527 +#include __ATOMIC_32_IMPL__
1.528 +#define __OP_ADD__
1.529 +#include __ATOMIC_32_IMPL__
1.530 +#define __OP_AND__
1.531 +#include __ATOMIC_32_IMPL__
1.532 +#define __OP_IOR__
1.533 +#include __ATOMIC_32_IMPL__
1.534 +#define __OP_XOR__
1.535 +#include __ATOMIC_32_IMPL__
1.536 +#define __OP_AXO__
1.537 +#include __ATOMIC_32_IMPL__
1.538 +#define __OP_TAU__
1.539 +#include __ATOMIC_32_IMPL__
1.540 +#define __OP_TAS__
1.541 +#include __ATOMIC_32_IMPL__
1.542 +#undef __DATA_SIZE__
1.543 +