diff -r 000000000000 -r bde4ae8d615e os/kernelhwsrv/kernel/eka/common/arm/atomics.cia --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/os/kernelhwsrv/kernel/eka/common/arm/atomics.cia Fri Jun 15 03:10:57 2012 +0200 @@ -0,0 +1,540 @@ +// Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). +// All rights reserved. +// This component and the accompanying materials are made available +// under the terms of the License "Eclipse Public License v1.0" +// which accompanies this distribution, and is available +// at the URL "http://www.eclipse.org/legal/epl-v10.html". +// +// Initial Contributors: +// Nokia Corporation - initial contribution. +// +// Contributors: +// +// Description: +// e32\common\arm\atomics.cia +// +// + + +#include +#include +//#include + +#if defined(__KERNEL_MODE__) +#include "nk_cpu.h" +#elif defined(__ATOMIC_USE_FAST_EXEC__) || defined(__ATOMIC64_USE_FAST_EXEC__) || defined(__ATOMIC64_USE_SLOW_EXEC__) +#include +#endif + +#define __concat__(a,b) a##b +#define __concat3__(a,b,c) a##b##c +#define __concat5__(a,b,c,d,e) a##b##c##d##e +#define __fname__(type,order,size) __concat5__(__e32_atomic_,type,_,order,size) +// __e32_atomic_##type##_##order##size + +#undef __BARRIERS_NEEDED__ +#undef __AVOID_READ_SIDE_EFFECTS__ +#ifdef __SMP__ +#define __BARRIERS_NEEDED__ +#else +#ifdef __KERNEL_MODE__ +// On non-SMP use interrupt disabling even on V6 and V6K just in case someone +// has used the atomic operations on I/O addresses. +#define __AVOID_READ_SIDE_EFFECTS__ +#endif +#endif + +#ifdef __BARRIERS_NEEDED__ +#define __LOCAL_DATA_MEMORY_BARRIER__(reg) __DATA_MEMORY_BARRIER__(reg) +#define __LOCAL_DATA_MEMORY_BARRIER_Z__(reg) __DATA_MEMORY_BARRIER_Z__(reg) +#define __LOCAL_DATA_SYNC_BARRIER__(reg) __DATA_SYNC_BARRIER__(reg) +#define __LOCAL_DATA_SYNC_BARRIER_Z__(reg) __DATA_SYNC_BARRIER_Z__(reg) +#define __LOCAL_INST_SYNC_BARRIER__(reg) __INST_SYNC_BARRIER__(reg) +#define __LOCAL_INST_SYNC_BARRIER_Z__(reg) __INST_SYNC_BARRIER_Z__(reg) +#else // __BARRIERS_NEEDED__ +#define __LOCAL_DATA_MEMORY_BARRIER__(reg) +#define __LOCAL_DATA_MEMORY_BARRIER_Z__(reg) +#define __LOCAL_DATA_SYNC_BARRIER__(reg) +#define __LOCAL_DATA_SYNC_BARRIER_Z__(reg) +#define __LOCAL_INST_SYNC_BARRIER__(reg) +#define __LOCAL_INST_SYNC_BARRIER_Z__(reg) +#endif // __BARRIERS_NEEDED__ + +#ifdef __CPU_ARM_HAS_CPS +#define __DISABLE_INTERRUPTS__(keep,temp) asm("mrs "#keep ", cpsr"); CPSIDAIF +#define __RESTORE_INTERRUPTS__(keep) asm("msr cpsr_c, "#keep ) // flags preserved +#else +#define __DISABLE_INTERRUPTS__(keep,temp) asm("mrs "#keep ", cpsr"); asm("orr "#temp ", "#keep ", #0xc0" ); asm("msr cpsr, "#temp ) +#define __RESTORE_INTERRUPTS__(keep) asm("msr cpsr_c, "#keep ) // flags preserved +#endif + +/****************************************************************************** + * Barriers + ******************************************************************************/ + +extern "C" EXPORT_C __NAKED__ void __e32_memory_barrier() + { + __LOCAL_DATA_MEMORY_BARRIER_Z__(r0); + __JUMP(,lr); + } + +/** Barrier guaranteeing completion as well as ordering + +*/ +#if defined(__KERNEL_MODE__) || defined(__CPU_ARM_SUPPORTS_USER_MODE_BARRIERS) +extern "C" EXPORT_C __NAKED__ void __e32_io_completion_barrier() + { + __DATA_SYNC_BARRIER_Z__(r0); + __JUMP(,lr); + } +#else +extern "C" EXPORT_C __NAKED__ void __e32_io_completion_barrier() + { + asm("mov r0, sp "); + asm("mov r1, #0 "); + SLOW_EXEC2(EExecIMBRange); + } +#endif + + +/****************************************************************************** + * Miscellaneous utility functions + ******************************************************************************/ + +/** Find the most significant 1 in a 32 bit word + + @param v The word to be scanned + @return The bit number of the most significant 1 if v != 0 + -1 if v == 0 +*/ +extern "C" EXPORT_C __NAKED__ TInt __e32_find_ms1_32(TUint32 /*v*/) + { +#ifdef __CPU_ARM_HAS_CLZ + CLZ( 1,0); // r1=31-MSB(r0), 32 if r0=0 + asm("rsb r0, r1, #31 "); // r0=MSB(r0), -1 if r0=0 +#else + asm("movs r1, r0 "); + asm("beq 0f "); + asm("mov r0, #31 "); + asm("cmp r1, #0x00010000 "); + asm("movcc r1, r1, lsl #16 "); + asm("subcc r0, r0, #16 "); + asm("cmp r1, #0x01000000 "); + asm("movcc r1, r1, lsl #8 "); + asm("subcc r0, r0, #8 "); + asm("cmp r1, #0x10000000 "); + asm("movcc r1, r1, lsl #4 "); + asm("subcc r0, r0, #4 "); + asm("cmp r1, #0x40000000 "); + asm("movcc r1, r1, lsl #2 "); + asm("subcc r0, r0, #2 "); + asm("cmp r1, #0x80000000 "); + asm("subcc r0, r0, #1 "); + __JUMP(, lr); + asm("0: "); + asm("mvn r0, #0 "); // if input zero, return -1 +#endif + __JUMP(, lr); + } + + +/** Find the least significant 1 in a 32 bit word + + @param v The word to be scanned + @return The bit number of the least significant 1 if v != 0 + -1 if v == 0 +*/ +extern "C" EXPORT_C __NAKED__ TInt __e32_find_ls1_32(TUint32 /*v*/) + { +#ifdef __CPU_ARM_HAS_CLZ + asm("subs r1, r0, #1 "); // r1 = arg - 1 + asm("eorcs r0, r0, r1 "); // if arg=0, leave alone else mask upper bits + CLZ( 1,0); // r1=31-MSB(r0), 32 if r0=0 + asm("rsb r0, r1, #31 "); // r0=MSB(r0), -1 if r0=0 +#else + asm("movs r1, r0 "); + asm("beq 0f "); + asm("mov r0, #0 "); + asm("movs r2, r1, lsl #16 "); + asm("movne r1, r2 "); + asm("addeq r0, r0, #16 "); + asm("movs r2, r1, lsl #8 "); + asm("movne r1, r2 "); + asm("addeq r0, r0, #8 "); + asm("movs r2, r1, lsl #4 "); + asm("movne r1, r2 "); + asm("addeq r0, r0, #4 "); + asm("movs r2, r1, lsl #2 "); + asm("movne r1, r2 "); + asm("addeq r0, r0, #2 "); + asm("movs r2, r1, lsl #1 "); + asm("addeq r0, r0, #1 "); + __JUMP(, lr); + asm("0: "); + asm("mvn r0, #0 "); // if input zero, return -1 +#endif + __JUMP(, lr); + } + + +/** Count the number of 1's in a 32 bit word + + @param v The word to be scanned + @return The number of 1's +*/ +extern "C" EXPORT_C __NAKED__ TInt __e32_bit_count_32(TUint32 /*v*/) + { + asm("mov r2, #0x0f "); // r2=0x0000000f + asm("orr r2, r2, r2, lsl #8 "); // r2=0x00000f0f + asm("orr r2, r2, r2, lsl #16 "); // r2=0x0f0f0f0f + asm("eor r3, r2, r2, lsl #2 "); // r3=0x33333333 + asm("eor ip, r3, r3, lsl #1 "); // ip=0x55555555 + asm("bic r1, r0, ip "); // r1=odd bits of input + asm("and r0, r0, ip "); // r0=even bits of input + asm("add r0, r0, r1, lsr #1 "); // r0[2n:2n+1] = in[2n]+in[2n+1], 0<=n<=15 + asm("bic r1, r0, r3 "); // r1 = r0[4n+2:4n+3] for 0<=n<=7, other bits 0 + asm("and r0, r0, r3 "); // r0 = r0[4n:4n+1] for 0<=n<=7, other bits 0 + asm("add r0, r0, r1, lsr #2 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3], 0<=n<=7 + asm("add r0, r0, r0, lsr #4 "); // r0[8n:8n+3]=in[8n]+in[8n+1]+...+in[8n+7], 0<=n<=3 + asm("and r0, r0, r2 "); // make sure other bits of r0 are zero + asm("add r0, r0, r0, lsr #8 "); // r0[16n:16n+7]=in[16n]+in[16n+1]+...+in[16n+15], n=0,1 + asm("add r0, r0, r0, lsr #16 "); // r0[0:7]=SUM{ in[n] : 0<=n<=31 } + asm("and r0, r0, #0xff "); // mask other unwanted bits + __JUMP(, lr); + } + + +/** Find the most significant 1 in a 64 bit word + + @param v The word to be scanned + @return The bit number of the most significant 1 if v != 0 + -1 if v == 0 +*/ +extern "C" EXPORT_C __NAKED__ TInt __e32_find_ms1_64(TUint64 /*v*/) + { + /* On entry argument in R1:R0 */ +#ifdef __CPU_ARM_HAS_CLZ + CLZ( 2,1); // r2=31-MSB(r1), 32 if r1=0 + asm("subs r2, r2, #32 "); // r2=-1-MSB(r1), 0 if r1=0 + CLZcc(CC_EQ,2,0); // if r1=0, r2=31-MSB(r0), 32 if r0=0 + asm("rsb r0, r2, #31 "); // if r1!=0, r0=32+MSB(r1) else if r0!=0 r0=MSB(r0) else r0=-1 +#else + asm("cmp r1, #1 "); // r1>=1 ? + asm("movcs r0, #63 "); // if so r0=63 + asm("movccs r1, r0 "); // else r1=r0, test for zero (C unaffected) + asm("beq 0f "); + asm("movcc r0, #31 "); // if r1=0 and r0!=0, r1=original r0 and r0=31 + asm("cmp r1, #0x00010000 "); + asm("movcc r1, r1, lsl #16 "); + asm("subcc r0, r0, #16 "); + asm("cmp r1, #0x01000000 "); + asm("movcc r1, r1, lsl #8 "); + asm("subcc r0, r0, #8 "); + asm("cmp r1, #0x10000000 "); + asm("movcc r1, r1, lsl #4 "); + asm("subcc r0, r0, #4 "); + asm("cmp r1, #0x40000000 "); + asm("movcc r1, r1, lsl #2 "); + asm("subcc r0, r0, #2 "); + asm("cmp r1, #0x80000000 "); + asm("subcc r0, r0, #1 "); + __JUMP(, lr); + asm("0: "); + asm("mvn r0, #0 "); // if input zero, return -1 +#endif + __JUMP(, lr); + } + + +/** Find the least significant 1 in a 64 bit word + + @param v The word to be scanned + @return The bit number of the least significant 1 if v != 0 + -1 if v == 0 +*/ +extern "C" EXPORT_C __NAKED__ TInt __e32_find_ls1_64(TUint64 /*v*/) + { + /* On entry argument in R1:R0 */ +#ifdef __CPU_ARM_HAS_CLZ + asm("subs r2, r0, #1 "); + asm("sbcs r3, r1, #0 "); // r3:r2 = arg - 1 + asm("eorcs r0, r0, r2 "); // if arg=0 leave alone else mask upper bits + asm("eorcs r1, r1, r3 "); + CLZ( 2,1); // r2=31-MSB(r1), 32 if r1=0 + asm("subs r2, r2, #32 "); // r2=-1-MSB(r1), 0 if r1=0 + CLZcc(CC_EQ,2,0); // if r1=0, r2=31-MSB(r0), 32 if r0=0 + asm("rsb r0, r2, #31 "); // if r1!=0, r0=32+MSB(r1) else if r0!=0 r0=MSB(r0) else r0=-1 +#else + asm("cmp r0, #1 "); // LSW(arg) >= 1? + asm("movcs r1, r0 "); // if so r1=r0 + asm("movcs r0, #32 "); // and r0=32 + asm("movcc r0, #0 "); // else r0=0 + asm("cmpcc r1, #1 "); // and test if MSW(arg) >= 1 + asm("bcc 0f "); // if not, return -1 + asm("movs r2, r1, lsl #16 "); + asm("movne r1, r2 "); + asm("addeq r0, r0, #16 "); + asm("movs r2, r1, lsl #8 "); + asm("movne r1, r2 "); + asm("addeq r0, r0, #8 "); + asm("movs r2, r1, lsl #4 "); + asm("movne r1, r2 "); + asm("addeq r0, r0, #4 "); + asm("movs r2, r1, lsl #2 "); + asm("movne r1, r2 "); + asm("addeq r0, r0, #2 "); + asm("movs r2, r1, lsl #1 "); + asm("addeq r0, r0, #1 "); + __JUMP(, lr); + asm("0: "); + asm("mvn r0, #0 "); // if input zero, return -1 +#endif + __JUMP(, lr); + } + + +/** Count the number of 1's in a 64 bit word + + @param v The word to be scanned + @return The number of 1's +*/ +extern "C" EXPORT_C __NAKED__ TInt __e32_bit_count_64(TUint64 /*v*/) + { + /* On entry argument in R1:R0 */ + asm("str r4, [sp, #-4]! "); + asm("mov r2, #0x0f "); // r2=0x0000000f + asm("orr r2, r2, r2, lsl #8 "); // r2=0x00000f0f + asm("orr r2, r2, r2, lsl #16 "); // r2=0x0f0f0f0f + asm("eor r3, r2, r2, lsl #2 "); // r3=0x33333333 + asm("eor ip, r3, r3, lsl #1 "); // ip=0x55555555 + + asm("bic r4, r0, ip "); // r4=odd bits of input LSW + asm("and r0, r0, ip "); // r0=even bits of input LSW + asm("add r0, r0, r4, lsr #1 "); // r0[2n:2n+1] = in[2n]+in[2n+1], 0<=n<=15 + asm("bic r4, r0, r3 "); // r4 = r0[4n+2:4n+3] for 0<=n<=7, other bits 0 + asm("and r0, r0, r3 "); // r0 = r0[4n:4n+1] for 0<=n<=7, other bits 0 + asm("add r0, r0, r4, lsr #2 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3], 0<=n<=7 + + asm("bic r4, r1, ip "); // r4=odd bits of input MSW + asm("and r1, r1, ip "); // r1=even bits of input MSW + asm("add r1, r1, r4, lsr #1 "); // r1[2n:2n+1] = in[2n+32]+in[2n+33], 0<=n<=15 + asm("bic r4, r1, r3 "); // r4 = r1[4n+34:4n+35] for 0<=n<=7, other bits 0 + asm("and r1, r1, r3 "); // r1 = r1[4n+32:4n+33] for 0<=n<=7, other bits 0 + asm("add r1, r1, r4, lsr #2 "); // r1 bits 4n:4n+3 = in[4n+32]+in[4n+33]+in[4n+34]+in[4n+35], 0<=n<=7 + asm("ldr r4, [sp], #4 "); + + asm("add r0, r0, r1 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3]+in[4n+32]+in[4n+33]+in[4n+34]+in[4n+35], 0<=n<=7 + asm("bic r1, r0, r2 "); // odd nibbles only + asm("and r0, r0, r2 "); // even nibbles only + asm("add r0, r0, r1, lsr #4 "); // r0[8n:8n+7]=bit count of byte n of MSW + bit count of byte n of LSW + asm("add r0, r0, r0, lsr #8 "); // r0[16n:16n+7]=bit count of hword n of MSW + bit count of hword n of LSW + asm("add r0, r0, r0, lsr #16 "); // r0[0:7]=total bit count + asm("and r0, r0, #0xff "); // mask other unwanted bits + __JUMP(, lr); + } + + + +/****************************************************************************** + * 64 bit operations + ******************************************************************************/ +#define __DATA_SIZE__ 64 +#if defined(__CPU_ARM_HAS_LDREX_STREX_V6K) && !defined(__AVOID_READ_SIDE_EFFECTS__) + +// Include LDREXD/STREXD-based 64 bit operations +#define __OP_LOAD__ +#include "atomic_64_v6k.h" +#define __OP_STORE__ +#include "atomic_64_v6k.h" +#define __OP_SWP__ +#include "atomic_64_v6k.h" +#define __OP_CAS__ +#include "atomic_64_v6k.h" +#define __OP_ADD__ +#include "atomic_64_v6k.h" +#define __OP_AND__ +#include "atomic_64_v6k.h" +#define __OP_IOR__ +#include "atomic_64_v6k.h" +#define __OP_XOR__ +#include "atomic_64_v6k.h" +#define __OP_AXO__ +#include "atomic_64_v6k.h" +#define __OP_TAU__ +#include "atomic_64_v6k.h" +#define __OP_TAS__ +#include "atomic_64_v6k.h" + +#else +#ifdef __KERNEL_MODE__ + +// Include interrupt-disabling 64 bit operations +#define __OP_LOAD__ +#include "atomic_64_v6_v5.h" +#define __OP_STORE__ +#include "atomic_64_v6_v5.h" +#define __OP_SWP__ +#include "atomic_64_v6_v5.h" +#define __OP_CAS__ +#include "atomic_64_v6_v5.h" +#define __OP_ADD__ +#include "atomic_64_v6_v5.h" +#define __OP_AND__ +#include "atomic_64_v6_v5.h" +#define __OP_IOR__ +#include "atomic_64_v6_v5.h" +#define __OP_XOR__ +#include "atomic_64_v6_v5.h" +#define __OP_AXO__ +#include "atomic_64_v6_v5.h" +#define __OP_TAU__ +#include "atomic_64_v6_v5.h" +#define __OP_TAS__ +#include "atomic_64_v6_v5.h" + +#else + +// Include 64 bit operations using Exec calls +#define __OP_LOAD__ +#include "atomic_64_v6_v5.h" +#define __OP_STORE__ +#include "atomic_64_v6_v5.h" +#define __OP_SWP__ +#include "atomic_64_exec.h" +#define __OP_CAS__ +#include "atomic_64_exec.h" +#define __OP_ADD__ +#include "atomic_64_exec.h" +#define __OP_AND__ +#include "atomic_64_exec.h" +#define __OP_IOR__ +#include "atomic_64_exec.h" +#define __OP_XOR__ +#include "atomic_64_exec.h" +#define __OP_AXO__ +#include "atomic_64_exec.h" +#define __OP_TAU__ +#include "atomic_64_exec.h" +#define __OP_TAS__ +#include "atomic_64_exec.h" + +#endif +#endif +#undef __DATA_SIZE__ + +/****************************************************************************** + * 8,16,32 bit load/store operations + ******************************************************************************/ + +#define __DATA_SIZE__ 8 +#define __OP_LOAD__ +#include "atomic_32_v6.h" +#define __OP_STORE__ +#include "atomic_32_v6.h" +#undef __DATA_SIZE__ + +#define __DATA_SIZE__ 16 +#define __OP_LOAD__ +#include "atomic_32_v6.h" +#define __OP_STORE__ +#include "atomic_32_v6.h" +#undef __DATA_SIZE__ + +#define __DATA_SIZE__ 32 +#define __OP_LOAD__ +#include "atomic_32_v6.h" +#define __OP_STORE__ +#include "atomic_32_v6.h" +#undef __DATA_SIZE__ + +/****************************************************************************** + * 8,16,32 bit RMW operations + ******************************************************************************/ + +#if defined(__CPU_ARM_HAS_LDREX_STREX_V6K) && !defined(__AVOID_READ_SIDE_EFFECTS__) +// V6K - Use variants of LDREX/STREX for everything +#define __ATOMIC_8_IMPL__ "atomic_32_v6.h" +#define __ATOMIC_16_IMPL__ "atomic_32_v6.h" +#define __ATOMIC_32_IMPL__ "atomic_32_v6.h" +#elif defined(__CPU_ARM_HAS_LDREX_STREX) && !defined(__AVOID_READ_SIDE_EFFECTS__) +// V6 - Use LDREX/STREX for 32 bit operations +// Use LDREX/STREX with shifts/rotates for 8/16 bit operations +#define __ATOMIC_8_IMPL__ "atomic_8_16_v6.h" +#define __ATOMIC_16_IMPL__ "atomic_8_16_v6.h" +#define __ATOMIC_32_IMPL__ "atomic_32_v6.h" +#else +// V5 - Use interrupt disabling kernel side, Exec calls user side +#ifdef __KERNEL_MODE__ +#define __ATOMIC_8_IMPL__ "atomic_8_16_32_irq.h" +#define __ATOMIC_16_IMPL__ "atomic_8_16_32_irq.h" +#define __ATOMIC_32_IMPL__ "atomic_8_16_32_irq.h" +#else +#define __ATOMIC_8_IMPL__ "atomic_8_16_32_exec.h" +#define __ATOMIC_16_IMPL__ "atomic_8_16_32_exec.h" +#define __ATOMIC_32_IMPL__ "atomic_8_16_32_exec.h" +#endif +#endif + +#define __DATA_SIZE__ 8 +#define __OP_SWP__ +#include __ATOMIC_8_IMPL__ +#define __OP_CAS__ +#include __ATOMIC_8_IMPL__ +#define __OP_ADD__ +#include __ATOMIC_8_IMPL__ +#define __OP_AND__ +#include __ATOMIC_8_IMPL__ +#define __OP_IOR__ +#include __ATOMIC_8_IMPL__ +#define __OP_XOR__ +#include __ATOMIC_8_IMPL__ +#define __OP_AXO__ +#include __ATOMIC_8_IMPL__ +#define __OP_TAU__ +#include __ATOMIC_8_IMPL__ +#define __OP_TAS__ +#include __ATOMIC_8_IMPL__ +#undef __DATA_SIZE__ + +#define __DATA_SIZE__ 16 +#define __OP_SWP__ +#include __ATOMIC_16_IMPL__ +#define __OP_CAS__ +#include __ATOMIC_16_IMPL__ +#define __OP_ADD__ +#include __ATOMIC_16_IMPL__ +#define __OP_AND__ +#include __ATOMIC_16_IMPL__ +#define __OP_IOR__ +#include __ATOMIC_16_IMPL__ +#define __OP_XOR__ +#include __ATOMIC_16_IMPL__ +#define __OP_AXO__ +#include __ATOMIC_16_IMPL__ +#define __OP_TAU__ +#include __ATOMIC_16_IMPL__ +#define __OP_TAS__ +#include __ATOMIC_16_IMPL__ +#undef __DATA_SIZE__ + +#define __DATA_SIZE__ 32 +#define __OP_SWP__ +#include __ATOMIC_32_IMPL__ +#define __OP_CAS__ +#include __ATOMIC_32_IMPL__ +#define __OP_ADD__ +#include __ATOMIC_32_IMPL__ +#define __OP_AND__ +#include __ATOMIC_32_IMPL__ +#define __OP_IOR__ +#include __ATOMIC_32_IMPL__ +#define __OP_XOR__ +#include __ATOMIC_32_IMPL__ +#define __OP_AXO__ +#include __ATOMIC_32_IMPL__ +#define __OP_TAU__ +#include __ATOMIC_32_IMPL__ +#define __OP_TAS__ +#include __ATOMIC_32_IMPL__ +#undef __DATA_SIZE__ +