// Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

// e32\common\x86\atomics.cpp

// 

//

#include <e32atomics.h>

#include <cpudefs.h>

/*

Versions needed:

	WINS/WINSCW		Use X86 locked operations. Assume Pentium or above CPU (CMPXCHG8B available)

	X86				For Pentium and above use locked operations

					For 486 use locked operations for 8, 16, 32 bit. For 64 bit must disable interrupts.

					NOTE: 486 not supported at the moment

	ARMv4/ARMv5		Must disable interrupts.

	ARMv6			LDREX/STREX for 8, 16, 32 bit. For 64 bit must disable interrupts (maybe).

	ARMv6K/ARMv7	LDREXB/LDREXH/LDREX/LDREXD

Need both kernel side and user side versions

*/

#if	defined(__SMP__) || !defined(__EPOC32__)

#define	__BARRIERS_NEEDED__

#define	__LOCK__	lock

#else

#define	__LOCK__

#endif

extern "C" {

#undef	__TUintX__

#undef	__TIntX__

#undef	__fname__

#undef	__A_REG__

#undef	__C_REG__

#undef	__D_REG__

#define	__TUintX__		TUint32

#define	__TIntX__		TInt32

#define	__fname__(x)	x##32

#define	__A_REG__		eax

#define	__C_REG__		ecx

#define	__D_REG__		edx

#include "atomic_skeleton.h"

#undef	__TUintX__

#undef	__TIntX__

#undef	__fname__

#undef	__A_REG__

#undef	__C_REG__

#undef	__D_REG__

#define	__TUintX__		TUint16

#define	__TIntX__		TInt16

#define	__fname__(x)	x##16

#define	__A_REG__		ax

#define	__C_REG__		cx

#define	__D_REG__		dx

#include "atomic_skeleton.h"

#undef	__TUintX__

#undef	__TIntX__

#undef	__fname__

#undef	__A_REG__

#undef	__C_REG__

#undef	__D_REG__

#define	__TUintX__		TUint8

#define	__TIntX__		TInt8

#define	__fname__(x)	x##8

#define	__A_REG__		al

#define	__C_REG__		cl

#define	__D_REG__		dl

#include "atomic_skeleton.h"

#undef	__TUintX__

#undef	__TIntX__

#undef	__fname__

#undef	__A_REG__

#undef	__C_REG__

#undef	__D_REG__

/** Full memory barrier for explicit memory accesses

*/

EXPORT_C __NAKED__ void __e32_memory_barrier()

	{

#ifdef __BARRIERS_NEEDED__

	_asm lock add dword ptr [esp], 0

#endif

	_asm ret

	}

/** Barrier guaranteeing completion as well as ordering

*/

EXPORT_C __NAKED__ void __e32_io_completion_barrier()

	{

	_asm push ebx

	_asm cpuid

	_asm pop ebx

	_asm ret

	}

/** Find the most significant 1 in a 32 bit word

	@param	v	The word to be scanned

	@return		The bit number of the most significant 1 if v != 0

				-1 if v == 0

*/

EXPORT_C __NAKED__ TInt __e32_find_ms1_32(TUint32 /*v*/)

	{

	_asm bsr eax, [esp+4]

	_asm jnz short done

	_asm mov eax, 0ffffffffh

done:

	_asm ret

	}

/** Find the least significant 1 in a 32 bit word

	@param	v	The word to be scanned

	@return		The bit number of the least significant 1 if v != 0

				-1 if v == 0

*/

EXPORT_C __NAKED__ TInt __e32_find_ls1_32(TUint32 /*v*/)

	{

	_asm bsf eax, [esp+4]

	_asm jnz short done

	_asm mov eax, 0ffffffffh

done:

	_asm ret

	}

/** Count the number of 1's in a 32 bit word

	@param	v	The word to be scanned

	@return		The number of 1's

*/

EXPORT_C __NAKED__ TInt __e32_bit_count_32(TUint32 /*v*/)

	{

	_asm mov eax, [esp+4]

	_asm mov edx, eax

	_asm and eax, 0aaaaaaaah

	_asm and edx, 055555555h

	_asm shr eax, 1

	_asm add eax, edx

	_asm mov edx, eax

	_asm and eax, 0cccccccch

	_asm and edx, 033333333h

	_asm shr eax, 2

	_asm add eax, edx

	_asm mov edx, eax

	_asm shr eax, 4

	_asm add eax, edx

	_asm and eax, 00f0f0f0fh

	_asm add al, ah

	_asm mov dl, al

	_asm shr eax, 16

	_asm add al, ah

	_asm xor ah, ah

	_asm add al, dl

	_asm ret

	}

/** Find the most significant 1 in a 64 bit word

	@param	v	The word to be scanned

	@return		The bit number of the most significant 1 if v != 0

				-1 if v == 0

*/

EXPORT_C __NAKED__ TInt __e32_find_ms1_64(TUint64 /*v*/)

	{

	_asm bsr eax, [esp+8]

	_asm jnz short mswnz

	_asm bsr eax, [esp+4]

	_asm jnz short lswnz

	_asm mov eax, 0ffffffffh

mswnz:

	_asm or eax, 32

lswnz:

	_asm ret

	}

/** Find the least significant 1 in a 64 bit word

	@param	v	The word to be scanned

	@return		The bit number of the least significant 1 if v != 0

				-1 if v == 0

*/

EXPORT_C __NAKED__ TInt __e32_find_ls1_64(TUint64 /*v*/)

	{

	_asm bsf eax, [esp+4]

	_asm jnz short lswnz

	_asm bsf eax, [esp+8]

	_asm jnz short mswnz

	_asm mov eax, 0ffffffffh

mswnz:

	_asm or eax, 32

lswnz:

	_asm ret

	}

/** Count the number of 1's in a 64 bit word

	@param	v	The word to be scanned

	@return		The number of 1's

*/

EXPORT_C __NAKED__ TInt __e32_bit_count_64(TUint64 /*v*/)

	{

	_asm mov eax, [esp+4]

	_asm mov edx, [esp+8]

	_asm mov ecx, eax

	_asm and eax, 0aaaaaaaah

	_asm and ecx, 055555555h

	_asm shr eax, 1

	_asm add eax, ecx			/* 16 groups of 2 bits, count=0,1,2 */

	_asm mov ecx, eax

	_asm and eax, 0cccccccch

	_asm and ecx, 033333333h

	_asm shr eax, 2

	_asm add ecx, eax			/* 8 groups of 4 bits, count=0...4 */

	_asm mov eax, edx

	_asm and eax, 0aaaaaaaah

	_asm and edx, 055555555h

	_asm shr eax, 1

	_asm add eax, edx			/* 16 groups of 2 bits, count=0,1,2 */

	_asm mov edx, eax

	_asm and eax, 0cccccccch

	_asm and edx, 033333333h

	_asm shr eax, 2

	_asm add eax, edx			/* 8 groups of 4 bits, count=0...4 */

	_asm add eax, ecx			/* 8 groups of 4 bits, count=0...8 */

	_asm mov edx, eax

	_asm and eax, 0f0f0f0f0h

	_asm and edx, 00f0f0f0fh

	_asm shr eax, 4

	_asm add eax, edx			/* 4 groups of 8 bits, count=0...16 */

	_asm add al, ah

	_asm mov dl, al

	_asm shr eax, 16

	_asm add al, ah

	_asm xor ah, ah

	_asm add al, dl

	_asm ret

	}

/** Read a 64 bit word with acquire semantics

	@param	a	Address of word to be read - must be a multiple of 8

	@return		The value read

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_load_acq64(const volatile TAny* /*a*/)

	{

	_asm push ebx

	_asm push edi

	_asm mov edi, [esp+12]

	_asm mov eax, 0badbeefh

	_asm mov edx, eax

	_asm mov ebx, eax

	_asm mov ecx, eax

	_asm __LOCK__ cmpxchg8b [edi]

	_asm pop edi

	_asm pop ebx

	_asm ret

	}

/** Write a 64 bit word with release semantics

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The value written

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_store_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm push ebx

	_asm push edi

	_asm mov edi, [esp+12]

	_asm mov ebx, [esp+16]

	_asm mov ecx, [esp+20]

	_asm mov eax, [edi]

	_asm mov edx, [edi+4]

	_asm retry:

	_asm __LOCK__ cmpxchg8b [edi]

	_asm jne short retry

	_asm mov eax, ebx

	_asm mov edx, ecx

	_asm pop edi

	_asm pop ebx

	_asm ret

	}

/** Write a 64 bit word with full barrier semantics

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The value written

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_store_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_store_rel64

	}

/** Write a 64 bit word to memory and return the original value of the memory.

	Relaxed ordering.

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_swp_ord64

	}

/** Write a 64 bit word to memory and return the original value of the memory.

	Acquire semantics.

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_swp_ord64

	}

/** Write a 64 bit word to memory and return the original value of the memory.

	Release semantics.

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_swp_ord64

	}

/** Write a 64 bit word to memory and return the original value of the memory.

	Full barrier semantics.

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm push ebx

	_asm push edi

	_asm mov edi, [esp+12]

	_asm mov ebx, [esp+16]

	_asm mov ecx, [esp+20]

	_asm mov eax, [edi]

	_asm mov edx, [edi+4]

	_asm retry:

	_asm __LOCK__ cmpxchg8b [edi]

	_asm jne short retry

	_asm pop edi

	_asm pop ebx

	_asm ret

	}

/** 64 bit compare and swap, relaxed ordering.

	Atomically performs the following operation:

		if (*a == *q)	{ *a = v; return TRUE; }

		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8

	@param	q	Address of location containing expected value

	@param	v	The new value to be written if the old value is as expected

	@return		TRUE if *a was updated, FALSE otherwise

*/

EXPORT_C __NAKED__ TBool		__e32_atomic_cas_rlx64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_cas_ord64

	}

/** 64 bit compare and swap, acquire semantics.

	Atomically performs the following operation:

		if (*a == *q)	{ *a = v; return TRUE; }

		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8

	@param	q	Address of location containing expected value

	@param	v	The new value to be written if the old value is as expected

	@return		TRUE if *a was updated, FALSE otherwise

*/

EXPORT_C __NAKED__ TBool		__e32_atomic_cas_acq64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_cas_ord64

	}

/** 64 bit compare and swap, release semantics.

	Atomically performs the following operation:

		if (*a == *q)	{ *a = v; return TRUE; }

		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8

	@param	q	Address of location containing expected value

	@param	v	The new value to be written if the old value is as expected

	@return		TRUE if *a was updated, FALSE otherwise

*/

EXPORT_C __NAKED__ TBool		__e32_atomic_cas_rel64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_cas_ord64

	}

/** 64 bit compare and swap, full barrier semantics.

	Atomically performs the following operation:

		if (*a == *q)	{ *a = v; return TRUE; }

		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8

	@param	q	Address of location containing expected value

	@param	v	The new value to be written if the old value is as expected

	@return		TRUE if *a was updated, FALSE otherwise

*/

EXPORT_C __NAKED__ TBool		__e32_atomic_cas_ord64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

	{

	_asm push ebx

	_asm push edi

	_asm push esi

	_asm mov edi, [esp+16]				// edi = a

	_asm mov esi, [esp+20]				// esi = q

	_asm mov ebx, [esp+24]				// ecx:ebx = v

	_asm mov ecx, [esp+28]

	_asm mov eax, [esi]					// edx:eax = *q

	_asm mov edx, [esi+4]

	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==*q) *a=v, ZF=1 else edx:eax=*a, ZF=0

	_asm jne short cas_fail

	_asm mov eax, 1

	_asm pop esi

	_asm pop edi

	_asm pop ebx

	_asm ret

	_asm cas_fail:

	_asm mov [esi], eax					// *q = edx:eax

	_asm mov [esi+4], edx

	_asm xor eax, eax

	_asm pop esi

	_asm pop edi

	_asm pop ebx

	_asm ret

	}

/** 64 bit atomic add, relaxed ordering.

	Atomically performs the following operation:

		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be added

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_add_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_add_ord64

	}

/** 64 bit atomic add, acquire semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be added

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_add_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_add_ord64

	}

/** 64 bit atomic add, release semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be added

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_add_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_add_ord64

	}

/** 64 bit atomic add, full barrier semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be added

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_add_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm push ebx

	_asm push edi

	_asm mov edi, [esp+12]				// edi = a

	_asm mov eax, [edi]					// edx:eax = oldv

	_asm mov edx, [edi+4]

	_asm retry:

	_asm mov ebx, eax

	_asm mov ecx, edx

	_asm add ebx, [esp+16]				// ecx:ebx = oldv + v

	_asm adc ecx, [esp+20]

	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv+v, ZF=1 else edx:eax=*a, ZF=0

	_asm jne short retry

	_asm pop edi

	_asm pop ebx

	_asm ret

	}

/** 64 bit atomic bitwise logical AND, relaxed ordering.

	Atomically performs the following operation:

		oldv = *a; *a = oldv & v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be ANDed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_and_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_and_ord64

	}

/** 64 bit atomic bitwise logical AND, acquire semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv & v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be ANDed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_and_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_and_ord64

	}

/** 64 bit atomic bitwise logical AND, release semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv & v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be ANDed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_and_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_and_ord64

	}

/** 64 bit atomic bitwise logical AND, full barrier semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv & v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be ANDed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_and_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm push ebx

	_asm push edi

	_asm mov edi, [esp+12]				// edi = a

	_asm mov eax, [edi]					// edx:eax = oldv

	_asm mov edx, [edi+4]

	_asm retry:

	_asm mov ebx, eax

	_asm mov ecx, edx

	_asm and ebx, [esp+16]				// ecx:ebx = oldv & v

	_asm and ecx, [esp+20]

	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv&v, ZF=1 else edx:eax=*a, ZF=0

	_asm jne short retry

	_asm pop edi

	_asm pop ebx

	_asm ret

	}

/** 64 bit atomic bitwise logical inclusive OR, relaxed ordering.

	Atomically performs the following operation:

		oldv = *a; *a = oldv | v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be ORed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_ior_ord64

	}

/** 64 bit atomic bitwise logical inclusive OR, acquire semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv | v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be ORed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_ior_ord64

	}

/** 64 bit atomic bitwise logical inclusive OR, release semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv | v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be ORed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_ior_ord64

	}

/** 64 bit atomic bitwise logical inclusive OR, full barrier semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv | v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be ORed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm push ebx

	_asm push edi

	_asm mov edi, [esp+12]				// edi = a

	_asm mov eax, [edi]					// edx:eax = oldv

	_asm mov edx, [edi+4]

	_asm retry:

	_asm mov ebx, eax

	_asm mov ecx, edx

	_asm or ebx, [esp+16]				// ecx:ebx = oldv | v

	_asm or ecx, [esp+20]

	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv|v, ZF=1 else edx:eax=*a, ZF=0

	_asm jne short retry

	_asm pop edi

	_asm pop ebx

	_asm ret

	}

/** 64 bit atomic bitwise logical exclusive OR, relaxed ordering.

	Atomically performs the following operation:

		oldv = *a; *a = oldv ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be XORed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_xor_ord64

	}

/** 64 bit atomic bitwise logical exclusive OR, acquire semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be XORed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_xor_ord64

	}

/** 64 bit atomic bitwise logical exclusive OR, release semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be XORed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_xor_ord64

	}

/** 64 bit atomic bitwise logical exclusive OR, full barrier semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be XORed with *a

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	_asm push ebx

	_asm push edi

	_asm mov edi, [esp+12]				// edi = a

	_asm mov eax, [edi]					// edx:eax = oldv

	_asm mov edx, [edi+4]

	_asm retry:

	_asm mov ebx, eax

	_asm mov ecx, edx

	_asm xor ebx, [esp+16]				// ecx:ebx = oldv ^ v

	_asm xor ecx, [esp+20]

	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv^v, ZF=1 else edx:eax=*a, ZF=0

	_asm jne short retry

	_asm pop edi

	_asm pop ebx

	_asm ret

	}

/** 64 bit atomic bitwise universal function, relaxed ordering.

	Atomically performs the following operation:

		oldv = *a; *a = (oldv & u) ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	u	The value to be ANDed with *a

	@param	v	The value to be XORed with (*a&u)

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_rlx64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_axo_ord64

	}

/** 64 bit atomic bitwise universal function, acquire semantics.

	Atomically performs the following operation:

		oldv = *a; *a = (oldv & u) ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	u	The value to be ANDed with *a

	@param	v	The value to be XORed with (*a&u)

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_acq64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_axo_ord64

	}

/** 64 bit atomic bitwise universal function, release semantics.

	Atomically performs the following operation:

		oldv = *a; *a = (oldv & u) ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	u	The value to be ANDed with *a

	@param	v	The value to be XORed with (*a&u)

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_rel64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_axo_ord64

	}

/** 64 bit atomic bitwise universal function, release semantics.

	Atomically performs the following operation:

		oldv = *a; *a = (oldv & u) ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	u	The value to be ANDed with *a

	@param	v	The value to be XORed with (*a&u)

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_ord64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)

	{

	_asm push ebx

	_asm push edi

	_asm mov edi, [esp+12]				// edi = a

	_asm mov eax, [edi]					// edx:eax = oldv

	_asm mov edx, [edi+4]

	_asm retry:

	_asm mov ebx, eax

	_asm mov ecx, edx

	_asm and ebx, [esp+16]				// ecx:ebx = oldv & u

	_asm and ecx, [esp+20]

	_asm xor ebx, [esp+24]				// ecx:ebx = (oldv & u) ^ v

	_asm xor ecx, [esp+28]

	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=(oldv&u)^v, ZF=1 else edx:eax=*a, ZF=0

	_asm jne short retry

	_asm pop edi

	_asm pop ebx

	_asm ret

	}

/** 64 bit threshold and add, unsigned, relaxed ordering.

	Atomically performs the following operation:

		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned

	@param	t	The threshold to compare *a to (unsigned compare)

	@param	u	The value to be added to *a if it is originally >= t

	@param	u	The value to be added to *a if it is originally < t

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_rlx64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_tau_ord64

	}

/** 64 bit threshold and add, unsigned, acquire semantics.

	Atomically performs the following operation:

		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned

	@param	t	The threshold to compare *a to (unsigned compare)

	@param	u	The value to be added to *a if it is originally >= t

	@param	u	The value to be added to *a if it is originally < t

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_acq64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_tau_ord64

	}

/** 64 bit threshold and add, unsigned, release semantics.

	Atomically performs the following operation:

		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned

	@param	t	The threshold to compare *a to (unsigned compare)

	@param	u	The value to be added to *a if it is originally >= t

	@param	u	The value to be added to *a if it is originally < t

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_rel64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)

	{

	_asm jmp __e32_atomic_tau_ord64

	}

/** 64 bit threshold and add, unsigned, full barrier semantics.

	Atomically performs the following operation:

		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned

	@param	t	The threshold to compare *a to (unsigned compare)

	@param	u	The value to be added to *a if it is originally >= t

	@param	u	The value to be added to *a if it is originally < t

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_ord64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)

	{

	_asm push ebx

	_asm push edi

	_asm mov edi, [esp+12]				// edi = a

	_asm mov eax, [edi]					// edx:eax = oldv

	_asm mov edx, [edi+4]

	_asm retry:

	_asm mov ebx, edx

	_asm cmp eax, [esp+16]				// eax - t.low, CF=borrow

	_asm sbb ebx, [esp+20]				// CF = borrow from (oldv - t)

	_asm jnc short use_u				// no borrow means oldv>=t so use u

	_asm mov ebx, [esp+32]				// ecx:ebx = v

	_asm mov ecx, [esp+36]

	_asm jmp short use_v

	_asm use_u:

	_asm mov ebx, [esp+24]				// ecx:ebx = u

	_asm mov ecx, [esp+28]

	_asm use_v:

	_asm add ebx, eax					// ecx:ebx = oldv + u or v

	_asm adc ecx, edx

	_asm __LOCK__ cmpxchg8b [edi]

	_asm jne short retry

	_asm pop edi

	_asm pop ebx

	_asm ret

	}

/** 64 bit threshold and add, signed, relaxed ordering.

	Atomically performs the following operation:

		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned

	@param	t	The threshold to compare *a to (signed compare)

	@param	u	The value to be added to *a if it is originally >= t

	@param	u	The value to be added to *a if it is originally < t

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TInt64	__e32_atomic_tas_rlx64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/)

	{

	_asm jmp __e32_atomic_tas_ord64

	}

/** 64 bit threshold and add, signed, acquire semantics.

	Atomically performs the following operation:

		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned

	@param	t	The threshold to compare *a to (signed compare)

	@param	u	The value to be added to *a if it is originally >= t

	@param	u	The value to be added to *a if it is originally < t

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TInt64	__e32_atomic_tas_acq64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/)

	{

	_asm jmp __e32_atomic_tas_ord64

	}

/** 64 bit threshold and add, signed, release semantics.

	Atomically performs the following operation:

		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned

	@param	t	The threshold to compare *a to (signed compare)

	@param	u	The value to be added to *a if it is originally >= t

	@param	u	The value to be added to *a if it is originally < t