sl@0: // Copyright (c) 1997-2009 Nokia Corporation and/or its subsidiary(-ies).
sl@0: // All rights reserved.
sl@0: // This component and the accompanying materials are made available
sl@0: // under the terms of the License "Eclipse Public License v1.0"
sl@0: // which accompanies this distribution, and is available
sl@0: // at the URL "http://www.eclipse.org/legal/epl-v10.html".
sl@0: //
sl@0: // Initial Contributors:
sl@0: // Nokia Corporation - initial contribution.
sl@0: //
sl@0: // Contributors:
sl@0: //
sl@0: // Description:
sl@0: // e32\klib\arm\cumem.cia
sl@0: // 
sl@0: //
sl@0: 
sl@0: #include <kernel/klib.h>
sl@0: #include <e32cia.h>
sl@0: #include <arm.h>
sl@0: #if defined(__REPLACE_GENERIC_UTILS)
sl@0: #include "replacement_utils.h"
sl@0: #endif
sl@0: 
sl@0: extern "C" {
sl@0: 
sl@0: #ifdef _DEBUG
sl@0: #define CUMEM_FAULT(cc, reason) asm("mov"#cc" r0, #%a0 " : : "i" (reason)); \
sl@0:                                 asm("b"#cc" " CSM_ZN2KL5PanicENS_13TKernLibPanicE)	
sl@0: #endif
sl@0: 
sl@0: 
sl@0: __NAKED__ void kumemget_no_paging_assert(TAny* /*aKernAddr*/, const TAny* /*aAddr*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	asm("mrs r3, spsr ");			// r3=spsr_svc
sl@0: 	asm("tst r3, #0x0f ");			// test for user mode
sl@0: 	asm("bne memcpy ");				// if not, just do memcpy
sl@0: #ifndef USE_REPLACEMENT_UMEMGET
sl@0: 	asm("b umemget_no_paging_assert");
sl@0: #else
sl@0: 	asm("b umemget");
sl@0: #endif
sl@0: 	}
sl@0: 
sl@0: 
sl@0: #ifndef USE_REPLACEMENT_UMEMGET
sl@0: 	
sl@0: #ifdef __CPU_ARMV6
sl@0: // Conditional returns are not predicted on ARMv6
sl@0: __NAKED__ void dummy_umemget32_exit()
sl@0: 	{
sl@0: 	asm("_umemget32_exit: ");
sl@0: 	asm("ldmfd sp!, {r4, pc} ");	
sl@0: 	}
sl@0: #define UMEMGET32_EXIT(cc)	asm("b"#cc" _umemget32_exit")
sl@0: #else
sl@0: #define UMEMGET32_EXIT(cc)	asm("ldm"#cc"fd sp!, {r4, pc}")
sl@0: #endif
sl@0: 
sl@0: 	
sl@0: EXPORT_C __NAKED__ void kumemget32(TAny* /*aKernAddr*/, const TAny* /*aAddr*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	asm("mrs r3, spsr ");			// r3=spsr_svc
sl@0: 	asm("tst r3, #0x0f ");			// test for user mode
sl@0: 	asm("bne wordmove ");			// if not, just do wordmove
sl@0: 	// otherwise fall through to umemget32
sl@0: 	}
sl@0: 
sl@0: 
sl@0: EXPORT_C __NAKED__ void umemget32(TAny* /*aKernAddr*/, const TAny* /*aUserAddr*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	ASM_ASSERT_PAGING_SAFE
sl@0: 
sl@0: #ifdef __USER_MEMORY_GUARDS_ENABLED__
sl@0: 	// Wrap the workings of this function in an internal call, so we can save/restore UMG state
sl@0: 	asm("stmfd sp!, {r11, lr} ");
sl@0: 	asm("subs r12, r2, #1");
sl@0: 	asm("ldrhsb r11, [r0]");				// test access to first byte of kernel memory
sl@0: 	asm("ldrhsb r11, [r0,r12]");			// test access to last byte of kernel memory
sl@0: 	USER_MEMORY_GUARD_OFF(,r11,r12);		// leaves UMG mode in r11
sl@0: 	asm("bl 0f");							// call to label below
sl@0: 	USER_MEMORY_GUARD_RESTORE(r11,r12);
sl@0: 	asm("ldmfd sp!, {r11, pc} ");	
sl@0: 
sl@0: 	asm("0:");
sl@0: #endif
sl@0: 
sl@0: #ifdef _DEBUG
sl@0: 	asm("tst r2, #3 ");						// check length is a whole number of words
sl@0: 	CUMEM_FAULT(ne, KL::EWordMoveLengthNotMultipleOf4);
sl@0: #endif
sl@0: 
sl@0: 	PLD(1);
sl@0: 	asm("_umemget_word_aligned: ");	
sl@0: 	asm("stmfd sp!, {r4, lr} ");
sl@0: 	asm("subs ip, r2, #32 ");
sl@0: 	asm("blo _umemget32_small_copy ");
sl@0: 	PLD_ioff(1, 32);
sl@0: 	asm("beq _umemget32_32_byte_case ");	// 32 byte case is common - don't bother to align
sl@0: 
sl@0: 	asm("rsb lr, r0, #32 ");				// align destination: 0 - 28 byte copy
sl@0: 	asm("movs lr, lr, lsl #27 ");
sl@0: 	asm("beq _umemget32_large_copy ");
sl@0: 	asm("sub r2, r2, lr, lsr #27 ");
sl@0: 	asm("msr cpsr_f, lr ");					// put length bits 4, 3, 2 into N, Z, C
sl@0: 	asm("ldrmit r3, [r1], #4 ");
sl@0: 	asm("ldrmit r4, [r1], #4 ");
sl@0: 	asm("ldrmit ip, [r1], #4 ");
sl@0: 	asm("ldrmit lr, [r1], #4 ");
sl@0: 	asm("stmmiia r0!, {r3, r4, ip, lr} ");
sl@0: 	asm("ldreqt r3, [r1], #4 ");
sl@0: 	asm("ldreqt r4, [r1], #4 ");
sl@0: 	asm("ldrcst ip, [r1], #4 ");
sl@0: 	asm("stmeqia r0!, {r3, r4} ");
sl@0: 	asm("strcs ip, [r0], #4 ");
sl@0: 	asm("subs ip, r2, #32 ");
sl@0: 	asm("blo _umemget32_small_copy ");
sl@0: 
sl@0: 	asm("_umemget32_large_copy: ");			// copy 32 byte blocks
sl@0: 	PLD_ioff(1, 64);
sl@0: 	asm("_umemget32_32_byte_case: ");
sl@0: 	asm("ldrt r2, [r1], #4 ");
sl@0: 	asm("ldrt r3, [r1], #4 ");
sl@0: 	asm("ldrt r4, [r1], #4 ");
sl@0: 	asm("ldrt lr, [r1], #4 ");
sl@0: 	asm("subs ip, ip, #32 ");
sl@0: 	asm("stmia r0!, {r2, r3, r4, lr} ");
sl@0: 	asm("ldrt r2, [r1], #4 ");
sl@0: 	asm("ldrt r3, [r1], #4 ");
sl@0: 	asm("ldrt r4, [r1], #4 ");
sl@0: 	asm("ldrt lr, [r1], #4 ");
sl@0: 	asm("stmia r0!, {r2, r3, r4, lr} ");
sl@0: 	asm("bhs _umemget32_large_copy ");
sl@0: 		
sl@0: 	asm("_umemget32_small_copy: ");			// 0 - 31 byte copy, length in ip bits 0-4
sl@0: 	asm("movs r2, ip, lsl #27 ");
sl@0: 	UMEMGET32_EXIT(eq);
sl@0: 	asm("msr cpsr_f, r2 ");					// put length bits 4, 3, 2 into N, Z, C
sl@0: 	asm("ldrmit r3, [r1], #4 ");
sl@0: 	asm("ldrmit r4, [r1], #4 ");
sl@0: 	asm("ldrmit ip, [r1], #4 ");
sl@0: 	asm("ldrmit lr, [r1], #4 ");
sl@0: 	asm("stmmiia r0!, {r3, r4, ip, lr} ");
sl@0: 	asm("ldreqt r3, [r1], #4 ");
sl@0: 	asm("ldreqt r4, [r1], #4 ");
sl@0: 	asm("ldrcst ip, [r1], #4 ");
sl@0: 	asm("stmeqia r0!, {r3, r4} ");
sl@0: 	asm("strcs ip, [r0], #4 ");	
sl@0: 	asm("movs r2, r2, lsl #3 ");
sl@0: 	UMEMGET32_EXIT(eq);
sl@0: 	asm("msr cpsr_f, r2 ");					// put length bits 1, 0 into N, Z	
sl@0: 	asm("ldrmibt r3, [r1], #1 ");
sl@0: 	asm("ldrmibt r4, [r1], #1 ");	
sl@0: 	asm("ldreqbt ip, [r1], #1 ");
sl@0: 	asm("strmib r3, [r0], #1 ");
sl@0: 	asm("strmib r4, [r0], #1 ");
sl@0: 	asm("streqb ip, [r0], #1 ");
sl@0: 	asm("ldmfd sp!, {r4, pc} ");	
sl@0: 	}
sl@0: 
sl@0: 
sl@0: EXPORT_C __NAKED__ void kumemget(TAny* /*aKernAddr*/, const TAny* /*aAddr*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	asm("mrs r3, spsr ");			// r3=spsr_svc
sl@0: 	asm("tst r3, #0x0f ");			// test for user mode
sl@0: 	asm("bne memcpy ");				// if not, just do memcpy
sl@0: 	// otherwise fall through to umemget
sl@0: 	}
sl@0: 
sl@0: 
sl@0: EXPORT_C __NAKED__ void umemget(TAny* /*aKernAddr*/, const TAny* /*aUserAddr*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	// Optimised for aligned transfers, as unaligned are very rare in practice
sl@0: 
sl@0: 	ASM_ASSERT_PAGING_SAFE
sl@0: 	asm("umemget_no_paging_assert:");
sl@0: 
sl@0: #ifdef __USER_MEMORY_GUARDS_ENABLED__
sl@0: 	// Wrap the workings of this function in an internal call, so we can save/restore UMG state
sl@0: 	asm("stmfd sp!, {r11, lr} ");
sl@0: 	asm("subs r12, r2, #1");
sl@0: 	asm("ldrhsb r11, [r0]");				// test access to first byte of kernel memory
sl@0: 	asm("ldrhsb r11, [r0,r12]");			// test access to last byte of kernel memory
sl@0: 	USER_MEMORY_GUARD_OFF(,r11,r12);		// leaves UMG mode in r11
sl@0: 	asm("bl 0f");							// call to label below
sl@0: 	USER_MEMORY_GUARD_RESTORE(r11,r12);
sl@0: 	asm("ldmfd sp!, {r11, pc} ");	
sl@0: 
sl@0: 	asm("0:");
sl@0: #endif
sl@0: 
sl@0: 	PLD(1);
sl@0: 	asm("tst r0, #3 ");
sl@0: 	asm("tsteq r1, #3 ");
sl@0: 	asm("beq _umemget_word_aligned ");
sl@0: 	asm("cmp r2, #8 ");
sl@0: 	asm("bhs 1f ");
sl@0: 
sl@0: 	asm("2: ");
sl@0: 	asm("subs r2, r2, #1 ");
sl@0:  	asm("ldrplbt r3, [r1], #1 ");
sl@0: 	asm("strplb r3, [r0], #1 ");
sl@0: 	asm("bgt 2b ");
sl@0: 	__JUMP(,lr);
sl@0: 
sl@0: 	asm("1: ");						// Attempt to word align
sl@0: 	asm("movs r3, r0, lsl #30 ");
sl@0: 	asm("beq 5f ");
sl@0: 	asm("rsbs r3, r3, #0 ");		// 01->c0000000 (MI,VC) 10->80000000 (MI,VS) 11->40000000 (PL,VC)
sl@0: 	asm("sub r2, r2, r3, lsr #30 ");
sl@0: 	asm("ldrmibt r3, [r1], #1 ");
sl@0: 	asm("strmib r3, [r0], #1 ");
sl@0: 	asm("ldrmibt r3, [r1], #1 ");
sl@0: 	asm("strmib r3, [r0], #1 ");
sl@0: 	asm("ldrvcbt r3, [r1], #1 ");
sl@0: 	asm("strvcb r3, [r0], #1 ");	// r0 now word aligned
sl@0: 	asm("5: ");
sl@0: 	asm("movs r3, r1, lsl #31 ");
sl@0: 	asm("bic r1, r1, #3 ");
sl@0: 	asm("bcs 3f ");					// branch if src mod 4 = 2 or 3
sl@0: 	asm("bpl _umemget_word_aligned ");	// branch if src mod 4 = 0
sl@0: 
sl@0: 	asm("4: ");						// src mod 4 = 1
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrget r3, [r1], #4 ");
sl@0: 	asm("ldrget ip, [r1] ");
sl@0: 	asm("movge r3, r3, lsr #8 ");
sl@0: 	asm("orrge r3, r3, ip, lsl #24 ");
sl@0: 	asm("strge r3, [r0], #4 ");
sl@0: 	asm("bgt 4b ");
sl@0: 	asm("add r1, r1, #1 ");
sl@0: 	asm("b umemget_do_end ");
sl@0: 	
sl@0: 	asm("3: ");						
sl@0: 	asm("bmi 5f ");
sl@0: 	asm("2: ");						// src mod 4 = 2
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrget r3, [r1], #4 ");
sl@0: 	asm("ldrget ip, [r1] ");
sl@0: 	asm("movge r3, r3, lsr #16 ");
sl@0: 	asm("orrge r3, r3, ip, lsl #16 ");
sl@0: 	asm("strge r3, [r0], #4 ");
sl@0: 	asm("bgt 2b ");
sl@0: 	asm("add r1, r1, #2 ");
sl@0: 	asm("b umemget_do_end ");
sl@0: 	
sl@0: 	asm("5: ");						// src mod 4 = 3
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrget r3, [r1], #4 ");
sl@0: 	asm("ldrget ip, [r1] ");
sl@0: 	asm("movge r3, r3, lsr #24 ");
sl@0: 	asm("orrge r3, r3, ip, lsl #8 ");
sl@0: 	asm("strge r3, [r0], #4 ");
sl@0: 	asm("bgt 5b ");
sl@0: 	asm("add r1, r1, #3 ");
sl@0: 
sl@0: 	asm("umemget_do_end: ");
sl@0: 	__JUMP(eq,lr);
sl@0: 	asm("adds r2, r2, #2 ");		// -1 if 1 left, 0 if 2 left, +1 if 3 left
sl@0: 	asm("ldrplbt r3, [r1], #1 ");
sl@0: 	asm("strplb r3, [r0], #1 ");
sl@0: 	asm("ldrplbt r3, [r1], #1 ");
sl@0: 	asm("strplb r3, [r0], #1 ");
sl@0: 	asm("ldrnebt r3, [r1], #1 ");
sl@0: 	asm("strneb r3, [r0], #1 ");
sl@0: 	__JUMP(,lr);
sl@0: 	}
sl@0: 
sl@0: #endif  // USE_REPLACEMENT_UMEMGET
sl@0: 
sl@0: __NAKED__ void kumemput_no_paging_assert(TAny* /*aAddr*/, const TAny* /*aKernAddr*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	asm("mrs r3, spsr ");			// r3=spsr_svc
sl@0: 	asm("tst r3, #0x0f ");			// test for user mode
sl@0: 	asm("bne memcpy ");				// if not, just do memcpy
sl@0: #ifndef USE_REPLACEMENT_UMEMPUT
sl@0: 	asm("b umemput_no_paging_assert");
sl@0: #else
sl@0: 	asm("b umemput");
sl@0: #endif
sl@0: 	}
sl@0: 
sl@0: 
sl@0: #ifndef USE_REPLACEMENT_UMEMPUT
sl@0: 	
sl@0: #ifdef __CPU_ARMV6
sl@0: // Conditional returns are not predicted on ARMv6
sl@0: __NAKED__ void dummy_umemput32_exit()
sl@0: 	{
sl@0: 	asm("_umemput32_exit: ");
sl@0: 	asm("ldmfd sp!, {r4, pc} ");	
sl@0: 	}
sl@0: #define UMEMPUT32_EXIT(cc)	asm("b"#cc" _umemput32_exit")
sl@0: #else
sl@0: #define UMEMPUT32_EXIT(cc)	asm("ldm"#cc"fd sp!, {r4, pc}")
sl@0: #endif
sl@0: 
sl@0: 	
sl@0: EXPORT_C __NAKED__ void kumemput32(TAny* /*aAddr*/, const TAny* /*aKernAddr*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	asm("mrs r3, spsr ");			// r3=spsr_svc
sl@0: 	asm("tst r3, #0x0f ");			// test for user mode
sl@0: 	asm("bne wordmove ");			// if not, just do wordmove
sl@0: 	// otherwise fall through to umemput32
sl@0: 	}
sl@0: 
sl@0: 	
sl@0: EXPORT_C __NAKED__ void umemput32(TAny* /*aUserAddr*/, const TAny* /*aKernAddr*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	ASM_ASSERT_DATA_PAGING_SAFE
sl@0: 
sl@0: #ifdef __USER_MEMORY_GUARDS_ENABLED__
sl@0: 	// Wrap the workings of this function in an internal call, so we can save/restore UMG state
sl@0: 	asm("stmfd sp!, {r11, lr} ");
sl@0: 	asm("subs r12, r2, #1");
sl@0: 	asm("ldrhsb r11, [r1]");				// test access to first byte of kernel memory
sl@0: 	asm("ldrhsb r11, [r1,r12]");			// test access to last byte of kernel memory
sl@0: 	USER_MEMORY_GUARD_OFF(,r11,r12);		// leaves UMG mode in r11
sl@0: 	asm("bl 0f");							// call to label below
sl@0: 	USER_MEMORY_GUARD_RESTORE(r11,r12);
sl@0: 	asm("ldmfd sp!, {r11, pc} ");	
sl@0: 
sl@0: 	asm("0:");
sl@0: #endif
sl@0: 
sl@0: #ifdef _DEBUG
sl@0: 	asm("tst r2, #3 ");						// check length is a whole number of words
sl@0: 	CUMEM_FAULT(ne, KL::EWordMoveLengthNotMultipleOf4);
sl@0: #endif
sl@0: 
sl@0: 	PLD(1);
sl@0: 	asm("cmp r2, #4 ");						// special case for 4 byte copy which is common
sl@0: 	asm("ldrhs r3, [r1], #4 ");
sl@0: 	asm("subhs r2, r2, #4 ");
sl@0: 	asm("strhst r3, [r0], #4 ");
sl@0: 	__JUMP(ls,lr);
sl@0: 	
sl@0: 	asm("_umemput_word_aligned: ");
sl@0: 	asm("stmfd sp!, {r4, lr} ");
sl@0: 	asm("subs r2, r2, #32 ");
sl@0: 	asm("bhs _umemput32_align_source ");
sl@0: 
sl@0: 	asm("_umemput32_small_copy: ");			// copy 1 - 31 bytes
sl@0: 	asm("mov r2, r2, lsl #27 ");
sl@0: 	asm("msr cpsr_f, r2 ");					// put length bits 4, 3, 2 into N, Z, C
sl@0: 	asm("ldmmiia r1!, {r3, r4, ip, lr} ");
sl@0: 	asm("strmit r3, [r0], #4 ");
sl@0: 	asm("strmit r4, [r0], #4 ");
sl@0: 	asm("ldmeqia r1!, {r3, r4} ");
sl@0: 	asm("strmit ip, [r0], #4 ");
sl@0: 	asm("ldrcs ip, [r1], #4 ");
sl@0: 	asm("strmit lr, [r0], #4 ");
sl@0: 	asm("streqt r3, [r0], #4 ");
sl@0: 	asm("streqt r4, [r0], #4 ");
sl@0: 	asm("strcst ip, [r0], #4 ");
sl@0: 	asm("movs r2, r2, lsl #3 ");
sl@0: 	UMEMPUT32_EXIT(eq);
sl@0: 	asm("msr cpsr_f, r2 ");					// put length bits 1, 0 into N, Z
sl@0: 	asm("ldrmih r3, [r1], #2 ");
sl@0: 	asm("ldreqb r4, [r1], #1 ");
sl@0: 	asm("strmibt r3, [r0], #1 ");
sl@0: 	asm("movmi r3, r3, lsr #8 ");
sl@0: 	asm("strmibt r3, [r0], #1 ");
sl@0: 	asm("streqbt r4, [r0], #1 ");
sl@0: 	asm("ldmfd sp!, {r4, pc} ");
sl@0: 	
sl@0: 	asm("_umemput32_align_source: ");
sl@0: 	PLD_ioff(1, 32);
sl@0: 	asm("cmp r2, #32 ");
sl@0: 	asm("bls _umemput32_large_copy ");		// don't bother if length <= 64
sl@0: 	asm("rsb ip, r1, #32 ");
sl@0: 	asm("movs ip, ip, lsl #27 ");
sl@0: 	asm("beq _umemput32_large_copy ");
sl@0: 	asm("msr cpsr_f, ip ");					// put length bits 4, 3, 2 into N, Z, C
sl@0: 	asm("sub r2, r2, ip, lsr #27 ");
sl@0: 	asm("ldmmiia r1!, {r3, r4, ip, lr} ");
sl@0: 	asm("strmit r3, [r0], #4 ");
sl@0: 	asm("strmit r4, [r0], #4 ");
sl@0: 	asm("ldmeqia r1!, {r3, r4} ");
sl@0: 	asm("strmit ip, [r0], #4 ");
sl@0: 	asm("ldrcs ip, [r1], #4 ");
sl@0: 	asm("strmit lr, [r0], #4 ");
sl@0: 	asm("streqt r3, [r0], #4 ");
sl@0: 	asm("streqt r4, [r0], #4 ");
sl@0: 	asm("strcst ip, [r0], #4 ");
sl@0: 
sl@0: 	asm("_umemput32_large_copy: ");			// copy 32 byte blocks
sl@0: 	PLD_ioff(1, 64);
sl@0: 	asm("ldmia r1!, {r3, r4, ip, lr} ");
sl@0: 	asm("strt r3, [r0], #4 ");
sl@0: 	asm("strt r4, [r0], #4 ");
sl@0: 	asm("strt ip, [r0], #4 ");
sl@0: 	asm("strt lr, [r0], #4 ");
sl@0: 	asm("ldmia r1!, {r3, r4, ip, lr} ");
sl@0: 	asm("strt r3, [r0], #4 ");
sl@0: 	asm("strt r4, [r0], #4 ");
sl@0: 	asm("strt ip, [r0], #4 ");
sl@0: 	asm("strt lr, [r0], #4 ");
sl@0: 	asm("subs r2, r2, #32 ");
sl@0: 	asm("bhs _umemput32_large_copy ");
sl@0: 	asm("adds r2, r2, #32 ");
sl@0: 	asm("bne _umemput32_small_copy ");
sl@0: 	asm("ldmfd sp!, {r4, pc} ");
sl@0: 	}
sl@0: 
sl@0: 
sl@0: __NAKED__ void uumemcpy32(TAny* /*aUserDst*/, const TAny* /*aUserSrc*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	ASM_ASSERT_PAGING_SAFE
sl@0: 
sl@0: #ifdef __USER_MEMORY_GUARDS_ENABLED__
sl@0: 	// Wrap the workings of this function in an internal call, so we can save/restore UMG state
sl@0: 	asm("stmfd sp!, {r11, lr} ");
sl@0: 	USER_MEMORY_GUARD_OFF(,r11,r12);		// leaves UMG mode in r11
sl@0: 	asm("bl 0f");							// call to label below
sl@0: 	USER_MEMORY_GUARD_RESTORE(r11,r12);
sl@0: 	asm("ldmfd sp!, {r11, pc} ");	
sl@0: 
sl@0: 	asm("0:");
sl@0: #endif
sl@0: 
sl@0: 	asm("1: ");
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrplt r3, [r1], #4 ");
sl@0: 	asm("strplt r3, [r0], #4 ");
sl@0: 	asm("bpl 1b ");
sl@0: 	__JUMP(,lr);
sl@0: 	}
sl@0: 
sl@0: 
sl@0: __NAKED__ void uumemcpy(TAny* /*aUserDst*/, const TAny* /*aUserSrc*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	ASM_ASSERT_PAGING_SAFE
sl@0: 
sl@0: #ifdef __USER_MEMORY_GUARDS_ENABLED__
sl@0: 	// Wrap the workings of this function in an internal call, so we can save/restore UMG state
sl@0: 	asm("stmfd sp!, {r11, lr} ");
sl@0: 	USER_MEMORY_GUARD_OFF(,r11,r12);		// leaves UMG mode in r11
sl@0: 	asm("bl 0f");							// call to label below
sl@0: 	USER_MEMORY_GUARD_RESTORE(r11,r12);
sl@0: 	asm("ldmfd sp!, {r11, pc} ");	
sl@0: 
sl@0: 	asm("0:");
sl@0: #endif
sl@0: 
sl@0: 	asm("cmp r2, #8 ");
sl@0: 	asm("bcs 1f ");
sl@0: 	asm("2: ");
sl@0: 	asm("subs r2, r2, #1 ");
sl@0: 	asm("ldrplbt r3, [r1], #1 ");
sl@0: 	asm("strplbt r3, [r0], #1 ");
sl@0: 	asm("bgt 2b ");
sl@0: 	__JUMP(,lr);
sl@0: 	asm("1: ");
sl@0: 	asm("movs r3, r0, lsl #30 ");
sl@0: 	asm("beq 5f ");
sl@0: 	asm("rsbs r3, r3, #0 ");		// 01->c0000000 (MI,VC) 10->80000000 (MI,VS) 11->40000000 (PL,VC)
sl@0: 	asm("sub r2, r2, r3, lsr #30 ");
sl@0: 	asm("ldrmibt r3, [r1], #1 ");
sl@0: 	asm("strmibt r3, [r0], #1 ");
sl@0: 	asm("ldrmibt r3, [r1], #1 ");
sl@0: 	asm("strmibt r3, [r0], #1 ");
sl@0: 	asm("ldrvcbt r3, [r1], #1 ");
sl@0: 	asm("strvcbt r3, [r0], #1 ");	// r0 now word aligned
sl@0: 	asm("5: ");
sl@0: 	asm("movs r3, r1, lsl #31 ");
sl@0: 	asm("bic r1, r1, #3 ");
sl@0: 	asm("bcs 3f ");					// branch if src mod 4 = 2 or 3
sl@0: 	asm("bmi 4f ");					// branch if src mod 4 = 1
sl@0: 	asm("2: ");
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrget r3, [r1], #4 ");
sl@0: 	asm("strget r3, [r0], #4 ");
sl@0: 	asm("bgt 2b ");
sl@0: 	asm("uumemcpy_do_end: ");
sl@0: 	__JUMP(eq,lr);
sl@0: 	asm("adds r2, r2, #2 ");		// -1 if 1 left, 0 if 2 left, +1 if 3 left
sl@0: 	asm("ldrplbt r3, [r1], #1 ");
sl@0: 	asm("strplbt r3, [r0], #1 ");
sl@0: 	asm("ldrplbt r3, [r1], #1 ");
sl@0: 	asm("strplbt r3, [r0], #1 ");
sl@0: 	asm("ldrnebt r3, [r1], #1 ");
sl@0: 	asm("strnebt r3, [r0], #1 ");
sl@0: 	__JUMP(,lr);
sl@0: 	asm("3: ");						// get here if src mod 4 = 2 or 3
sl@0: 	asm("bmi 5f ");					// branch if 3
sl@0: 	asm("2: ");
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrget r3, [r1], #4 ");
sl@0: 	asm("ldrget ip, [r1] ");
sl@0: 	asm("movge r3, r3, lsr #16 ");
sl@0: 	asm("orrge r3, r3, ip, lsl #16 ");
sl@0: 	asm("strget r3, [r0], #4 ");
sl@0: 	asm("bgt 2b ");
sl@0: 	asm("add r1, r1, #2 ");
sl@0: 	asm("b uumemcpy_do_end ");
sl@0: 	asm("5: ");
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrget r3, [r1], #4 ");
sl@0: 	asm("ldrget ip, [r1] ");
sl@0: 	asm("movge r3, r3, lsr #24 ");
sl@0: 	asm("orrge r3, r3, ip, lsl #8 ");
sl@0: 	asm("strget r3, [r0], #4 ");
sl@0: 	asm("bgt 5b ");
sl@0: 	asm("add r1, r1, #3 ");
sl@0: 	asm("b uumemcpy_do_end ");
sl@0: 	asm("4: ");
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrget r3, [r1], #4 ");
sl@0: 	asm("ldrget ip, [r1] ");
sl@0: 	asm("movge r3, r3, lsr #8 ");
sl@0: 	asm("orrge r3, r3, ip, lsl #24 ");
sl@0: 	asm("strget r3, [r0], #4 ");
sl@0: 	asm("bgt 4b ");
sl@0: 	asm("add r1, r1, #1 ");
sl@0: 	asm("b uumemcpy_do_end ");
sl@0: 	}
sl@0: 
sl@0: 
sl@0: EXPORT_C __NAKED__ void kumemput(TAny* /*aAddr*/, const TAny* /*aKernAddr*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	asm("mrs r3, spsr ");			// r3=spsr_svc
sl@0: 	asm("tst r3, #0x0f ");			// test for user mode
sl@0: 	asm("bne memcpy ");				// if not, just do memcpy
sl@0: 	// otherwise fall through to umemput
sl@0: 	}
sl@0: 
sl@0: 
sl@0: EXPORT_C __NAKED__ void umemput(TAny* /*aUserAddr*/, const TAny* /*aKernAddr*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	// Optimised for word-aligned transfers, as unaligned are very rare in practice	
sl@0: 
sl@0: 	ASM_ASSERT_DATA_PAGING_SAFE
sl@0: 	asm("umemput_no_paging_assert:");
sl@0: 
sl@0: #ifdef __USER_MEMORY_GUARDS_ENABLED__
sl@0: 	// Wrap the workings of this function in an internal call, so we can save/restore UMG state
sl@0: 	asm("stmfd sp!, {r11, lr} ");
sl@0: 	asm("subs r12, r2, #1");
sl@0: 	asm("ldrhsb r11, [r1]");				// test access to first byte of kernel memory
sl@0: 	asm("ldrhsb r11, [r1,r12]");			// test access to last byte of kernel memory
sl@0: 	USER_MEMORY_GUARD_OFF(,r11,r12);		// leaves UMG mode in r11
sl@0: 	asm("bl 0f");							// call to label below
sl@0: 	USER_MEMORY_GUARD_RESTORE(r11,r12);
sl@0: 	asm("ldmfd sp!, {r11, pc} ");	
sl@0: 
sl@0: 	asm("0:");
sl@0: #endif
sl@0: 
sl@0: 	PLD(1);
sl@0: 	asm("tst r0, #3 ");
sl@0: 	asm("tsteq r1, #3 ");
sl@0: 	asm("beq _umemput_word_aligned ");
sl@0: 	
sl@0: 	asm("cmp r2, #8 ");
sl@0: 	asm("bcs 1f ");
sl@0: 	asm("2: ");						// Copy 0 - 7 bytes
sl@0: 	asm("subs r2, r2, #1 ");
sl@0: 	asm("ldrplb r3, [r1], #1 ");
sl@0: 	asm("strplbt r3, [r0], #1 ");
sl@0: 	asm("bgt 2b ");
sl@0: 	__JUMP(,lr);
sl@0: 	
sl@0: 	asm("1: ");						// Word-align dest
sl@0: 	asm("movs r3, r0, lsl #30 ");
sl@0: 	asm("beq 5f ");
sl@0: 	asm("rsbs r3, r3, #0 ");		// 01->c0000000 (MI,VC) 10->80000000 (MI,VS) 11->40000000 (PL,VC)
sl@0: 	asm("sub r2, r2, r3, lsr #30 ");
sl@0: 	asm("ldrmib r3, [r1], #1 ");
sl@0: 	asm("strmibt r3, [r0], #1 ");
sl@0: 	asm("ldrmib r3, [r1], #1 ");
sl@0: 	asm("strmibt r3, [r0], #1 ");
sl@0: 	asm("ldrvcb r3, [r1], #1 ");
sl@0: 	asm("strvcbt r3, [r0], #1 ");	// r0 now word aligned
sl@0: 	asm("5: ");
sl@0: 	asm("movs r3, r1, lsl #31 ");
sl@0: 	asm("bic r1, r1, #3 ");
sl@0: 	asm("bcs 3f ");					// branch if src mod 4 = 2 or 3
sl@0: 	asm("bpl _umemput_word_aligned "); // branch if src mod 4 = 0
sl@0: 
sl@0: 	asm("4: ");						// get here if src mod 4 = 1
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrge r3, [r1], #4 ");
sl@0: 	asm("ldrge ip, [r1] ");
sl@0: 	asm("movge r3, r3, lsr #8 ");
sl@0: 	asm("orrge r3, r3, ip, lsl #24 ");
sl@0: 	asm("strget r3, [r0], #4 ");
sl@0: 	asm("bgt 4b ");
sl@0: 	asm("add r1, r1, #1 ");
sl@0: 	asm("b _umemput_do_end ");
sl@0: 	
sl@0: 	asm("3: ");						// get here if src mod 4 = 2 or 3
sl@0: 	asm("bmi 5f ");					// branch if 3
sl@0: 	asm("2: ");
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrge r3, [r1], #4 ");
sl@0: 	asm("ldrge ip, [r1] ");
sl@0: 	asm("movge r3, r3, lsr #16 ");
sl@0: 	asm("orrge r3, r3, ip, lsl #16 ");
sl@0: 	asm("strget r3, [r0], #4 ");
sl@0: 	asm("bgt 2b ");
sl@0: 	asm("add r1, r1, #2 ");
sl@0: 	asm("b _umemput_do_end ");
sl@0: 	
sl@0: 	asm("5: ");						// get here if src mod 4 = 3
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("ldrge r3, [r1], #4 ");
sl@0: 	asm("ldrge ip, [r1] ");
sl@0: 	asm("movge r3, r3, lsr #24 ");
sl@0: 	asm("orrge r3, r3, ip, lsl #8 ");
sl@0: 	asm("strget r3, [r0], #4 ");
sl@0: 	asm("bgt 5b ");
sl@0: 	asm("add r1, r1, #3 ");
sl@0: 
sl@0: 	asm("_umemput_do_end: ");		// z set if done, else r2 == length remaining - 4
sl@0: 	__JUMP(eq,lr);
sl@0: 	asm("adds r2, r2, #2 ");		// r2 = -1 if 1 left, 0 if 2 left, +1 if 3 left
sl@0: 	asm("ldrplb r3, [r1], #1 ");
sl@0: 	asm("strplbt r3, [r0], #1 ");
sl@0: 	asm("ldrplb r3, [r1], #1 ");
sl@0: 	asm("strplbt r3, [r0], #1 ");
sl@0: 	asm("ldrneb r3, [r1], #1 ");
sl@0: 	asm("strnebt r3, [r0], #1 ");
sl@0: 	__JUMP(,lr);
sl@0: 	}
sl@0: 
sl@0: #endif  // USE_REPLACEMENT_UMEMPUT
sl@0: 	
sl@0: 	
sl@0: EXPORT_C __NAKED__ void kumemset(TAny* /*aAddr*/, const TUint8 /*aValue*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	asm("mrs r3, spsr ");			// r3=spsr_svc
sl@0: 	asm("tst r3, #0x0f ");			// test for user mode
sl@0: 	asm("bne memset ");				// if not, just do memset
sl@0: 	// otherwise fall through to umemset
sl@0: 	}
sl@0: 
sl@0: 
sl@0: EXPORT_C __NAKED__ void umemset(TAny* /*aUserAddr*/, const TUint8 /*aValue*/, TInt /*aLength*/)
sl@0: 	{
sl@0: 	ASM_ASSERT_DATA_PAGING_SAFE
sl@0: 
sl@0: #ifdef __USER_MEMORY_GUARDS_ENABLED__
sl@0: 	// Wrap the workings of this function in an internal call, so we can save/restore UMG state
sl@0: 	asm("stmfd sp!, {r11, lr} ");
sl@0: 	USER_MEMORY_GUARD_OFF(,r11,r12);		// leaves UMG mode in r11
sl@0: 	asm("bl 0f");							// call to label below
sl@0: 	USER_MEMORY_GUARD_RESTORE(r11,r12);
sl@0: 	asm("ldmfd sp!, {r11, pc} ");	
sl@0: 
sl@0: 	asm("0:");
sl@0: #endif
sl@0: 
sl@0: 	asm("cmp r2, #7 ");
sl@0: 	asm("bhi 2f ");
sl@0: 	asm("1: ");
sl@0: 	asm("subs r2, r2, #1 ");
sl@0: 	asm("strplbt r1, [r0], #1 ");
sl@0: 	asm("bgt 1b ");
sl@0: 	__JUMP(,lr);	
sl@0: 	asm("2: ");
sl@0: 	asm("and r1, r1, #0xff ");
sl@0: 	asm("orr r1, r1, r1, lsl #8 ");
sl@0: 	asm("orr r1, r1, r1, lsl #16 ");
sl@0: 	asm("movs r3, r0, lsl #30 ");
sl@0: 	asm("beq 3f ");
sl@0: 	asm("rsbs r3, r3, #0 ");		// 01->c0000000 (MI,VC) 10->80000000 (MI,VS) 11->40000000 (PL,VC)
sl@0: 	asm("strmibt r1, [r0], #1 ");	// if 01 or 10 do 2 byte stores
sl@0: 	asm("strmibt r1, [r0], #1 ");
sl@0: 	asm("strvcbt r1, [r0], #1 ");	// if 01 or 11 do 1 byte store
sl@0: 	asm("sub r2, r2, r3, lsr #30 ");
sl@0: 	asm("3: ");						// r0 now word aligned
sl@0: 	asm("subs r2, r2, #4 ");
sl@0: 	asm("strplt r1, [r0], #4 ");
sl@0: 	asm("bgt 3b ");
sl@0: 	__JUMP(eq,lr);					// return if finished
sl@0: 	asm("adds r2, r2, #2 ");		// -1 if 1 left, 0 if 2 left, +1 if 3 left
sl@0: 	asm("strplbt r1, [r0], #1 ");
sl@0: 	asm("strplbt r1, [r0], #1 ");
sl@0: 	asm("strnebt r1, [r0], #1 ");
sl@0: 	__JUMP(,lr);
sl@0: 	}
sl@0: 	
sl@0: }