1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/os/kernelhwsrv/kernel/eka/klib/arm/cumem.cia Fri Jun 15 03:10:57 2012 +0200
1.3 @@ -0,0 +1,684 @@
1.4 +// Copyright (c) 1997-2009 Nokia Corporation and/or its subsidiary(-ies).
1.5 +// All rights reserved.
1.6 +// This component and the accompanying materials are made available
1.7 +// under the terms of the License "Eclipse Public License v1.0"
1.8 +// which accompanies this distribution, and is available
1.9 +// at the URL "http://www.eclipse.org/legal/epl-v10.html".
1.10 +//
1.11 +// Initial Contributors:
1.12 +// Nokia Corporation - initial contribution.
1.13 +//
1.14 +// Contributors:
1.15 +//
1.16 +// Description:
1.17 +// e32\klib\arm\cumem.cia
1.18 +//
1.19 +//
1.20 +
1.21 +#include <kernel/klib.h>
1.22 +#include <e32cia.h>
1.23 +#include <arm.h>
1.24 +#if defined(__REPLACE_GENERIC_UTILS)
1.25 +#include "replacement_utils.h"
1.26 +#endif
1.27 +
1.28 +extern "C" {
1.29 +
1.30 +#ifdef _DEBUG
1.31 +#define CUMEM_FAULT(cc, reason) asm("mov"#cc" r0, #%a0 " : : "i" (reason)); \
1.32 + asm("b"#cc" " CSM_ZN2KL5PanicENS_13TKernLibPanicE)
1.33 +#endif
1.34 +
1.35 +
1.36 +__NAKED__ void kumemget_no_paging_assert(TAny* /*aKernAddr*/, const TAny* /*aAddr*/, TInt /*aLength*/)
1.37 + {
1.38 + asm("mrs r3, spsr "); // r3=spsr_svc
1.39 + asm("tst r3, #0x0f "); // test for user mode
1.40 + asm("bne memcpy "); // if not, just do memcpy
1.41 +#ifndef USE_REPLACEMENT_UMEMGET
1.42 + asm("b umemget_no_paging_assert");
1.43 +#else
1.44 + asm("b umemget");
1.45 +#endif
1.46 + }
1.47 +
1.48 +
1.49 +#ifndef USE_REPLACEMENT_UMEMGET
1.50 +
1.51 +#ifdef __CPU_ARMV6
1.52 +// Conditional returns are not predicted on ARMv6
1.53 +__NAKED__ void dummy_umemget32_exit()
1.54 + {
1.55 + asm("_umemget32_exit: ");
1.56 + asm("ldmfd sp!, {r4, pc} ");
1.57 + }
1.58 +#define UMEMGET32_EXIT(cc) asm("b"#cc" _umemget32_exit")
1.59 +#else
1.60 +#define UMEMGET32_EXIT(cc) asm("ldm"#cc"fd sp!, {r4, pc}")
1.61 +#endif
1.62 +
1.63 +
1.64 +EXPORT_C __NAKED__ void kumemget32(TAny* /*aKernAddr*/, const TAny* /*aAddr*/, TInt /*aLength*/)
1.65 + {
1.66 + asm("mrs r3, spsr "); // r3=spsr_svc
1.67 + asm("tst r3, #0x0f "); // test for user mode
1.68 + asm("bne wordmove "); // if not, just do wordmove
1.69 + // otherwise fall through to umemget32
1.70 + }
1.71 +
1.72 +
1.73 +EXPORT_C __NAKED__ void umemget32(TAny* /*aKernAddr*/, const TAny* /*aUserAddr*/, TInt /*aLength*/)
1.74 + {
1.75 + ASM_ASSERT_PAGING_SAFE
1.76 +
1.77 +#ifdef __USER_MEMORY_GUARDS_ENABLED__
1.78 + // Wrap the workings of this function in an internal call, so we can save/restore UMG state
1.79 + asm("stmfd sp!, {r11, lr} ");
1.80 + asm("subs r12, r2, #1");
1.81 + asm("ldrhsb r11, [r0]"); // test access to first byte of kernel memory
1.82 + asm("ldrhsb r11, [r0,r12]"); // test access to last byte of kernel memory
1.83 + USER_MEMORY_GUARD_OFF(,r11,r12); // leaves UMG mode in r11
1.84 + asm("bl 0f"); // call to label below
1.85 + USER_MEMORY_GUARD_RESTORE(r11,r12);
1.86 + asm("ldmfd sp!, {r11, pc} ");
1.87 +
1.88 + asm("0:");
1.89 +#endif
1.90 +
1.91 +#ifdef _DEBUG
1.92 + asm("tst r2, #3 "); // check length is a whole number of words
1.93 + CUMEM_FAULT(ne, KL::EWordMoveLengthNotMultipleOf4);
1.94 +#endif
1.95 +
1.96 + PLD(1);
1.97 + asm("_umemget_word_aligned: ");
1.98 + asm("stmfd sp!, {r4, lr} ");
1.99 + asm("subs ip, r2, #32 ");
1.100 + asm("blo _umemget32_small_copy ");
1.101 + PLD_ioff(1, 32);
1.102 + asm("beq _umemget32_32_byte_case "); // 32 byte case is common - don't bother to align
1.103 +
1.104 + asm("rsb lr, r0, #32 "); // align destination: 0 - 28 byte copy
1.105 + asm("movs lr, lr, lsl #27 ");
1.106 + asm("beq _umemget32_large_copy ");
1.107 + asm("sub r2, r2, lr, lsr #27 ");
1.108 + asm("msr cpsr_f, lr "); // put length bits 4, 3, 2 into N, Z, C
1.109 + asm("ldrmit r3, [r1], #4 ");
1.110 + asm("ldrmit r4, [r1], #4 ");
1.111 + asm("ldrmit ip, [r1], #4 ");
1.112 + asm("ldrmit lr, [r1], #4 ");
1.113 + asm("stmmiia r0!, {r3, r4, ip, lr} ");
1.114 + asm("ldreqt r3, [r1], #4 ");
1.115 + asm("ldreqt r4, [r1], #4 ");
1.116 + asm("ldrcst ip, [r1], #4 ");
1.117 + asm("stmeqia r0!, {r3, r4} ");
1.118 + asm("strcs ip, [r0], #4 ");
1.119 + asm("subs ip, r2, #32 ");
1.120 + asm("blo _umemget32_small_copy ");
1.121 +
1.122 + asm("_umemget32_large_copy: "); // copy 32 byte blocks
1.123 + PLD_ioff(1, 64);
1.124 + asm("_umemget32_32_byte_case: ");
1.125 + asm("ldrt r2, [r1], #4 ");
1.126 + asm("ldrt r3, [r1], #4 ");
1.127 + asm("ldrt r4, [r1], #4 ");
1.128 + asm("ldrt lr, [r1], #4 ");
1.129 + asm("subs ip, ip, #32 ");
1.130 + asm("stmia r0!, {r2, r3, r4, lr} ");
1.131 + asm("ldrt r2, [r1], #4 ");
1.132 + asm("ldrt r3, [r1], #4 ");
1.133 + asm("ldrt r4, [r1], #4 ");
1.134 + asm("ldrt lr, [r1], #4 ");
1.135 + asm("stmia r0!, {r2, r3, r4, lr} ");
1.136 + asm("bhs _umemget32_large_copy ");
1.137 +
1.138 + asm("_umemget32_small_copy: "); // 0 - 31 byte copy, length in ip bits 0-4
1.139 + asm("movs r2, ip, lsl #27 ");
1.140 + UMEMGET32_EXIT(eq);
1.141 + asm("msr cpsr_f, r2 "); // put length bits 4, 3, 2 into N, Z, C
1.142 + asm("ldrmit r3, [r1], #4 ");
1.143 + asm("ldrmit r4, [r1], #4 ");
1.144 + asm("ldrmit ip, [r1], #4 ");
1.145 + asm("ldrmit lr, [r1], #4 ");
1.146 + asm("stmmiia r0!, {r3, r4, ip, lr} ");
1.147 + asm("ldreqt r3, [r1], #4 ");
1.148 + asm("ldreqt r4, [r1], #4 ");
1.149 + asm("ldrcst ip, [r1], #4 ");
1.150 + asm("stmeqia r0!, {r3, r4} ");
1.151 + asm("strcs ip, [r0], #4 ");
1.152 + asm("movs r2, r2, lsl #3 ");
1.153 + UMEMGET32_EXIT(eq);
1.154 + asm("msr cpsr_f, r2 "); // put length bits 1, 0 into N, Z
1.155 + asm("ldrmibt r3, [r1], #1 ");
1.156 + asm("ldrmibt r4, [r1], #1 ");
1.157 + asm("ldreqbt ip, [r1], #1 ");
1.158 + asm("strmib r3, [r0], #1 ");
1.159 + asm("strmib r4, [r0], #1 ");
1.160 + asm("streqb ip, [r0], #1 ");
1.161 + asm("ldmfd sp!, {r4, pc} ");
1.162 + }
1.163 +
1.164 +
1.165 +EXPORT_C __NAKED__ void kumemget(TAny* /*aKernAddr*/, const TAny* /*aAddr*/, TInt /*aLength*/)
1.166 + {
1.167 + asm("mrs r3, spsr "); // r3=spsr_svc
1.168 + asm("tst r3, #0x0f "); // test for user mode
1.169 + asm("bne memcpy "); // if not, just do memcpy
1.170 + // otherwise fall through to umemget
1.171 + }
1.172 +
1.173 +
1.174 +EXPORT_C __NAKED__ void umemget(TAny* /*aKernAddr*/, const TAny* /*aUserAddr*/, TInt /*aLength*/)
1.175 + {
1.176 + // Optimised for aligned transfers, as unaligned are very rare in practice
1.177 +
1.178 + ASM_ASSERT_PAGING_SAFE
1.179 + asm("umemget_no_paging_assert:");
1.180 +
1.181 +#ifdef __USER_MEMORY_GUARDS_ENABLED__
1.182 + // Wrap the workings of this function in an internal call, so we can save/restore UMG state
1.183 + asm("stmfd sp!, {r11, lr} ");
1.184 + asm("subs r12, r2, #1");
1.185 + asm("ldrhsb r11, [r0]"); // test access to first byte of kernel memory
1.186 + asm("ldrhsb r11, [r0,r12]"); // test access to last byte of kernel memory
1.187 + USER_MEMORY_GUARD_OFF(,r11,r12); // leaves UMG mode in r11
1.188 + asm("bl 0f"); // call to label below
1.189 + USER_MEMORY_GUARD_RESTORE(r11,r12);
1.190 + asm("ldmfd sp!, {r11, pc} ");
1.191 +
1.192 + asm("0:");
1.193 +#endif
1.194 +
1.195 + PLD(1);
1.196 + asm("tst r0, #3 ");
1.197 + asm("tsteq r1, #3 ");
1.198 + asm("beq _umemget_word_aligned ");
1.199 + asm("cmp r2, #8 ");
1.200 + asm("bhs 1f ");
1.201 +
1.202 + asm("2: ");
1.203 + asm("subs r2, r2, #1 ");
1.204 + asm("ldrplbt r3, [r1], #1 ");
1.205 + asm("strplb r3, [r0], #1 ");
1.206 + asm("bgt 2b ");
1.207 + __JUMP(,lr);
1.208 +
1.209 + asm("1: "); // Attempt to word align
1.210 + asm("movs r3, r0, lsl #30 ");
1.211 + asm("beq 5f ");
1.212 + asm("rsbs r3, r3, #0 "); // 01->c0000000 (MI,VC) 10->80000000 (MI,VS) 11->40000000 (PL,VC)
1.213 + asm("sub r2, r2, r3, lsr #30 ");
1.214 + asm("ldrmibt r3, [r1], #1 ");
1.215 + asm("strmib r3, [r0], #1 ");
1.216 + asm("ldrmibt r3, [r1], #1 ");
1.217 + asm("strmib r3, [r0], #1 ");
1.218 + asm("ldrvcbt r3, [r1], #1 ");
1.219 + asm("strvcb r3, [r0], #1 "); // r0 now word aligned
1.220 + asm("5: ");
1.221 + asm("movs r3, r1, lsl #31 ");
1.222 + asm("bic r1, r1, #3 ");
1.223 + asm("bcs 3f "); // branch if src mod 4 = 2 or 3
1.224 + asm("bpl _umemget_word_aligned "); // branch if src mod 4 = 0
1.225 +
1.226 + asm("4: "); // src mod 4 = 1
1.227 + asm("subs r2, r2, #4 ");
1.228 + asm("ldrget r3, [r1], #4 ");
1.229 + asm("ldrget ip, [r1] ");
1.230 + asm("movge r3, r3, lsr #8 ");
1.231 + asm("orrge r3, r3, ip, lsl #24 ");
1.232 + asm("strge r3, [r0], #4 ");
1.233 + asm("bgt 4b ");
1.234 + asm("add r1, r1, #1 ");
1.235 + asm("b umemget_do_end ");
1.236 +
1.237 + asm("3: ");
1.238 + asm("bmi 5f ");
1.239 + asm("2: "); // src mod 4 = 2
1.240 + asm("subs r2, r2, #4 ");
1.241 + asm("ldrget r3, [r1], #4 ");
1.242 + asm("ldrget ip, [r1] ");
1.243 + asm("movge r3, r3, lsr #16 ");
1.244 + asm("orrge r3, r3, ip, lsl #16 ");
1.245 + asm("strge r3, [r0], #4 ");
1.246 + asm("bgt 2b ");
1.247 + asm("add r1, r1, #2 ");
1.248 + asm("b umemget_do_end ");
1.249 +
1.250 + asm("5: "); // src mod 4 = 3
1.251 + asm("subs r2, r2, #4 ");
1.252 + asm("ldrget r3, [r1], #4 ");
1.253 + asm("ldrget ip, [r1] ");
1.254 + asm("movge r3, r3, lsr #24 ");
1.255 + asm("orrge r3, r3, ip, lsl #8 ");
1.256 + asm("strge r3, [r0], #4 ");
1.257 + asm("bgt 5b ");
1.258 + asm("add r1, r1, #3 ");
1.259 +
1.260 + asm("umemget_do_end: ");
1.261 + __JUMP(eq,lr);
1.262 + asm("adds r2, r2, #2 "); // -1 if 1 left, 0 if 2 left, +1 if 3 left
1.263 + asm("ldrplbt r3, [r1], #1 ");
1.264 + asm("strplb r3, [r0], #1 ");
1.265 + asm("ldrplbt r3, [r1], #1 ");
1.266 + asm("strplb r3, [r0], #1 ");
1.267 + asm("ldrnebt r3, [r1], #1 ");
1.268 + asm("strneb r3, [r0], #1 ");
1.269 + __JUMP(,lr);
1.270 + }
1.271 +
1.272 +#endif // USE_REPLACEMENT_UMEMGET
1.273 +
1.274 +__NAKED__ void kumemput_no_paging_assert(TAny* /*aAddr*/, const TAny* /*aKernAddr*/, TInt /*aLength*/)
1.275 + {
1.276 + asm("mrs r3, spsr "); // r3=spsr_svc
1.277 + asm("tst r3, #0x0f "); // test for user mode
1.278 + asm("bne memcpy "); // if not, just do memcpy
1.279 +#ifndef USE_REPLACEMENT_UMEMPUT
1.280 + asm("b umemput_no_paging_assert");
1.281 +#else
1.282 + asm("b umemput");
1.283 +#endif
1.284 + }
1.285 +
1.286 +
1.287 +#ifndef USE_REPLACEMENT_UMEMPUT
1.288 +
1.289 +#ifdef __CPU_ARMV6
1.290 +// Conditional returns are not predicted on ARMv6
1.291 +__NAKED__ void dummy_umemput32_exit()
1.292 + {
1.293 + asm("_umemput32_exit: ");
1.294 + asm("ldmfd sp!, {r4, pc} ");
1.295 + }
1.296 +#define UMEMPUT32_EXIT(cc) asm("b"#cc" _umemput32_exit")
1.297 +#else
1.298 +#define UMEMPUT32_EXIT(cc) asm("ldm"#cc"fd sp!, {r4, pc}")
1.299 +#endif
1.300 +
1.301 +
1.302 +EXPORT_C __NAKED__ void kumemput32(TAny* /*aAddr*/, const TAny* /*aKernAddr*/, TInt /*aLength*/)
1.303 + {
1.304 + asm("mrs r3, spsr "); // r3=spsr_svc
1.305 + asm("tst r3, #0x0f "); // test for user mode
1.306 + asm("bne wordmove "); // if not, just do wordmove
1.307 + // otherwise fall through to umemput32
1.308 + }
1.309 +
1.310 +
1.311 +EXPORT_C __NAKED__ void umemput32(TAny* /*aUserAddr*/, const TAny* /*aKernAddr*/, TInt /*aLength*/)
1.312 + {
1.313 + ASM_ASSERT_DATA_PAGING_SAFE
1.314 +
1.315 +#ifdef __USER_MEMORY_GUARDS_ENABLED__
1.316 + // Wrap the workings of this function in an internal call, so we can save/restore UMG state
1.317 + asm("stmfd sp!, {r11, lr} ");
1.318 + asm("subs r12, r2, #1");
1.319 + asm("ldrhsb r11, [r1]"); // test access to first byte of kernel memory
1.320 + asm("ldrhsb r11, [r1,r12]"); // test access to last byte of kernel memory
1.321 + USER_MEMORY_GUARD_OFF(,r11,r12); // leaves UMG mode in r11
1.322 + asm("bl 0f"); // call to label below
1.323 + USER_MEMORY_GUARD_RESTORE(r11,r12);
1.324 + asm("ldmfd sp!, {r11, pc} ");
1.325 +
1.326 + asm("0:");
1.327 +#endif
1.328 +
1.329 +#ifdef _DEBUG
1.330 + asm("tst r2, #3 "); // check length is a whole number of words
1.331 + CUMEM_FAULT(ne, KL::EWordMoveLengthNotMultipleOf4);
1.332 +#endif
1.333 +
1.334 + PLD(1);
1.335 + asm("cmp r2, #4 "); // special case for 4 byte copy which is common
1.336 + asm("ldrhs r3, [r1], #4 ");
1.337 + asm("subhs r2, r2, #4 ");
1.338 + asm("strhst r3, [r0], #4 ");
1.339 + __JUMP(ls,lr);
1.340 +
1.341 + asm("_umemput_word_aligned: ");
1.342 + asm("stmfd sp!, {r4, lr} ");
1.343 + asm("subs r2, r2, #32 ");
1.344 + asm("bhs _umemput32_align_source ");
1.345 +
1.346 + asm("_umemput32_small_copy: "); // copy 1 - 31 bytes
1.347 + asm("mov r2, r2, lsl #27 ");
1.348 + asm("msr cpsr_f, r2 "); // put length bits 4, 3, 2 into N, Z, C
1.349 + asm("ldmmiia r1!, {r3, r4, ip, lr} ");
1.350 + asm("strmit r3, [r0], #4 ");
1.351 + asm("strmit r4, [r0], #4 ");
1.352 + asm("ldmeqia r1!, {r3, r4} ");
1.353 + asm("strmit ip, [r0], #4 ");
1.354 + asm("ldrcs ip, [r1], #4 ");
1.355 + asm("strmit lr, [r0], #4 ");
1.356 + asm("streqt r3, [r0], #4 ");
1.357 + asm("streqt r4, [r0], #4 ");
1.358 + asm("strcst ip, [r0], #4 ");
1.359 + asm("movs r2, r2, lsl #3 ");
1.360 + UMEMPUT32_EXIT(eq);
1.361 + asm("msr cpsr_f, r2 "); // put length bits 1, 0 into N, Z
1.362 + asm("ldrmih r3, [r1], #2 ");
1.363 + asm("ldreqb r4, [r1], #1 ");
1.364 + asm("strmibt r3, [r0], #1 ");
1.365 + asm("movmi r3, r3, lsr #8 ");
1.366 + asm("strmibt r3, [r0], #1 ");
1.367 + asm("streqbt r4, [r0], #1 ");
1.368 + asm("ldmfd sp!, {r4, pc} ");
1.369 +
1.370 + asm("_umemput32_align_source: ");
1.371 + PLD_ioff(1, 32);
1.372 + asm("cmp r2, #32 ");
1.373 + asm("bls _umemput32_large_copy "); // don't bother if length <= 64
1.374 + asm("rsb ip, r1, #32 ");
1.375 + asm("movs ip, ip, lsl #27 ");
1.376 + asm("beq _umemput32_large_copy ");
1.377 + asm("msr cpsr_f, ip "); // put length bits 4, 3, 2 into N, Z, C
1.378 + asm("sub r2, r2, ip, lsr #27 ");
1.379 + asm("ldmmiia r1!, {r3, r4, ip, lr} ");
1.380 + asm("strmit r3, [r0], #4 ");
1.381 + asm("strmit r4, [r0], #4 ");
1.382 + asm("ldmeqia r1!, {r3, r4} ");
1.383 + asm("strmit ip, [r0], #4 ");
1.384 + asm("ldrcs ip, [r1], #4 ");
1.385 + asm("strmit lr, [r0], #4 ");
1.386 + asm("streqt r3, [r0], #4 ");
1.387 + asm("streqt r4, [r0], #4 ");
1.388 + asm("strcst ip, [r0], #4 ");
1.389 +
1.390 + asm("_umemput32_large_copy: "); // copy 32 byte blocks
1.391 + PLD_ioff(1, 64);
1.392 + asm("ldmia r1!, {r3, r4, ip, lr} ");
1.393 + asm("strt r3, [r0], #4 ");
1.394 + asm("strt r4, [r0], #4 ");
1.395 + asm("strt ip, [r0], #4 ");
1.396 + asm("strt lr, [r0], #4 ");
1.397 + asm("ldmia r1!, {r3, r4, ip, lr} ");
1.398 + asm("strt r3, [r0], #4 ");
1.399 + asm("strt r4, [r0], #4 ");
1.400 + asm("strt ip, [r0], #4 ");
1.401 + asm("strt lr, [r0], #4 ");
1.402 + asm("subs r2, r2, #32 ");
1.403 + asm("bhs _umemput32_large_copy ");
1.404 + asm("adds r2, r2, #32 ");
1.405 + asm("bne _umemput32_small_copy ");
1.406 + asm("ldmfd sp!, {r4, pc} ");
1.407 + }
1.408 +
1.409 +
1.410 +__NAKED__ void uumemcpy32(TAny* /*aUserDst*/, const TAny* /*aUserSrc*/, TInt /*aLength*/)
1.411 + {
1.412 + ASM_ASSERT_PAGING_SAFE
1.413 +
1.414 +#ifdef __USER_MEMORY_GUARDS_ENABLED__
1.415 + // Wrap the workings of this function in an internal call, so we can save/restore UMG state
1.416 + asm("stmfd sp!, {r11, lr} ");
1.417 + USER_MEMORY_GUARD_OFF(,r11,r12); // leaves UMG mode in r11
1.418 + asm("bl 0f"); // call to label below
1.419 + USER_MEMORY_GUARD_RESTORE(r11,r12);
1.420 + asm("ldmfd sp!, {r11, pc} ");
1.421 +
1.422 + asm("0:");
1.423 +#endif
1.424 +
1.425 + asm("1: ");
1.426 + asm("subs r2, r2, #4 ");
1.427 + asm("ldrplt r3, [r1], #4 ");
1.428 + asm("strplt r3, [r0], #4 ");
1.429 + asm("bpl 1b ");
1.430 + __JUMP(,lr);
1.431 + }
1.432 +
1.433 +
1.434 +__NAKED__ void uumemcpy(TAny* /*aUserDst*/, const TAny* /*aUserSrc*/, TInt /*aLength*/)
1.435 + {
1.436 + ASM_ASSERT_PAGING_SAFE
1.437 +
1.438 +#ifdef __USER_MEMORY_GUARDS_ENABLED__
1.439 + // Wrap the workings of this function in an internal call, so we can save/restore UMG state
1.440 + asm("stmfd sp!, {r11, lr} ");
1.441 + USER_MEMORY_GUARD_OFF(,r11,r12); // leaves UMG mode in r11
1.442 + asm("bl 0f"); // call to label below
1.443 + USER_MEMORY_GUARD_RESTORE(r11,r12);
1.444 + asm("ldmfd sp!, {r11, pc} ");
1.445 +
1.446 + asm("0:");
1.447 +#endif
1.448 +
1.449 + asm("cmp r2, #8 ");
1.450 + asm("bcs 1f ");
1.451 + asm("2: ");
1.452 + asm("subs r2, r2, #1 ");
1.453 + asm("ldrplbt r3, [r1], #1 ");
1.454 + asm("strplbt r3, [r0], #1 ");
1.455 + asm("bgt 2b ");
1.456 + __JUMP(,lr);
1.457 + asm("1: ");
1.458 + asm("movs r3, r0, lsl #30 ");
1.459 + asm("beq 5f ");
1.460 + asm("rsbs r3, r3, #0 "); // 01->c0000000 (MI,VC) 10->80000000 (MI,VS) 11->40000000 (PL,VC)
1.461 + asm("sub r2, r2, r3, lsr #30 ");
1.462 + asm("ldrmibt r3, [r1], #1 ");
1.463 + asm("strmibt r3, [r0], #1 ");
1.464 + asm("ldrmibt r3, [r1], #1 ");
1.465 + asm("strmibt r3, [r0], #1 ");
1.466 + asm("ldrvcbt r3, [r1], #1 ");
1.467 + asm("strvcbt r3, [r0], #1 "); // r0 now word aligned
1.468 + asm("5: ");
1.469 + asm("movs r3, r1, lsl #31 ");
1.470 + asm("bic r1, r1, #3 ");
1.471 + asm("bcs 3f "); // branch if src mod 4 = 2 or 3
1.472 + asm("bmi 4f "); // branch if src mod 4 = 1
1.473 + asm("2: ");
1.474 + asm("subs r2, r2, #4 ");
1.475 + asm("ldrget r3, [r1], #4 ");
1.476 + asm("strget r3, [r0], #4 ");
1.477 + asm("bgt 2b ");
1.478 + asm("uumemcpy_do_end: ");
1.479 + __JUMP(eq,lr);
1.480 + asm("adds r2, r2, #2 "); // -1 if 1 left, 0 if 2 left, +1 if 3 left
1.481 + asm("ldrplbt r3, [r1], #1 ");
1.482 + asm("strplbt r3, [r0], #1 ");
1.483 + asm("ldrplbt r3, [r1], #1 ");
1.484 + asm("strplbt r3, [r0], #1 ");
1.485 + asm("ldrnebt r3, [r1], #1 ");
1.486 + asm("strnebt r3, [r0], #1 ");
1.487 + __JUMP(,lr);
1.488 + asm("3: "); // get here if src mod 4 = 2 or 3
1.489 + asm("bmi 5f "); // branch if 3
1.490 + asm("2: ");
1.491 + asm("subs r2, r2, #4 ");
1.492 + asm("ldrget r3, [r1], #4 ");
1.493 + asm("ldrget ip, [r1] ");
1.494 + asm("movge r3, r3, lsr #16 ");
1.495 + asm("orrge r3, r3, ip, lsl #16 ");
1.496 + asm("strget r3, [r0], #4 ");
1.497 + asm("bgt 2b ");
1.498 + asm("add r1, r1, #2 ");
1.499 + asm("b uumemcpy_do_end ");
1.500 + asm("5: ");
1.501 + asm("subs r2, r2, #4 ");
1.502 + asm("ldrget r3, [r1], #4 ");
1.503 + asm("ldrget ip, [r1] ");
1.504 + asm("movge r3, r3, lsr #24 ");
1.505 + asm("orrge r3, r3, ip, lsl #8 ");
1.506 + asm("strget r3, [r0], #4 ");
1.507 + asm("bgt 5b ");
1.508 + asm("add r1, r1, #3 ");
1.509 + asm("b uumemcpy_do_end ");
1.510 + asm("4: ");
1.511 + asm("subs r2, r2, #4 ");
1.512 + asm("ldrget r3, [r1], #4 ");
1.513 + asm("ldrget ip, [r1] ");
1.514 + asm("movge r3, r3, lsr #8 ");
1.515 + asm("orrge r3, r3, ip, lsl #24 ");
1.516 + asm("strget r3, [r0], #4 ");
1.517 + asm("bgt 4b ");
1.518 + asm("add r1, r1, #1 ");
1.519 + asm("b uumemcpy_do_end ");
1.520 + }
1.521 +
1.522 +
1.523 +EXPORT_C __NAKED__ void kumemput(TAny* /*aAddr*/, const TAny* /*aKernAddr*/, TInt /*aLength*/)
1.524 + {
1.525 + asm("mrs r3, spsr "); // r3=spsr_svc
1.526 + asm("tst r3, #0x0f "); // test for user mode
1.527 + asm("bne memcpy "); // if not, just do memcpy
1.528 + // otherwise fall through to umemput
1.529 + }
1.530 +
1.531 +
1.532 +EXPORT_C __NAKED__ void umemput(TAny* /*aUserAddr*/, const TAny* /*aKernAddr*/, TInt /*aLength*/)
1.533 + {
1.534 + // Optimised for word-aligned transfers, as unaligned are very rare in practice
1.535 +
1.536 + ASM_ASSERT_DATA_PAGING_SAFE
1.537 + asm("umemput_no_paging_assert:");
1.538 +
1.539 +#ifdef __USER_MEMORY_GUARDS_ENABLED__
1.540 + // Wrap the workings of this function in an internal call, so we can save/restore UMG state
1.541 + asm("stmfd sp!, {r11, lr} ");
1.542 + asm("subs r12, r2, #1");
1.543 + asm("ldrhsb r11, [r1]"); // test access to first byte of kernel memory
1.544 + asm("ldrhsb r11, [r1,r12]"); // test access to last byte of kernel memory
1.545 + USER_MEMORY_GUARD_OFF(,r11,r12); // leaves UMG mode in r11
1.546 + asm("bl 0f"); // call to label below
1.547 + USER_MEMORY_GUARD_RESTORE(r11,r12);
1.548 + asm("ldmfd sp!, {r11, pc} ");
1.549 +
1.550 + asm("0:");
1.551 +#endif
1.552 +
1.553 + PLD(1);
1.554 + asm("tst r0, #3 ");
1.555 + asm("tsteq r1, #3 ");
1.556 + asm("beq _umemput_word_aligned ");
1.557 +
1.558 + asm("cmp r2, #8 ");
1.559 + asm("bcs 1f ");
1.560 + asm("2: "); // Copy 0 - 7 bytes
1.561 + asm("subs r2, r2, #1 ");
1.562 + asm("ldrplb r3, [r1], #1 ");
1.563 + asm("strplbt r3, [r0], #1 ");
1.564 + asm("bgt 2b ");
1.565 + __JUMP(,lr);
1.566 +
1.567 + asm("1: "); // Word-align dest
1.568 + asm("movs r3, r0, lsl #30 ");
1.569 + asm("beq 5f ");
1.570 + asm("rsbs r3, r3, #0 "); // 01->c0000000 (MI,VC) 10->80000000 (MI,VS) 11->40000000 (PL,VC)
1.571 + asm("sub r2, r2, r3, lsr #30 ");
1.572 + asm("ldrmib r3, [r1], #1 ");
1.573 + asm("strmibt r3, [r0], #1 ");
1.574 + asm("ldrmib r3, [r1], #1 ");
1.575 + asm("strmibt r3, [r0], #1 ");
1.576 + asm("ldrvcb r3, [r1], #1 ");
1.577 + asm("strvcbt r3, [r0], #1 "); // r0 now word aligned
1.578 + asm("5: ");
1.579 + asm("movs r3, r1, lsl #31 ");
1.580 + asm("bic r1, r1, #3 ");
1.581 + asm("bcs 3f "); // branch if src mod 4 = 2 or 3
1.582 + asm("bpl _umemput_word_aligned "); // branch if src mod 4 = 0
1.583 +
1.584 + asm("4: "); // get here if src mod 4 = 1
1.585 + asm("subs r2, r2, #4 ");
1.586 + asm("ldrge r3, [r1], #4 ");
1.587 + asm("ldrge ip, [r1] ");
1.588 + asm("movge r3, r3, lsr #8 ");
1.589 + asm("orrge r3, r3, ip, lsl #24 ");
1.590 + asm("strget r3, [r0], #4 ");
1.591 + asm("bgt 4b ");
1.592 + asm("add r1, r1, #1 ");
1.593 + asm("b _umemput_do_end ");
1.594 +
1.595 + asm("3: "); // get here if src mod 4 = 2 or 3
1.596 + asm("bmi 5f "); // branch if 3
1.597 + asm("2: ");
1.598 + asm("subs r2, r2, #4 ");
1.599 + asm("ldrge r3, [r1], #4 ");
1.600 + asm("ldrge ip, [r1] ");
1.601 + asm("movge r3, r3, lsr #16 ");
1.602 + asm("orrge r3, r3, ip, lsl #16 ");
1.603 + asm("strget r3, [r0], #4 ");
1.604 + asm("bgt 2b ");
1.605 + asm("add r1, r1, #2 ");
1.606 + asm("b _umemput_do_end ");
1.607 +
1.608 + asm("5: "); // get here if src mod 4 = 3
1.609 + asm("subs r2, r2, #4 ");
1.610 + asm("ldrge r3, [r1], #4 ");
1.611 + asm("ldrge ip, [r1] ");
1.612 + asm("movge r3, r3, lsr #24 ");
1.613 + asm("orrge r3, r3, ip, lsl #8 ");
1.614 + asm("strget r3, [r0], #4 ");
1.615 + asm("bgt 5b ");
1.616 + asm("add r1, r1, #3 ");
1.617 +
1.618 + asm("_umemput_do_end: "); // z set if done, else r2 == length remaining - 4
1.619 + __JUMP(eq,lr);
1.620 + asm("adds r2, r2, #2 "); // r2 = -1 if 1 left, 0 if 2 left, +1 if 3 left
1.621 + asm("ldrplb r3, [r1], #1 ");
1.622 + asm("strplbt r3, [r0], #1 ");
1.623 + asm("ldrplb r3, [r1], #1 ");
1.624 + asm("strplbt r3, [r0], #1 ");
1.625 + asm("ldrneb r3, [r1], #1 ");
1.626 + asm("strnebt r3, [r0], #1 ");
1.627 + __JUMP(,lr);
1.628 + }
1.629 +
1.630 +#endif // USE_REPLACEMENT_UMEMPUT
1.631 +
1.632 +
1.633 +EXPORT_C __NAKED__ void kumemset(TAny* /*aAddr*/, const TUint8 /*aValue*/, TInt /*aLength*/)
1.634 + {
1.635 + asm("mrs r3, spsr "); // r3=spsr_svc
1.636 + asm("tst r3, #0x0f "); // test for user mode
1.637 + asm("bne memset "); // if not, just do memset
1.638 + // otherwise fall through to umemset
1.639 + }
1.640 +
1.641 +
1.642 +EXPORT_C __NAKED__ void umemset(TAny* /*aUserAddr*/, const TUint8 /*aValue*/, TInt /*aLength*/)
1.643 + {
1.644 + ASM_ASSERT_DATA_PAGING_SAFE
1.645 +
1.646 +#ifdef __USER_MEMORY_GUARDS_ENABLED__
1.647 + // Wrap the workings of this function in an internal call, so we can save/restore UMG state
1.648 + asm("stmfd sp!, {r11, lr} ");
1.649 + USER_MEMORY_GUARD_OFF(,r11,r12); // leaves UMG mode in r11
1.650 + asm("bl 0f"); // call to label below
1.651 + USER_MEMORY_GUARD_RESTORE(r11,r12);
1.652 + asm("ldmfd sp!, {r11, pc} ");
1.653 +
1.654 + asm("0:");
1.655 +#endif
1.656 +
1.657 + asm("cmp r2, #7 ");
1.658 + asm("bhi 2f ");
1.659 + asm("1: ");
1.660 + asm("subs r2, r2, #1 ");
1.661 + asm("strplbt r1, [r0], #1 ");
1.662 + asm("bgt 1b ");
1.663 + __JUMP(,lr);
1.664 + asm("2: ");
1.665 + asm("and r1, r1, #0xff ");
1.666 + asm("orr r1, r1, r1, lsl #8 ");
1.667 + asm("orr r1, r1, r1, lsl #16 ");
1.668 + asm("movs r3, r0, lsl #30 ");
1.669 + asm("beq 3f ");
1.670 + asm("rsbs r3, r3, #0 "); // 01->c0000000 (MI,VC) 10->80000000 (MI,VS) 11->40000000 (PL,VC)
1.671 + asm("strmibt r1, [r0], #1 "); // if 01 or 10 do 2 byte stores
1.672 + asm("strmibt r1, [r0], #1 ");
1.673 + asm("strvcbt r1, [r0], #1 "); // if 01 or 11 do 1 byte store
1.674 + asm("sub r2, r2, r3, lsr #30 ");
1.675 + asm("3: "); // r0 now word aligned
1.676 + asm("subs r2, r2, #4 ");
1.677 + asm("strplt r1, [r0], #4 ");
1.678 + asm("bgt 3b ");
1.679 + __JUMP(eq,lr); // return if finished
1.680 + asm("adds r2, r2, #2 "); // -1 if 1 left, 0 if 2 left, +1 if 3 left
1.681 + asm("strplbt r1, [r0], #1 ");
1.682 + asm("strplbt r1, [r0], #1 ");
1.683 + asm("strnebt r1, [r0], #1 ");
1.684 + __JUMP(,lr);
1.685 + }
1.686 +
1.687 +}