diff -r 000000000000 -r bde4ae8d615e os/kernelhwsrv/kernel/eka/nkernsmp/x86/ncsched.cia --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/os/kernelhwsrv/kernel/eka/nkernsmp/x86/ncsched.cia Fri Jun 15 03:10:57 2012 +0200 @@ -0,0 +1,624 @@ +// Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies). +// All rights reserved. +// This component and the accompanying materials are made available +// under the terms of the License "Eclipse Public License v1.0" +// which accompanies this distribution, and is available +// at the URL "http://www.eclipse.org/legal/epl-v10.html". +// +// Initial Contributors: +// Nokia Corporation - initial contribution. +// +// Contributors: +// +// Description: +// e32\nkernsmp\x86\ncsched.cia +// +// + +#include +#include + +// SubSchedulerLookupTable : global data, type: TSubScheduler* [256]; +// BTraceLock : global data, type: TSpinLock + +const TLinAddr TScheduler_Reschedule = (TLinAddr)&TScheduler::Reschedule; +//const TLinAddr TheScheduler_iRescheduleNeededFlag = (TLinAddr)&TheScheduler.iRescheduleNeededFlag; +const TLinAddr NKern_FastCounter = (TLinAddr)&NKern::FastCounter; +const TLinAddr NKern_Lock = (TLinAddr)&NKern::Lock; +const TLinAddr NKern_Unlock = (TLinAddr)&NKern::Unlock; +const TLinAddr addressof_TheScheduler = (TLinAddr)&TheScheduler; +const TUint32 new_thread_trace_header = ((8< original thread + asm("mov eax, cr0"); + asm("push eax"); + asm("mov [ebp+%0], esp" : : "i" _FOFF(NThreadBase, iSavedSP)); // Save original thread stack pointer + + // We must move to a temporary stack before selecting the next thread. + // This is because another CPU may begin executing this thread before the + // select_next_thread() function returns and our stack would then be + // corrupted. We use the stack belonging to this CPU's initial thread since + // we are guaranteed that will never run on another CPU. + asm("mov ecx, [esi+%0]" : : "i" _FOFF(TSubScheduler, iInitialThread)); + asm("mov esp, [ecx+%0]" : : "i" _FOFF(NThreadBase, iSavedSP)); + + asm("select_thread:"); + asm("mov ecx, esi "); + asm("call %a0" : : "i" (&select_next_thread)); + asm("mov ebx, eax "); + asm("cmp ebx, 0 "); + asm("jz no_thread "); + asm("mov esp, [ebx+%0]" : : "i" _FOFF(NThreadBase, iSavedSP)); // move to new thread's stack + +#ifdef BTRACE_CPU_USAGE + asm("cmp byte ptr %a0, 0" : : "i" (&BTraceData.iFilter[4])); + asm("jz short no_trace "); + asm("push ebx "); + asm("call %a0" : : "i" (NewThreadTrace)); + asm("pop ebx "); + asm("no_trace: "); +#endif // BTRACE_CPU_USAGE + + asm("cmp ebp, ebx "); + asm("je same_thread "); + asm("mov eax, [ebx+%0]" : : "i" _FOFF(NThreadBase, iStackBase)); + asm("add eax, [ebx+%0]" : : "i" _FOFF(NThreadBase, iStackSize)); + asm("mov ecx, [esi+60+%0]" : : "i" _FOFF(TSubScheduler, iExtras)); // iExtras[15] points to TSS + asm("mov [ecx+%0], eax" : : "i" _FOFF(TX86Tss, iEsp0)); // set ESP0 to top of new thread supervisor stack + + asm("test byte ptr [ebx+%0], 2" : : "i" _FOFF(NThreadBase,i_ThrdAttr)); // test for address space switch + asm("jz short resched_no_as_switch "); + asm("call [edi+%0]" : : "i" _FOFF(TScheduler, iProcessHandler)); // call handler with + // EBX=pointer to new thread, EDI->scheduler, ESI->subscheduler + asm("resched_no_as_switch: "); + asm("same_thread: "); + asm("pop eax "); + asm("mov cr0, eax "); + asm("cli "); +// asm("cmp dword ptr [esi]TSubScheduler.iRescheduleNeededFlag, 0 VC6 ignores the "dword ptr" + asm("lea eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag)); + asm("cmp dword ptr [eax], 0 "); + asm("jnz start_resched "); + + asm("resched_not_needed: "); + asm("mov edi, [esi+%0]" : : "i" _FOFF(TSubScheduler, iCurrentThread)); + asm("cmp dword ptr [edi+%0], -3" : : "i" _FOFF(NThreadBase, iCsFunction)); // ECSDivertPending + asm("je resched_thread_divert "); + asm("mov dword ptr [esi+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount)); + asm("pop eax "); + asm("ret "); + + asm("resched_thread_divert: "); + asm("push edi "); + asm("xor eax, eax "); + asm("lock xchg eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iReschedIPIs)); + asm("test eax, eax "); + asm("jz short no_resched_ipis "); + asm("push eax "); + asm("call %a0" : : "i" (&send_resched_ipis)); + asm("add esp, 4 "); + asm("no_resched_ipis: "); + + asm("sti "); + asm("mov ecx, [esp+12] "); // SThreadReschedStack iReason 0 not run 1 unlock 2 IRQ + asm("cmp ecx, 2 "); + asm("ja short rtd_unknown "); // unknown - die + asm("shl ecx, 2 "); // reason * 4 + asm("mov eax, 0xa1a "); + asm("shr eax, cl "); + asm("and eax, 15 "); + asm("mov gs, [esp+eax*4+16] "); // restore GS + + asm("pop ecx "); // exiting thread pointer + asm("call %a0" : : "i" (&do_forced_exit)); + asm("int 0xff "); // should never get here + + asm("rtd_unknown: "); + asm("int 0xff "); // should never get here + + + // There is no thread ready to run + asm("no_thread: "); + asm("cli "); + asm("xor eax, eax "); + asm("lock xchg eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iReschedIPIs)); + asm("test eax, eax "); + asm("jz short no_resched_ipis2 "); + asm("push eax "); + asm("call %a0" : : "i" (&send_resched_ipis)); + asm("add esp, 4 "); + asm("no_resched_ipis2: "); + asm("sti "); + asm("hlt "); + asm("no_thread2: "); +// _asm cmp dword ptr [esi]TSubScheduler.iRescheduleNeededFlag, 10000h VC6 ignores the "dword ptr" + asm("lea eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag)); + asm("cmp dword ptr [eax], 0x10000 "); + asm("jb short no_thread "); + asm("mov ecx, esi "); + asm("call %a0" : : "i" (&queue_dfcs)); + asm("cmp byte ptr [esi+%0], 0" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag)); + asm("jz short no_thread2 "); + asm("jmp select_thread "); + } + + +/** Disable interrupts to the specified level + +If aLevel = 0 does not affect interrupt state +If aLevel <>0 disables all maskable interrupts. + +@param aLevel level to which to disable +@return Cookie to pass into RestoreInterrupts() +*/ +EXPORT_C __NAKED__ TInt NKern::DisableInterrupts(TInt /*aLevel*/) + { + asm("pushfd"); + asm("mov ecx, [esp+4]"); + asm("pop eax"); + asm("and eax, 0x200"); + asm("test ecx, ecx"); + asm("jz disable_ints_0"); + asm("cli"); + asm("disable_ints_0:"); + asm("ret"); + } + + +/** Disable all maskable interrupts + +@return Cookie to pass into RestoreInterrupts() +*/ +EXPORT_C __NAKED__ TInt NKern::DisableAllInterrupts() + { + asm("pushfd"); + asm("pop eax"); + asm("and eax, 0x200"); + asm("cli"); + asm("ret"); + } + + +/** Restore interrupt mask to state preceding a DisableInterrupts() call + +@param aLevel Cookie returned by Disable(All)Interrupts() +*/ +EXPORT_C __NAKED__ void NKern::RestoreInterrupts(TInt aLevel) + { + asm("test byte ptr [esp+5], 2"); // test saved I flag + asm("jz restore_irq_off"); // jump if clear + asm("sti"); // else reenable interrupts + asm("ret"); + asm("restore_irq_off:"); + asm("cli"); + asm("ret"); + } + + +/** Enable all maskable interrupts + +@internalComponent +*/ +EXPORT_C __NAKED__ void NKern::EnableAllInterrupts() + { + asm("sti"); + asm("ret"); + } + + +/** Unlocks the kernel + Decrements iKernCSLocked; if it becomes zero and IDFCs or a reschedule are + pending, calls the scheduler to process them. + + @pre Thread or IDFC context. Don't call from ISRs. + */ +EXPORT_C __NAKED__ void NKern::Unlock() + { + asm("mov eax, ds:[%0]" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID)); // OK since kernel locked + asm("shr eax, 24 "); + asm("push esi "); + asm("mov esi, [eax*4+%0]" : : "i" (&SubSchedulerLookupTable)); +#ifdef _DEBUG + asm("cmp dword ptr [esi+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount)); + asm("jg short _dbg1 "); + asm("int 0xff "); + asm("_dbg1: "); +#endif + asm("cli "); + asm("dec dword ptr [esi+%0]" : : "i" _FOFF(TSubScheduler, iKernLockCount)); + asm("jnz short still_locked "); +// asm("cmp dword ptr [esi]TSubScheduler.iRescheduleNeededFlag, 0 VC6 ignores the "dword ptr" + asm("lea eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag)); + asm("cmp dword ptr [eax], 0 "); + asm("jz short no_resched "); + + asm("mov dword ptr [esi+%0], 1" : : "i" _FOFF(TSubScheduler, iKernLockCount)); + asm("push edi "); + asm("push ebp "); + asm("push ebx "); + asm("push gs "); + asm("push fs "); + asm("sti "); + + // Reschedule - return with local interrupts disabled, iKernLockCount=0 + asm("push 1 "); + asm("call %a0" : : "i" (TScheduler_Reschedule)); + asm("add esp, 4 "); + + asm("xor eax, eax "); + asm("lock xchg eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iReschedIPIs)); + asm("test eax, eax "); + asm("jz short no_resched_ipis_ul "); + + asm("unlock_do_resched_ipis: "); + asm("push eax "); + asm("call %a0" : : "i" (&send_resched_ipis)); + asm("add esp, 4 "); + + asm("no_resched_ipis_ul: "); + asm("pop fs "); + asm("pop gs "); + asm("pop ebx "); + asm("pop ebp "); + asm("pop edi "); + + asm("still_locked: "); + asm("sti "); + asm("pop esi "); + asm("ret "); + + asm("no_resched: "); + asm("xor eax, eax "); + asm("lock xchg eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iReschedIPIs)); + asm("test eax, eax "); + asm("jz short still_locked "); + asm("push edi "); + asm("push ebp "); + asm("push ebx "); + asm("push gs "); + asm("push fs "); + asm("jmp short unlock_do_resched_ipis "); + } + + +/** Locks the kernel + Defer IDFCs and preemption + + @pre Thread or IDFC context. Don't call from ISRs. + */ +EXPORT_C __NAKED__ void NKern::Lock() + { + asm("cli"); // stop thread migration between reading APIC ID and subscheduler stuff + asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID)); + asm("shr eax, 24"); + asm("mov ecx, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable)); + asm("inc dword ptr [ecx+%0]": : "i"_FOFF(TSubScheduler, iKernLockCount)); + asm("sti"); + asm("ret"); + } + + +/** Locks the kernel and returns a pointer to the current thread + Defer IDFCs and preemption + + @pre Thread or IDFC context. Don't call from ISRs. + */ +EXPORT_C __NAKED__ NThread* NKern::LockC() + { + asm("cli"); // stop thread migration between reading APIC ID and subscheduler stuff + asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID)); + asm("shr eax, 24"); + asm("mov ecx, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable)); + asm("inc dword ptr [ecx+%0]": : "i"_FOFF(TSubScheduler, iKernLockCount)); + asm("mov eax, [ecx+%0]" : : "i"_FOFF(TSubScheduler, iCurrentThread)); + asm("sti"); + asm("ret"); + } + + +/** Allows IDFCs and rescheduling if they are pending. + If IDFCs or a reschedule are pending and iKernCSLocked is exactly equal to 1 + calls the scheduler to process the IDFCs and possibly reschedule. + + @return Nonzero if a reschedule actually occurred, zero if not. + @pre Thread or IDFC context. Don't call from ISRs. + */ +EXPORT_C __NAKED__ TInt NKern::PreemptionPoint() + { + asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID)); + asm("shr eax, 24"); + asm("mov ecx, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable)); +#ifdef _DEBUG + asm("cmp dword ptr [ecx+%0], 0": : "i"_FOFF(TSubScheduler, iKernLockCount)); + asm("jg _dbg1_pp"); + asm("int 0xff"); + asm("_dbg1_pp:"); +#endif + asm("cmp dword ptr [ecx+%0], 1": : "i"_FOFF(TSubScheduler, iKernLockCount)); + asm("jnz still_locked_pp"); +// asm("cmp dword ptr [ecx]TSubScheduler.iRescheduleNeededFlag, 0 VC6 ignores the "dword ptr" + asm("lea eax, [ecx+%0]": : "i"_FOFF(TSubScheduler, iRescheduleNeededFlag)); + asm("cmp dword ptr [eax], 0"); + asm("jnz do_resched"); + asm("cli"); + asm("lock xchg eax, [ecx+%0]": : "i"_FOFF(TSubScheduler, iReschedIPIs)); + asm("test eax, eax"); + asm("jz pp_no_resched_ipis"); + asm("push eax"); + asm("call %a0": :"i"(&send_resched_ipis)); + asm("add esp, 4"); + asm("pp_no_resched_ipis:"); + asm("sti"); + + asm("still_locked_pp:"); + asm("xor eax, eax"); + asm("ret"); + + asm("do_resched:"); + asm("call %a0" : : "i"(NKern_Unlock)); + asm("call %a0" : : "i"(NKern_Lock)); + asm("mov eax, 1"); + asm("ret"); + } + + +/** Complete the saving of a thread's context + +This saves the FPU registers if necessary once we know that we are definitely +switching threads. + +@internalComponent +*/ +__NAKED__ void NThread::CompleteContextSave() + { + THISCALL_PROLOG0() + asm("mov edx, [ecx+%0]": : "i"_FOFF(NThreadBase,iSavedSP)); // EDX points to saved state on thread stack + asm("test byte ptr [edx], 8"); // test thread's saved TS flag + asm("jnz no_fpu"); // if set, thread did not use FPU + asm("clts"); + asm("fnsave [ecx+%0]": : "i"_FOFF(NThread, iCoprocessorState)); // else thread did use FPU - save its state + asm("or byte ptr [edx], 8"); // set TS flag so thread aborts next time it uses FPU + asm("fwait"); + + asm("no_fpu:"); + THISCALL_EPILOG0() + } + + +/** Check if the kernel is locked the specified number of times. + + @param aCount The number of times the kernel should be locked + If zero, tests if it is locked at all + @return TRUE if the tested condition is true. + + @internalTechnology +*/ +EXPORT_C __NAKED__ TBool NKern::KernelLocked(TInt /*aCount*/) + { + asm("pushfd"); + asm("cli"); + asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID)); + asm("shr eax, 24"); + asm("mov eax, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable)); + asm("mov edx, [eax+%0]": : "i"_FOFF(TSubScheduler, iKernLockCount)); + asm("popfd"); + asm("cmp edx, 0"); + asm("jz not_locked"); + asm("mov eax, [esp+4]"); + asm("cmp eax, 0"); + asm("jz locked"); + asm("cmp eax, edx"); + asm("jnz not_locked"); + asm("locked:"); + asm("mov eax, 1"); + asm("ret"); + asm("not_locked:"); + asm("xor eax, eax"); + asm("ret"); + } + + +// Only call this if thread migration is disabled, i.e. +// interrupts disabled, kernel locked or current thread in 'freeze cpu' mode +extern "C" __NAKED__ TSubScheduler& SubScheduler() + { + asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID)); + asm("shr eax, 24"); + asm("mov eax, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable)); + asm("ret"); + } + +/** Returns the NThread control block for the currently scheduled thread. + + Note that this is the calling thread if called from a thread context, or the + interrupted thread if called from an interrupt context. + + @return A pointer to the NThread for the currently scheduled thread. + + @pre Call in any context. +*/ +EXPORT_C __NAKED__ NThread* NKern::CurrentThread() + { + asm("pushfd"); + asm("cli"); // stop thread migration between reading APIC ID and thread pointer + asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID)); + asm("shr eax, 24"); + asm("mov eax, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable)); + asm("cmp eax, 0"); + asm("jz done"); + asm("test al, 3"); + asm("jnz bad_ct"); + asm("mov eax, [eax+%0]": : "i"_FOFF(TSubScheduler, iCurrentThread)); + asm("done:"); + asm("popfd"); + asm("ret"); + asm("bad_ct:"); + asm("popfd"); + asm("xor eax, eax"); + asm("ret"); + } + + +/** Returns the NThread control block for the currently scheduled thread. + + Note that this is the calling thread if called from a thread context, or the + interrupted thread if called from an interrupt context. + + @return A pointer to the NThread for the currently scheduled thread. + + @pre Call with migration disabled - i.e. from an ISR, IDFC, with interrupts + disabled or with preemption disabled. +*/ +extern "C" __NAKED__ NThread* NCurrentThreadL() + { + asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID)); + asm("shr eax, 24"); + asm("mov eax, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable)); + asm("mov eax, [eax+%0]": : "i"_FOFF(TSubScheduler, iCurrentThread)); + asm("ret"); + } + + +/** Returns the CPU number of the calling CPU. + + @return the CPU number of the calling CPU. + + @pre Call in any context. +*/ +EXPORT_C __NAKED__ TInt NKern::CurrentCpu() + { + asm("xor eax, eax"); + asm("str ax"); + asm("sub al, 0x28"); + asm("shr al, 3"); + asm("ret"); + } + + +/** Return the current processor context type (thread, IDFC or interrupt) + + @return A value from NKern::TContext enumeration (but never EEscaped) + @pre Any context + + @see NKern::TContext + */ +EXPORT_C __NAKED__ TInt NKern::CurrentContext() + { + asm("pushfd"); + asm("cli"); // stop thread migration between reading APIC ID and subscheduler stuff + asm("mov edx, ds:[%0]": :"i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID)); + asm("xor eax, eax"); + asm("shr edx, 24"); + asm("mov edx, [edx*4+%0]" : : "i"(&SubSchedulerLookupTable)); + asm("cmp edx, eax"); + asm("jz bad_cc"); + asm("test dl, 3"); + asm("jnz bad_cc"); + asm("cmp eax, [edx+52+%0]": : "i"_FOFF(TSubScheduler,iExtras)); // i_IrqNestCount + asm("jle irq"); + asm("cmp al, [edx+%0]": : "i"_FOFF(TSubScheduler, iInIDFC)); + asm("jz thread"); + asm("jmp idfc"); + + asm("bad_cc:"); // no subscheduler yet [initialising] - return EInterrupt + asm("irq:"); // return NKern::EInterrupt [=2] + asm("inc eax"); + asm("idfc:"); // return NKern::EIDFC [=1] + asm("inc eax"); + asm("thread:"); // return NKern::EThread [=0] + asm("popfd"); + asm("ret"); + } + + +#ifdef __USE_LOGICAL_DEST_MODE__ +extern "C" __NAKED__ void __fastcall do_send_resched_ipis(TUint32) + { + asm("shl ecx, 24 "); // CPUs mask into bits 24-31 + asm("jz short sri0 "); // no CPUs, so nothing to do + asm("pushfd "); + asm("cli "); + asm("mov ds:[%0], ecx" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ICRH)); + asm("mov eax, %0" : : "i" (RESCHED_IPI_VECTOR | 0x4800)); + asm("mov ds:[%0], eax" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ICRL)); + asm("popfd "); + asm("sri0: "); + asm("ret "); + } +#endif + +extern "C" __NAKED__ void __fastcall send_ipi(TUint32) + { + asm("pushfd "); + asm("cli "); + asm("mov ds:[%0], ecx" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ICRH)); + asm("mov eax, %0" : : "i" (RESCHED_IPI_VECTOR | 0x4000)); + asm("mov ds:[%0], eax" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ICRL)); + asm("popfd "); + asm("ret "); + } + +// Send a reschedule IPI to the current processor +// *** DON'T DO ANY TRACING OR INSTRUMENTATION *** +extern "C" __NAKED__ void send_self_resched_ipi() + { + asm("pushfd "); + asm("cli "); + asm("xor ecx, ecx "); + asm("mov ds:[%0], ecx" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ICRH)); + asm("mov eax, %0" : : "i" (RESCHED_IPI_VECTOR | 0x44000)); // destination shorthand = self + asm("mov ds:[%0], eax" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ICRL)); + asm("popfd "); + asm("ret "); + } + +extern "C" __NAKED__ void send_irq_ipi(TSubScheduler*) + { + asm("mov ecx, [esp+4] "); + asm("pushfd "); + asm("mov edx, [ecx+%0]" : : "i" _FOFF(TSubScheduler, i_APICID)); + asm("cli "); + asm("mov ds:[%0], edx" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ICRH)); + asm("mov eax, %0" : : "i" (TRANSFERRED_IRQ_VECTOR | 0x4000)); + asm("mov ds:[%0], eax" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ICRL)); + asm("popfd "); + asm("ret "); + } +