// Copyright (c) 1998-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

//

#include "memmodel.h"

#include "kernel/cache_maintenance.inl"

#include <kernel/cache.h>

#include <ramalloc.h>

#include <defrag.h>

#include "mm.h"

#include "mmu.h"

#include "mpager.h"

#include "mmapping.h"

#include "mobject.h"

#include "mmanager.h"

#include "mpagearray.h"

//

// SPageInfo

//

// check enough space for page infos...

__ASSERT_COMPILE((KPageInfoLinearEnd-KPageInfoLinearBase)/sizeof(SPageInfo)==(1<<(32-KPageShift)));

// check KPageInfoShift...

__ASSERT_COMPILE(sizeof(SPageInfo)==(1<<KPageInfoShift));

SPageInfo* SPageInfo::SafeFromPhysAddr(TPhysAddr aAddress)

	{

	__NK_ASSERT_DEBUG((aAddress&KPageMask)==0);

	TUint index = aAddress>>(KPageShift+KPageShift-KPageInfoShift);

	TUint flags = ((TUint8*)KPageInfoMap)[index>>3];

	TUint mask = 1<<(index&7);

	if(!(flags&mask))

		return 0; // no SPageInfo for aAddress

	SPageInfo* info = FromPhysAddr(aAddress);

	if(info->iType==SPageInfo::EInvalid)

		return 0;

	return info;

	}

#ifdef _DEBUG

void SPageInfo::CheckAccess(const char* aMessage, TUint aFlags)

	{

	if(K::Initialising || NKern::Crashed())

		return;

	if((aFlags&ECheckNotAllocated) && (iType!=EUnknown))

		{

		Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x, iType==%d : %s",this,PhysAddr(),iType,aMessage);

		__NK_ASSERT_DEBUG(0);

		goto fail;

		}

	if((aFlags&ECheckNotUnused) && (iType==EUnused))

		{

		Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x, iType==%d : %s",this,PhysAddr(),iType,aMessage);

		__NK_ASSERT_DEBUG(0);

		goto fail;

		}

	if((aFlags&ECheckUnused) && (iType!=EUnused))

		{

		Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x, iType==%d : %s",this,PhysAddr(),iType,aMessage);

		__NK_ASSERT_DEBUG(0);

		goto fail;

		}

	if((aFlags&ECheckNotPaged) && (iPagedState!=EUnpaged))

		{

		Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x, iPagedState=%d : %s",this,PhysAddr(),iPagedState,aMessage);

		__NK_ASSERT_DEBUG(0);

		goto fail;

		}

	if((aFlags&ECheckRamAllocLock) && !RamAllocLock::IsHeld())

		{

		Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x, iType==%d : %s",this,PhysAddr(),iType,aMessage);

		__NK_ASSERT_DEBUG(0);

		goto fail;

		}

	if((aFlags&ENoCheckMmuLock) || MmuLock::IsHeld())

		return;

fail:

	Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x : %s",this,PhysAddr(),aMessage);

	Mmu::Panic(Mmu::EUnsafePageInfoAccess);

	}

void SPageInfo::Dump()

	{

	Kern::Printf("SPageInfo for page %x = %d,%d,%02x,0x%08x,0x%x,%d",PhysAddr(),iType,iPagedState,iFlags,iOwner,iIndex,iPinCount);

	}

#endif

//

// SPageTableInfo

//

// check enough space for page table infos...

__ASSERT_COMPILE((KPageTableInfoEnd-KPageTableInfoBase)/sizeof(SPageTableInfo)

					>=(KPageTableEnd-KPageTableBase)/KPageTableSize);

// check KPtBlockShift...

__ASSERT_COMPILE((sizeof(SPageTableInfo)<<KPtBlockShift)==KPageSize);

#ifdef _DEBUG

TBool SPageTableInfo::CheckPageCount()

	{

	__NK_ASSERT_DEBUG(MmuLock::IsHeld());

	TPte* pt = PageTable();

	TUint realCount = 0;

	do if(*pt++) ++realCount;

	while(TLinAddr(pt)&(KPageTableMask/sizeof(TPte)*sizeof(TPte)));

	if(iPageCount==realCount)

		return true;

	Kern::Printf("CheckPageCount Failed: pt=0x%08x count=%d realCount=%d",TLinAddr(pt)-KPageTableSize,iPageCount,realCount);

	return false;

	}

void SPageTableInfo::CheckChangeUse(const char* aName)

	{

	if(K::Initialising)

		return;

	if(PageTablesLockIsHeld() && MmuLock::IsHeld())

		return;

	Kern::Printf("SPageTableInfo::CheckChangeUse failed : %s",aName);

	Mmu::Panic(Mmu::EUnsafePageTableInfoAccess);

	}

void SPageTableInfo::CheckCheckUse(const char* aName)

	{

	if(K::Initialising)

		return;

	if(PageTablesLockIsHeld() || MmuLock::IsHeld())

		return;

	Kern::Printf("SPageTableInfo::CheckCheckUse failed : %s",aName);

	Mmu::Panic(Mmu::EUnsafePageTableInfoAccess);

	}

void SPageTableInfo::CheckAccess(const char* aName)

	{

	if(K::Initialising)

		return;

	if(MmuLock::IsHeld())

		return;

	Kern::Printf("SPageTableInfo::CheckAccess failed : %s",aName);

	Mmu::Panic(Mmu::EUnsafePageTableInfoAccess);

	}

void SPageTableInfo::CheckInit(const char* aName)

	{

	if(K::Initialising)

		return;

	if(PageTablesLockIsHeld() && iType==EUnused)

		return;

	Kern::Printf("SPageTableInfo::CheckInit failed : %s",aName);

	Mmu::Panic(Mmu::EUnsafePageTableInfoAccess);

	}

#endif

//

// RamAllocLock

//

_LIT(KLitRamAlloc,"RamAlloc");

_LIT(KLitPhysMemSync,"PhysMemSync");

void RamAllocLock::Lock()

	{

	Mmu& m = TheMmu;

	Kern::MutexWait(*m.iRamAllocatorMutex);

	if(!m.iRamAllocLockCount++)

		{

		// first lock, so setup memory fail data...

		m.iRamAllocFailed = EFalse;

		__NK_ASSERT_DEBUG(m.iRamAllocInitialFreePages==m.FreeRamInPages()); // free RAM shouldn't have changed whilst lock was held

		}

	}

void RamAllocLock::Unlock()

	{

	Mmu& m = TheMmu;

	if(--m.iRamAllocLockCount)

		{

		Kern::MutexSignal(*m.iRamAllocatorMutex);

		return;

		}

	TBool failed = m.iRamAllocFailed;

	TUint initial = m.iRamAllocInitialFreePages;

	TUint final = m.FreeRamInPages();

	m.iRamAllocInitialFreePages = final; // new baseline value

	TUint changes = K::CheckFreeMemoryLevel(initial*KPageSize,final*KPageSize,failed);

	if(changes)

		{

		__KTRACE_OPT(KMMU,Kern::Printf("RamAllocLock::Unlock() changes=%x",changes));

		}

	Kern::MutexSignal(*m.iRamAllocatorMutex);

	}

TBool RamAllocLock::Flash()

	{

	Unlock();

	Lock();

	return true; // lock was released

	}

TBool RamAllocLock::IsHeld()

	{

	Mmu& m = TheMmu;

	return m.iRamAllocatorMutex->iCleanup.iThread == &Kern::CurrentThread() && m.iRamAllocLockCount;

	}

//

// MmuLock

//

#ifdef _DEBUG

TUint MmuLock::UnlockGuardNest =0;

TUint MmuLock::UnlockGuardFail =0;

#endif

NFastMutex MmuLock::iLock;

void MmuLock::Lock()

	{

	NKern::FMWait(&iLock);

	}

void MmuLock::Unlock()

	{

	UnlockGuardCheck();

	NKern::FMSignal(&iLock);

	}

TBool MmuLock::Flash()

	{

	UnlockGuardCheck();

	return NKern::FMFlash(&iLock);

	}

TBool MmuLock::IsHeld()

	{

	NFastMutex& m = iLock;

	return m.HeldByCurrentThread();

	}

//

// Initialisation

//

Mmu TheMmu;

void Mmu::Init1Common()

	{

	__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Mmu::Init1Common"));

	// Mmu data

	TUint pteType = PteType(ESupervisorReadWrite,true);

	iTempPteCached = BlankPte((TMemoryAttributes)(EMemoryAttributeNormalCached|EMemoryAttributeDefaultShareable),pteType);

	iTempPteUncached = BlankPte((TMemoryAttributes)(EMemoryAttributeNormalUncached|EMemoryAttributeDefaultShareable),pteType);

	iTempPteCacheMaintenance = BlankPte((TMemoryAttributes)(CacheMaintenance::TemporaryMapping()|EMemoryAttributeDefaultShareable),pteType);

	// other

	PP::MaxUserThreadStack=0x14000;			// 80K - STDLIB asks for 64K for PosixServer!!!!

	PP::UserThreadStackGuard=0x2000;		// 8K

	PP::MaxStackSpacePerProcess=0x200000;	// 2Mb

	K::SupervisorThreadStackSize=0x1000;	// 4K

	PP::SupervisorThreadStackGuard=0x1000;	// 4K

	K::MachineConfig=(TMachineConfig*)KMachineConfigLinAddr;

	PP::RamDriveStartAddress=0;

	PP::RamDriveRange=0;

	PP::RamDriveMaxSize=0x20000000;	// 512MB, probably will be reduced later

	K::MemModelAttributes=EMemModelTypeFlexible|EMemModelAttrNonExProt|EMemModelAttrKernProt|EMemModelAttrWriteProt|

						EMemModelAttrVA|EMemModelAttrProcessProt|EMemModelAttrSameVA|EMemModelAttrSvKernProt|

						EMemModelAttrIPCKernProt|EMemModelAttrRamCodeProt;

	}

#if 0

void Mmu::VerifyRam()

	{

	Kern::Printf("Mmu::VerifyRam() pass 1");

	RamAllocLock::Lock();

	TPhysAddr p = 0;

	do

		{

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(p);

		if(pi)

			{

			Kern::Printf("%08x %d",p,pi->Type());

			if(pi->Type()==SPageInfo::EUnused)

				{

				volatile TPhysAddr* b = (volatile TPhysAddr*)MapTemp(p,0);

				b[0] = p;

				b[1] = ~p;

				__NK_ASSERT_DEBUG(b[0]==p);

				__NK_ASSERT_DEBUG(b[1]==~p);

				UnmapTemp();

				}

			}

		p += KPageSize;

		}

	while(p);

	TBool fail = false;

	Kern::Printf("Mmu::VerifyRam() pass 2");

	do

		{

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(p);

		if(pi)

			{

			if(pi->Type()==SPageInfo::EUnused)

				{

				volatile TPhysAddr* b = (volatile TPhysAddr*)MapTemp(p,0);

				if(b[0]!=p || b[1]!=~p)

					{

					fail = true;

					Kern::Printf("%08x FAILED %x %x",b[0],b[1]);

					}

				UnmapTemp();

				}

			}

		p += KPageSize;

		}

	while(p);

	__NK_ASSERT_DEBUG(!fail);

	RamAllocLock::Unlock();

	}

#endif

void Mmu::Init2Common()

	{

	__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Mmu::Init2Common"));

	// create allocator...

	const SRamInfo& info = *(const SRamInfo*)TheSuperPage().iRamBootData;

	iRamPageAllocator = DRamAllocator::New(info, iRamZones, iRamZoneCallback);

	// initialise all pages in banks as unused...

	const SRamBank* bank = info.iBanks;

	while(bank->iSize)

		{

		TUint32 base = bank->iBase;

		TUint32 size = bank->iSize;

		__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Found RAM bank 0x%08x size %d",base,size));

		if(base+size<=base || ((base|size)&KPageMask))

			Panic(EInvalidRamBankAtBoot);

		SPageInfo* pi = SPageInfo::FromPhysAddr(base);

		SPageInfo* piEnd = pi+(size>>KPageShift);

		while(pi<piEnd)

			(pi++)->SetUnused();

		++bank;

		}

	// step over the last bank to get to the reserved banks.

	++bank;

	// mark any reserved regions as allocated...

	while(bank->iSize)

		{

		TUint32 base = bank->iBase;

		TUint32 size = bank->iSize;

		__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Found reserved bank 0x%08x size %d",base,size));

		if(base+size<=base || ((base|size)&KPageMask))

			Panic(EInvalidReservedBankAtBoot);

		SPageInfo* pi = SPageInfo::FromPhysAddr(base);

		SPageInfo* piEnd = pi+(size>>KPageShift);

		while(pi<piEnd)

			(pi++)->SetPhysAlloc();

		++bank;

		}

	// Clear the inital (and only so far) page table info page so all unused

	// page tables infos will be marked as unused.

	__ASSERT_COMPILE(SPageTableInfo::EUnused == 0);

	memclr((TAny*)KPageTableInfoBase, KPageSize);

	// look for page tables - assume first page table maps page tables

	TPte* pPte = (TPte*)KPageTableBase;

	TInt i;

	for(i=0; i<KChunkSize/KPageSize; ++i)

		{

		TPte pte = *pPte++;

		if(pte==KPteUnallocatedEntry)	// after boot, page tables are contiguous

			break;

		TPhysAddr ptpgPhys = Mmu::PtePhysAddr(pte,i);

		__KTRACE_OPT(KBOOT,Kern::Printf("Page Table Group %08x -> Phys %08x", KPageTableBase+i*KPageSize, ptpgPhys));

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(ptpgPhys);

		__ASSERT_ALWAYS(pi, Panic(EInvalidPageTableAtBoot));

		pi->SetFixed(i); // this also sets the SPageInfo::iOffset so that linear-to-physical works

		}

	// look for mapped pages

	TPde* pd = Mmu::PageDirectory(KKernelOsAsid);

	for(i=0; i<(1<<(32-KChunkShift)); ++i)

		{

		TPde pde = pd[i];

		if(pde==KPdeUnallocatedEntry)

			continue;

		TPhysAddr pdePhys = Mmu::PdePhysAddr(pde);

		TPte* pt = 0;

		if(pdePhys!=KPhysAddrInvalid)

			{

			__KTRACE_OPT(KBOOT,Kern::Printf("Addr %08x -> Whole PDE Phys %08x", i<<KChunkShift, pdePhys));

			}

		else

			{

			pt = Mmu::PageTableFromPde(pde);

			__KTRACE_OPT(KBOOT,Kern::Printf("Addr %08x -> page table %08x", i<<KChunkShift, pt));

			__ASSERT_ALWAYS(pt,Panic(EInvalidPdeAtBoot)); // bad PDE

			}

		TInt j;

		TInt np = 0;

		for(j=0; j<KChunkSize/KPageSize; ++j)

			{

			TBool present = ETrue;	// all pages present if whole PDE mapping

			TPte pte = 0;

			if(pt)

				{

				pte = pt[j];

				present = pte!=KPteUnallocatedEntry;

				}

			if(present)

				{

				++np;

				TPhysAddr pa = pt ? Mmu::PtePhysAddr(pte,j) : (pdePhys + (j<<KPageShift));

				SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pa);

				__KTRACE_OPT(KBOOT,Kern::Printf("Addr: %08x PA=%08x",

													(i<<KChunkShift)+(j<<KPageShift), pa));

				if(pi)	// ignore non-RAM mappings

					{

					TInt r = iRamPageAllocator->MarkPageAllocated(pa, EPageFixed);

					// allow KErrAlreadyExists since it's possible that a page is doubly mapped

					__ASSERT_ALWAYS(r==KErrNone || r==KErrAlreadyExists, Panic(EBadMappedPageAfterBoot));

					if(pi->Type()==SPageInfo::EUnused)

						pi->SetFixed();

					}

				}

			}

		__KTRACE_OPT(KBOOT,Kern::Printf("Addr: %08x #PTEs=%d",(i<<KChunkShift),np));

		if(pt)

			{

			SPageTableInfo* pti = SPageTableInfo::FromPtPtr(pt);

			pti->Boot(np);

			}

		}

	TInt r = K::MutexCreate(iRamAllocatorMutex, KLitRamAlloc, NULL, EFalse, KMutexOrdRamAlloc);

	if(r!=KErrNone)

		Panic(ERamAllocMutexCreateFailed);

	iRamAllocLockCount = 0;

	iRamAllocInitialFreePages = FreeRamInPages();

	__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Mmu::DoInit2"));

	for(i=0; i<KNumTempMappingSlots; ++i)

		iTempMap[i].Alloc(1);

	iPhysMemSyncTemp.Alloc(1);

	r = K::MutexCreate(iPhysMemSyncMutex, KLitPhysMemSync, NULL, EFalse, KMutexOrdSyncPhysMem);

	if(r!=KErrNone)

		Panic(EPhysMemSyncMutexCreateFailed);

//	VerifyRam();

	}

void Mmu::Init2FinalCommon()

	{

	__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Mmu::Init2FinalCommon"));

	// hack, reduce free memory to <2GB...

	while(FreeRamInPages()>=0x80000000/KPageSize)

		{

		TPhysAddr dummyPage;

		TInt r = iRamPageAllocator->AllocRamPages(&dummyPage,1, EPageFixed);

		__NK_ASSERT_ALWAYS(r==KErrNone);

		}

	// hack, reduce total RAM to <2GB...

	if(TheSuperPage().iTotalRamSize<0)

		TheSuperPage().iTotalRamSize = 0x80000000-KPageSize;

	// Save current free RAM size - there can never be more free RAM than this

	TUint maxFreePages = FreeRamInPages();

	K::MaxFreeRam = maxFreePages*KPageSize;

	if(maxFreePages < (TUint(PP::RamDriveMaxSize)>>KPageShift))

		PP::RamDriveMaxSize = maxFreePages*KPageSize;

	// update this to stop assert triggering in RamAllocLock::Lock()

	iRamAllocInitialFreePages = maxFreePages;

	}

void Mmu::Init3()

	{

	iDefrag = new Defrag;

	if (!iDefrag)

		Panic(EDefragAllocFailed);

	iDefrag->Init3(TheMmu.iRamPageAllocator);

	}

//

// Utils

//

void Mmu::Panic(TPanic aPanic)

	{

	Kern::Fault("MMU",aPanic);

	}

TUint Mmu::FreeRamInPages()

	{

	return iRamPageAllocator->FreeRamInPages()+ThePager.NumberOfFreePages();

	}

TUint Mmu::TotalPhysicalRamPages()

	{

	return iRamPageAllocator->TotalPhysicalRamPages();

	}

const SRamZone* Mmu::RamZoneConfig(TRamZoneCallback& aCallback) const

	{

	aCallback = iRamZoneCallback;

	return iRamZones;

	}

void Mmu::SetRamZoneConfig(const SRamZone* aZones, TRamZoneCallback aCallback)

	{

	iRamZones = aZones;

	iRamZoneCallback = aCallback;

	}

TInt Mmu::ModifyRamZoneFlags(TUint aId, TUint aClearMask, TUint aSetMask)

	{

	return iRamPageAllocator->ModifyZoneFlags(aId, aClearMask, aSetMask);

	}

TInt Mmu::GetRamZonePageCount(TUint aId, SRamZonePageCount& aPageData)

	{

	return iRamPageAllocator->GetZonePageCount(aId, aPageData);

	}

TInt Mmu::ZoneAllocPhysicalRam(TUint* aZoneIdList, TUint aZoneIdCount, TInt aBytes, TPhysAddr& aPhysAddr, TInt aAlign)

	{

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::ZoneAllocPhysicalRam(?,%d,%d,?,%d)", aZoneIdCount, aBytes, aPhysAddr, aAlign));

	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());

	TInt r = iRamPageAllocator->ZoneAllocContiguousRam(aZoneIdList, aZoneIdCount, aBytes, aPhysAddr, EPageFixed, aAlign);

	if(r!=KErrNone)

		iRamAllocFailed = ETrue;

	else

		{

		TUint pages = MM::RoundToPageCount(aBytes);

		AllocatedPhysicalRam(aPhysAddr, pages,  (Mmu::TRamAllocFlags)EMemAttStronglyOrdered);

		}

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::ZoneAllocPhysicalRam returns %d and aPhysAddr=0x%08x",r,aPhysAddr));

	return r;

	}

TInt Mmu::ZoneAllocPhysicalRam(TUint* aZoneIdList, TUint aZoneIdCount, TInt aNumPages, TPhysAddr* aPageList)

	{

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::ZoneAllocPhysicalRam(?,%d,%d,?)", aZoneIdCount, aNumPages));

	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());

	TInt r = iRamPageAllocator->ZoneAllocRamPages(aZoneIdList, aZoneIdCount, aPageList, aNumPages, EPageFixed);

	if(r!=KErrNone)

		iRamAllocFailed = ETrue;

	else

		{

		PagesAllocated(aPageList, aNumPages, (Mmu::TRamAllocFlags)EMemAttStronglyOrdered);

		// update page infos...

		TUint flash = 0;

		TPhysAddr* pageEnd = aPageList + aNumPages;

		MmuLock::Lock();

		TPhysAddr* page = aPageList;

		while (page < pageEnd)

			{

			MmuLock::Flash(flash,KMaxPageInfoUpdatesInOneGo/2);

			TPhysAddr pagePhys = *page++;

			__NK_ASSERT_DEBUG(pagePhys != KPhysAddrInvalid);

			SPageInfo::FromPhysAddr(pagePhys)->SetPhysAlloc();

			}

		MmuLock::Unlock();

		}

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::ZoneAllocPhysicalRam returns %d",r));

	return r;

	}

TInt Mmu::RamHalFunction(TInt aFunction, TAny* a1, TAny* a2)

	{

	// This function should only be registered with hal and therefore can only 

	// be invoked after the ram allocator has been created.

	__NK_ASSERT_DEBUG(iRamPageAllocator);

	return iRamPageAllocator->HalFunction(aFunction, a1, a2);

	}

void Mmu::ChangePageType(SPageInfo* aPageInfo, TZonePageType aOldPageType, TZonePageType aNewPageType)

	{

	iRamPageAllocator->ChangePageType(aPageInfo, aOldPageType, aNewPageType);

	}

TInt Mmu::HandlePageFault(TLinAddr aPc, TLinAddr aFaultAddress, TUint aAccessPermissions, TAny* aExceptionInfo)

	{

	TRACE(("Mmu::HandlePageFault(0x%08x,0x%08x,%d)",aPc,aFaultAddress,aAccessPermissions));

	DMemModelThread* thread = (DMemModelThread*)TheCurrentThread;

	// Get the os asid of the process taking the fault, no need to open a reference 

	// as it is the current thread's process so can't be freed.

	TUint faultOsAsid = ((DMemModelProcess*)thread->iNThread.iAddressSpace)->OsAsid();

	// check if any fast mutexes held...

	NFastMutex* fm = NKern::HeldFastMutex();

	TPagingExcTrap* trap = thread->iPagingExcTrap;

	if(fm)

		{

		// check there is an XTRAP_PAGING in effect...

		if(!trap)

			{

			// oops, kill system...

			__KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("Fault with FM Held! addr=0x%08x (%O pc=%x)",aFaultAddress,thread,aPc));

			Exc::Fault(aExceptionInfo);

			}

		// release the fast mutex...

		NKern::FMSignal(fm);

		}

	NKern::ThreadEnterCS();

	// work out address space for aFaultAddress...

	TUint osAsid = faultOsAsid;

	TLinAddr addr = aFaultAddress;

	if(thread->iAliasLinAddr && TUint(addr - thread->iAliasLinAddr) < TUint(KPageSize))

		{

		// Address in aliased memory...

		addr = (addr - thread->iAliasLinAddr) + thread->iAliasTarget;

		// Get the os asid of the process thread is aliasing, no need to open 

		// a reference on it as one was already opened when the alias was created.

		osAsid = thread->iAliasProcess->OsAsid();

		}

	else if(addr>=KGlobalMemoryBase)

		{

		// Address in global region, so look it up in kernel's address space...

		osAsid = KKernelOsAsid;

		}

	// NOTE, osAsid will remain valid for duration of this function because it is either

	// - The current thread's address space, which can't go away whilst the thread

	//   is running.

	// - The address space of another thread which we are aliasing memory from,

	//   and we would only do this if we have a reference on this other thread,

	//   which has a reference on it's process, which should own the address space!

#ifdef __BROADCAST_CACHE_MAINTENANCE__

	TInt aliasAsid = -1;

	if (thread->iAliasLinAddr)

		{

		// If an alias is in effect, the the thread will be locked to the current CPU,

		// but we need to be able to migrate between CPUs for cache maintainance.  This

		// must be dealt with by removing the alias and restoring it with a paging trap

		// handler.

		if(!trap)

			{

			// oops, kill system...

			__KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("Fault with thread locked to current CPU! addr=0x%08x (%O pc=%x)",aFaultAddress,thread,aPc));

			Exc::Fault(aExceptionInfo);

			}

		// Open a reference on the aliased process's os asid before removing the alias

		// so that the address space can't be freed while we try to access its members.

		aliasAsid = thread->iAliasProcess->TryOpenOsAsid();

		// This should never fail as until we remove the alias there will 

		// always be at least one reference on the os asid.

		__NK_ASSERT_DEBUG(aliasAsid >= 0);

		thread->RemoveAlias();

		}

#endif

	// find mapping...

	TUint offsetInMapping;

	TUint mapInstanceCount;

	DMemoryMapping* mapping = MM::FindMappingInAddressSpace(osAsid, addr, 1, offsetInMapping, mapInstanceCount);

//	TRACE(("%O mapping=0x%08x",TheCurrentThread,mapping));

	TInt r = KErrNotFound;

	if(mapping)

		{

		MmuLock::Lock();

		// check if we need to process page fault...

		if(!Mmu::CheckPteTypePermissions(mapping->PteType(),aAccessPermissions) ||

			mapInstanceCount != mapping->MapInstanceCount())

			{

			// Invalid access to the page.

			MmuLock::Unlock();

			r = KErrAbort;

			}

		else

			{

			// Should not be able to take a fault on a pinned mapping if accessing it 

			// with the correct permissions.

			__NK_ASSERT_DEBUG(!mapping->IsPinned());

			// we do need to handle fault so is this a demand paging or page moving fault

			DMemoryObject* memory = mapping->Memory();

			if(!memory)

				MmuLock::Unlock();

			else

				{

				TUint faultIndex = (offsetInMapping >> KPageShift) + mapping->iStartIndex;

				memory->Open();

				// This is safe as we have the instance count so can detect the mapping 

				// being reused and we have a reference to the memory object so it can't 

				// be deleted.

				MmuLock::Unlock();

				if(memory->IsDemandPaged())

					{

					// Let the pager handle the fault...

					r = ThePager.HandlePageFault(	aPc, aFaultAddress, faultOsAsid, faultIndex,

													aAccessPermissions, memory, mapping, mapInstanceCount,

													thread, aExceptionInfo);

					}

				else

					{// The page could be being moved so verify that with its manager.

					DMemoryManager* manager = memory->iManager;

					r = manager->HandleFault(memory, faultIndex, mapping, mapInstanceCount, aAccessPermissions);

					}

				if (r == KErrNone)

					{// alias PDE needs updating because page tables have changed...

					thread->RefreshAlias();

					}

				memory->Close();

				}

			}

		mapping->Close();

		}

	if (trap)

		{

		// restore address space (because the trap will bypass any code

		// which would have done this.)...

		DMemModelThread::RestoreAddressSpace();

		}

#ifdef __BROADCAST_CACHE_MAINTENANCE__

	// Close any reference on the aliased process's os asid before we leave the

	// critical section.

	if (aliasAsid >= 0)

		{

		thread->iAliasProcess->CloseOsAsid();

		}

#endif

	NKern::ThreadLeaveCS();  // thread will die now if CheckRealtimeThreadFault caused a panic

	// deal with XTRAP_PAGING...

	if(trap)

		{

		// re-acquire any fast mutex which was held before the page fault...

		if(fm)

			NKern::FMWait(fm);

		if (r == KErrNone)

			{

			trap->Exception(1); // return from exception trap with result '1' (value>0)

			// code doesn't continue beyond this point.

			__NK_ASSERT_DEBUG(0);

			}

		}

	return r;

	}

//

// Memory allocation

//

TInt Mmu::AllocRam(	TPhysAddr* aPages, TUint aCount, TRamAllocFlags aFlags, TZonePageType aZonePageType, 

					TUint aBlockZoneId, TBool aBlockRest)

	{

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::AllocRam(?,%d,%x)",aCount,aFlags));

	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());

#ifdef _DEBUG

	if(K::CheckForSimulatedAllocFail())

		{

		__KTRACE_OPT(KMMU,Kern::Printf("Mmu::AllocRam returns simulated OOM %d",KErrNoMemory));

		return KErrNoMemory;

		}

#endif

	TInt missing = iRamPageAllocator->AllocRamPages(aPages, aCount, aZonePageType, aBlockZoneId, aBlockRest);

	if(missing && !(aFlags&EAllocNoPagerReclaim) && ThePager.GetFreePages(missing))

		missing = iRamPageAllocator->AllocRamPages(aPages, aCount, aZonePageType, aBlockZoneId, aBlockRest);

	TInt r = missing ? KErrNoMemory : KErrNone;

	if(r!=KErrNone)

		iRamAllocFailed = ETrue;

	else

		PagesAllocated(aPages,aCount,aFlags);

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::AllocRam returns %d",r));

	return r;

	}

void Mmu::FreeRam(TPhysAddr* aPages, TUint aCount, TZonePageType aZonePageType)

	{

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::FreeRam(?,%d)",aCount));

	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());

	// update page infos...

	TPhysAddr* pages = aPages;

	TPhysAddr* pagesEnd = pages+aCount;

	TPhysAddr* pagesOut = aPages;

	MmuLock::Lock();

	TUint flash = 0;

	while(pages<pagesEnd)

		{

		MmuLock::Flash(flash,KMaxPageInfoUpdatesInOneGo/2);

		TPhysAddr pagePhys = *pages++;

		__NK_ASSERT_DEBUG(pagePhys!=KPhysAddrInvalid);

		SPageInfo* pi = SPageInfo::FromPhysAddr(pagePhys);

		PageFreed(pi);

		// If this is an old page of a page being moved that was previously pinned

		// then make sure it is freed as discardable otherwise despite DPager::DonatePages()

		// having marked it as discardable it would be freed as movable.

		__NK_ASSERT_DEBUG(pi->PagedState() != SPageInfo::EPagedPinnedMoved || aCount == 1);

		if (pi->PagedState() == SPageInfo::EPagedPinnedMoved)

			aZonePageType = EPageDiscard;

		if(ThePager.PageFreed(pi)==KErrNone)

			--aCount; // pager has dealt with this page, so one less for us

		else

			{

			// All paged pages should have been dealt with by the pager above.

			__NK_ASSERT_DEBUG(pi->PagedState() == SPageInfo::EUnpaged);

			*pagesOut++ = pagePhys; // store page address for freeing later

			}

		}

	MmuLock::Unlock();

	iRamPageAllocator->FreeRamPages(aPages, aCount, aZonePageType);

	}

TInt Mmu::AllocContiguousRam(TPhysAddr& aPhysAddr, TUint aCount, TUint aAlign, TRamAllocFlags aFlags)

	{

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::AllocContiguousRam(?,0x%x,%d,%x)",aCount,aAlign,aFlags));

	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());

#ifdef _DEBUG

	if(K::CheckForSimulatedAllocFail())

		{

		__KTRACE_OPT(KMMU,Kern::Printf("Mmu::AllocContiguousRam returns simulated OOM %d",KErrNoMemory));

		return KErrNoMemory;

		}

	// Only the page sets EAllocNoPagerReclaim and it shouldn't allocate contiguous ram.

	__NK_ASSERT_DEBUG(!(aFlags&EAllocNoPagerReclaim));

#endif

	TInt r = iRamPageAllocator->AllocContiguousRam(aCount, aPhysAddr, EPageFixed, aAlign+KPageShift);

	if(r==KErrNoMemory && aCount > KMaxFreeableContiguousPages)

		{

		// flush paging cache and retry...

		ThePager.FlushAll();

		r = iRamPageAllocator->AllocContiguousRam(aCount, aPhysAddr, EPageFixed, aAlign+KPageShift);

		}

	if(r!=KErrNone)

		iRamAllocFailed = ETrue;

	else

		PagesAllocated((TPhysAddr*)(aPhysAddr|1), aCount, aFlags);

	__KTRACE_OPT(KMMU,Kern::Printf("AllocContiguouseRam returns %d and aPhysAddr=0x%08x",r,aPhysAddr));

	return r;

	}

void Mmu::FreeContiguousRam(TPhysAddr aPhysAddr, TUint aCount)

	{

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::FreeContiguousRam(0x%08x,0x%x)",aPhysAddr,aCount));

	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());

	__NK_ASSERT_DEBUG((aPhysAddr&KPageMask)==0);

	TUint pageCount = aCount;

	// update page infos...

	SPageInfo* pi = SPageInfo::FromPhysAddr(aPhysAddr);

	SPageInfo* piEnd = pi+pageCount;

	TUint flash = 0;

	MmuLock::Lock();

	while(pi<piEnd)

		{

		MmuLock::Flash(flash,KMaxPageInfoUpdatesInOneGo);

		PageFreed(pi++);

		}

	MmuLock::Unlock();

	// free pages...

	while(pageCount)

		{

		iRamPageAllocator->FreeRamPage(aPhysAddr, EPageFixed);

		aPhysAddr += KPageSize;

		--pageCount;

		}

	}

TInt Mmu::AllocPhysicalRam(TPhysAddr* aPages, TUint aCount, TRamAllocFlags aFlags)

	{

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::AllocPhysicalRam(?,%d,%x)",aCount,aFlags));

	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());

	// Allocate fixed pages as physically allocated pages aren't movable or discardable.

	TInt r = AllocRam(aPages, aCount, aFlags, EPageFixed);

	if (r!=KErrNone)

		return r;

	// update page infos...

	TPhysAddr* pages = aPages;

	TPhysAddr* pagesEnd = pages+aCount;

	MmuLock::Lock();

	TUint flash = 0;

	while(pages<pagesEnd)

		{

		MmuLock::Flash(flash,KMaxPageInfoUpdatesInOneGo/2);

		TPhysAddr pagePhys = *pages++;

		__NK_ASSERT_DEBUG(pagePhys!=KPhysAddrInvalid);

		SPageInfo* pi = SPageInfo::FromPhysAddr(pagePhys);

		pi->SetPhysAlloc();

		}

	MmuLock::Unlock();

	return KErrNone;

	}

void Mmu::FreePhysicalRam(TPhysAddr* aPages, TUint aCount)

	{

	__KTRACE_OPT(KMMU,Kern::Printf("Mmu::FreePhysicalRam(?,%d)",aCount));

	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());

	// update page infos...

	TPhysAddr* pages = aPages;

	TPhysAddr* pagesEnd = pages+aCount;

	MmuLock::Lock();

	TUint flash = 0;

	while(pages<pagesEnd)

		{

		MmuLock::Flash(flash,KMaxPageInfoUpdatesInOneGo/2);

		TPhysAddr pagePhys = *pages++;