// Copyright (c) 2002-2009 Nokia Corporation and/or its subsidiary(-ies).

 // All rights reserved.

 // This component and the accompanying materials are made available

 // under the terms of the License "Eclipse Public License v1.0"

 // which accompanies this distribution, and is available

 // at the URL "http://www.eclipse.org/legal/epl-v10.html".

 //

 // Initial Contributors:

 // Nokia Corporation - initial contribution.

 //

 // Contributors:

 //

 // Description:

 // e32\drivers\dmapil.cpp

 // DMA Platform Independent Layer (PIL)

 // 

 //

 #include <drivers/dma.h>

 #include <kernel/kern_priv.h>

 static const char KDmaPanicCat[] = "DMA";

 NFastMutex DmaChannelMgr::Lock;

 class TDmaCancelInfo : public SDblQueLink

 	{

 public:

 	TDmaCancelInfo();

 	void Signal();

 public:

 	NFastSemaphore iSem;

 	};

 TDmaCancelInfo::TDmaCancelInfo()

 	:	iSem(0)

 	{

 	iNext = this;

 	iPrev = this;

 	}

 void TDmaCancelInfo::Signal()

 	{

 	TDmaCancelInfo* p = this;

 	FOREVER

 		{

 		TDmaCancelInfo* next = (TDmaCancelInfo*)p->iNext;

 		if (p!=next)

 			p->Deque();

 		NKern::FSSignal(&p->iSem);	// Don't dereference p after this

 		if (p==next)

 			break;

 		p = next;

 		}

 	}

 //////////////////////////////////////////////////////////////////////////////

 #ifdef __DMASIM__

 #ifdef __WINS__

 typedef TLinAddr TPhysAddr;

 #endif

 static inline TPhysAddr LinToPhys(TLinAddr aLin) {return aLin;}

 #else

 static inline TPhysAddr LinToPhys(TLinAddr aLin) {return Epoc::LinearToPhysical(aLin);}

 #endif

 //

 // Return minimum of aMaxSize and size of largest physically contiguous block

 // starting at aLinAddr.

 //

 static TInt MaxPhysSize(TLinAddr aLinAddr, const TInt aMaxSize)

 	{

 	const TPhysAddr physBase = LinToPhys(aLinAddr);

 	TLinAddr lin = aLinAddr;

 	TInt size = 0;

 	for (;;)

 		{

 		// Round up the linear address to the next MMU page boundary

 		const TLinAddr linBoundary = Kern::RoundToPageSize(lin + 1);

 		size += linBoundary - lin;

 		if (size >= aMaxSize)

 			return aMaxSize;

 		if ((physBase + size) != LinToPhys(linBoundary))

 			return size;

 		lin = linBoundary;

 		}

 	}

 //////////////////////////////////////////////////////////////////////////////

 // TDmac

 TDmac::TDmac(const SCreateInfo& aInfo)

 	: iMaxDesCount(aInfo.iDesCount),

 	  iAvailDesCount(aInfo.iDesCount),

 	  iDesSize(aInfo.iDesSize),

 	  iCaps(aInfo.iCaps)

 	{

 	__DMA_ASSERTD(iMaxDesCount > 0);

 	__DMA_ASSERTD((iCaps & ~KCapsBitHwDes) == 0); // undefined bits set?

 	__DMA_ASSERTD(iDesSize > 0);

 	}

 //

 // Second-phase c'tor

 //

 TInt TDmac::Create(const SCreateInfo& aInfo)

 	{

 	iHdrPool = new SDmaDesHdr[iMaxDesCount];

 	if (iHdrPool == NULL)

 		return KErrNoMemory;

 	TInt r = AllocDesPool(aInfo.iDesChunkAttribs);

 	if (r != KErrNone)

 		return KErrNoMemory;

 	// Link all descriptor headers together on the free list

 	iFreeHdr = iHdrPool;

 	TInt i;

 	for (i = 0; i < iMaxDesCount - 1; i++)

 		iHdrPool[i].iNext = iHdrPool + i + 1;

 	iHdrPool[iMaxDesCount-1].iNext = NULL;

 	__DMA_INVARIANT();

 	return KErrNone;

 	}

 TDmac::~TDmac()

 	{

 	__DMA_INVARIANT();

 	FreeDesPool();

 	delete[] iHdrPool;

 	}

 // Calling thread must be in CS

 TInt TDmac::AllocDesPool(TUint aAttribs)

 	{

 	TInt r;

 	if (iCaps & KCapsBitHwDes)

 		{

 		TInt size = iMaxDesCount*iDesSize;

 #ifdef __WINS__

 		(void)aAttribs;

 		iDesPool = new TUint8[size];

 		r = iDesPool ? KErrNone : KErrNoMemory;

 #else

 		// Chunk not mapped as supervisor r/w user none? incorrect mask passed by PSL

 		__DMA_ASSERTD((aAttribs & EMapAttrAccessMask) == EMapAttrSupRw);

 		TPhysAddr phys;

 		r = Epoc::AllocPhysicalRam(size, phys);

 		if (r == KErrNone)

 			{

 			r = DPlatChunkHw::New(iHwDesChunk, phys, size, aAttribs);

 			if (r == KErrNone)

 				{

 				iDesPool = (TAny*)iHwDesChunk->LinearAddress();

 				__KTRACE_OPT(KDMA, Kern::Printf("descriptor hw chunk created lin=0x%08X phys=0x%08X, size=0x%X",

 												iHwDesChunk->iLinAddr, iHwDesChunk->iPhysAddr, size));

 				}

 			else

 				Epoc::FreePhysicalRam(phys, size);

 			}

 #endif

 		}

 	else

 		{

 		iDesPool = new SDmaPseudoDes[iMaxDesCount];

 		r = iDesPool ? KErrNone : KErrNoMemory;

 		}

 	return r;

 	}

 // Calling thread must be in CS

 void TDmac::FreeDesPool()

 	{

 	if (iCaps & KCapsBitHwDes)

 		{

 #ifdef __WINS__

 		delete[] iDesPool;

 #else

 		if (iHwDesChunk)

 			{

 			TPhysAddr phys = iHwDesChunk->PhysicalAddress();

 			TInt size = iHwDesChunk->iSize;

 			iHwDesChunk->Close(NULL);

 			Epoc::FreePhysicalRam(phys, size);

 			}

 #endif

 		}

 	else

 		Kern::Free(iDesPool); 

 	}

 /**

  Prealloc the given number of descriptors.

  */

 TInt TDmac::ReserveSetOfDes(TInt aCount)

 	{

 	__KTRACE_OPT(KDMA, Kern::Printf(">TDmac::ReserveSetOfDes count=%d", aCount));

 	__DMA_ASSERTD(aCount > 0);

 	TInt r = KErrTooBig;

 	Wait();

 	if (iAvailDesCount - aCount >= 0)

 		{

 		iAvailDesCount -= aCount;

 		r = KErrNone;

 		}

 	Signal();

 	__DMA_INVARIANT();

 	__KTRACE_OPT(KDMA, Kern::Printf("<TDmac::ReserveSetOfDes r=%d", r));

 	return r;

 	}

 /**

  Return the given number of preallocated descriptors to the free pool.

  */

 void TDmac::ReleaseSetOfDes(TInt aCount)

 	{

 	__DMA_ASSERTD(aCount >= 0);

 	Wait();

 	iAvailDesCount += aCount;

 	Signal();

 	__DMA_INVARIANT();

 	}

 /**

  Queue DFC and update word used to communicate with DFC.

  Called in interrupt context by PSL.

  */

 void TDmac::HandleIsr(TDmaChannel& aChannel, TBool aIsComplete)

 	{

 	//__KTRACE_OPT(KDMA, Kern::Printf("TDmac::HandleIsr channel=%d complete=%d", aChannelIdx, aIsComplete));

 	// Queue DFC if necessary.  The possible scenarios are:

 	// * no DFC queued --> need to queue DFC

 	// * DFC queued (not running yet) --> just need to update iIsrDfc

 	// * DFC running / iIsrDfc already reset --> need to requeue DFC

 	// * DFC running /  iIsrDfc not reset yet --> just need to update iIsrDfc

 	// Set error flag if necessary.

 	TUint32 inc = aIsComplete ? 1u : TUint32(TDmaChannel::KErrorFlagMask)|1u;

 	TUint32 orig = __e32_atomic_tau_ord32(&aChannel.iIsrDfc, TUint32(TDmaChannel::KCancelFlagMask), 0, inc);

 	// As transfer should be suspended when an error occurs, we

 	// should never get there with the error flag already set.

 	__DMA_ASSERTD((orig & inc & (TUint32)TDmaChannel::KErrorFlagMask) == 0);

 	if (orig == 0)

 		aChannel.iDfc.Add();

 	}

 void TDmac::InitDes(const SDmaDesHdr& aHdr, TUint32 aSrc, TUint32 aDest, TInt aCount,

 					TUint aFlags, TUint32 aPslInfo, TUint32 aCookie)

 	{

  	if (iCaps & KCapsBitHwDes)

 		InitHwDes(aHdr, aSrc, aDest, aCount, aFlags, aPslInfo, aCookie);

 	else

 		{

 		SDmaPseudoDes& des = HdrToDes(aHdr);

 		des.iSrc = aSrc;

 		des.iDest = aDest;

 		des.iCount = aCount;

 		des.iFlags = aFlags;

 		des.iPslInfo = aPslInfo;

 		des.iCookie = aCookie;

 		}

 	}

 void TDmac::InitHwDes(const SDmaDesHdr& /*aHdr*/, TUint32 /*aSrc*/, TUint32 /*aDest*/, TInt /*aCount*/,

 					  TUint /*aFlags*/, TUint32 /*aPslInfo*/, TUint32 /*aCookie*/)

 	{

 	// concrete controller must override if KCapsBitHwDes set

 	__DMA_CANT_HAPPEN();

 	}

 void TDmac::ChainHwDes(const SDmaDesHdr& /*aHdr*/, const SDmaDesHdr& /*aNextHdr*/)

 	{

 	// concrete controller must override if KCapsBitHwDes set

 	__DMA_CANT_HAPPEN();

 	}

 void TDmac::AppendHwDes(const TDmaChannel& /*aChannel*/, const SDmaDesHdr& /*aLastHdr*/,

 						const SDmaDesHdr& /*aNewHdr*/)

 	{

  	// concrete controller must override if KCapsBitHwDes set

 	__DMA_CANT_HAPPEN();

 	}

 void TDmac::UnlinkHwDes(const TDmaChannel& /*aChannel*/, SDmaDesHdr& /*aHdr*/)

 	{

  	// concrete controller must override if KCapsBitHwDes set

 	__DMA_CANT_HAPPEN();

 	}

 TInt TDmac::FailNext(const TDmaChannel& /*aChannel*/)

 	{

 	return KErrNotSupported;

 	}

 TInt TDmac::MissNextInterrupts(const TDmaChannel& /*aChannel*/, TInt /*aInterruptCount*/)

 	{

 	return KErrNotSupported;

 	}

 TInt TDmac::Extension(TDmaChannel& /*aChannel*/, TInt /*aCmd*/, TAny* /*aArg*/)

 	{

 	// default implementation - NOP

 	return KErrNotSupported;

 	}

 #ifdef _DEBUG

 void TDmac::Invariant()

 	{

 	Wait();

 	__DMA_ASSERTD(0 <= iAvailDesCount && iAvailDesCount <= iMaxDesCount);

 	__DMA_ASSERTD(! iFreeHdr || IsValidHdr(iFreeHdr));

 	for (TInt i = 0; i < iMaxDesCount; i++)

 		__DMA_ASSERTD(iHdrPool[i].iNext == NULL || IsValidHdr(iHdrPool[i].iNext));

 	Signal();

 	}

 TBool TDmac::IsValidHdr(const SDmaDesHdr* aHdr)

 	{

 	return (iHdrPool <= aHdr) && (aHdr < iHdrPool + iMaxDesCount);

 	}

 #endif

 //////////////////////////////////////////////////////////////////////////////

 // DDmaRequest

 EXPORT_C DDmaRequest::DDmaRequest(TDmaChannel& aChannel, TCallback aCb, TAny* aCbArg, TInt aMaxTransferSize)

 	: iChannel(aChannel),

 	  iCb(aCb),

 	  iCbArg(aCbArg),

 	  iMaxTransferSize(aMaxTransferSize)

 	{

 	// iDesCount = 0;

 	// iFirstHdr = iLastHdr = NULL;

 	// iQueued = EFalse;

 	iChannel.iReqCount++;

 	__DMA_INVARIANT();

 	}

 EXPORT_C DDmaRequest::~DDmaRequest()

 	{

 	__DMA_ASSERTD(!iQueued);

 	__DMA_INVARIANT();

 	FreeDesList();

 	iChannel.iReqCount--;

 	}

 EXPORT_C TInt DDmaRequest::Fragment(TUint32 aSrc, TUint32 aDest, TInt aCount, TUint aFlags, TUint32 aPslInfo)

 	{

 	__KTRACE_OPT(KDMA, Kern::Printf("DDmaRequest::Fragment thread %O "

 									"src=0x%08X dest=0x%08X count=%d flags=0x%X psl=0x%08X",

 									&Kern::CurrentThread(), aSrc, aDest, aCount, aFlags, aPslInfo));

 	__DMA_ASSERTD(aCount > 0);

 	__DMA_ASSERTD(!iQueued);

 	const TUint alignMask = iChannel.MemAlignMask(aFlags, aPslInfo);

 	const TBool memSrc  = aFlags & KDmaMemSrc;

 	const TBool memDest = aFlags & KDmaMemDest;

 	// Memory buffers must satisfy alignment constraint

 	__DMA_ASSERTD(!memSrc || ((aSrc & alignMask) == 0));

 	__DMA_ASSERTD(!memDest || ((aDest & alignMask) == 0));

 	// Ask the PSL what the maximum size possible for this transfer is

 	TInt maxTransferSize = iChannel.MaxTransferSize(aFlags, aPslInfo);

 	if (!maxTransferSize)

 		{

 		__KTRACE_OPT(KPANIC, Kern::Printf("Error: maxTransferSize == 0"));

 		return KErrArgument;

 		}

 	if (iMaxTransferSize)

 		{

 		// User has set a size cap

 		__DMA_ASSERTA((iMaxTransferSize <= maxTransferSize) || (maxTransferSize == -1));

 		maxTransferSize = iMaxTransferSize;

 		}

 	else

 		{

 		// User doesn't care about max size

 		if (maxTransferSize == -1)

 			{

 			// No maximum imposed by controller

 			maxTransferSize = aCount;

 			}

 		}

 	const TInt maxAlignedSize = (maxTransferSize & ~alignMask);

 	__DMA_ASSERTD(maxAlignedSize > 0);						// bug in PSL if not true

 	FreeDesList();

 	TInt r = KErrNone;

 	do

 		{

 		// Allocate fragment

 		r = ExpandDesList();

 		if (r != KErrNone)

 			{

 			FreeDesList();

 			break;

 			}

 		// Compute fragment size

 		TInt c = Min(maxTransferSize, aCount);

 		if (memSrc && ((aFlags & KDmaPhysAddrSrc) == 0))

 			c = MaxPhysSize(aSrc, c);

 		if (memDest && ((aFlags & KDmaPhysAddrDest) == 0))

 			c = MaxPhysSize(aDest, c);

 		if ((memSrc || memDest) && (c < aCount) && (c > maxAlignedSize))

 			{

 			// This is not last fragment of transfer to/from memory. We must

 			// round down fragment size so next one is correctly aligned.

 			c = maxAlignedSize;

 			}

 		// Initialise fragment

 		__KTRACE_OPT(KDMA, Kern::Printf("fragment: src=0x%08X dest=0x%08X count=%d", aSrc, aDest, c));

 		iChannel.iController->InitDes(*iLastHdr, aSrc, aDest, c, aFlags, aPslInfo, iChannel.PslId());

 		// Update for next iteration

 		aCount -= c;

 		if (memSrc)

 			aSrc += c;

 		if (memDest)

 			aDest += c;

 		}

 	while (aCount > 0);

 	__DMA_INVARIANT();

 	return r;

 	}

 EXPORT_C void DDmaRequest::Queue()

 	{

 	__KTRACE_OPT(KDMA, Kern::Printf("DDmaRequest::Queue thread %O", &Kern::CurrentThread()));

 	__DMA_ASSERTD(iDesCount > 0);	// Not configured? call Fragment() first !

 	__DMA_ASSERTD(!iQueued);

 	// append request to queue and link new descriptor list to existing one.

 	iChannel.Wait();

 	TUint32 req_count = iChannel.iQueuedRequests++;

 	if (req_count == 0)

 		{

 		iChannel.Signal();

 		iChannel.QueuedRequestCountChanged();

 		iChannel.Wait();

 		}

 	if (!(iChannel.iIsrDfc & (TUint32)TDmaChannel::KCancelFlagMask))

 		{

 		iQueued = ETrue;

 		iChannel.iReqQ.Add(&iLink);

 		*iChannel.iNullPtr = iFirstHdr;

 		iChannel.iNullPtr = &(iLastHdr->iNext);

 		iChannel.DoQueue(*this);

 		iChannel.Signal();

 		}

 	else

 		{

 		// Someone is cancelling all requests...

 		req_count = --iChannel.iQueuedRequests;

 		iChannel.Signal();

 		if (req_count == 0)

 			{

 			iChannel.QueuedRequestCountChanged();

 			}

 		}

 	__DMA_INVARIANT();

 	}

 EXPORT_C TInt DDmaRequest::ExpandDesList(TInt aCount)

 	{

 	__DMA_ASSERTD(!iQueued);

 	__DMA_ASSERTD(aCount > 0);

 	if (aCount > iChannel.iAvailDesCount)

 		return KErrTooBig;

 	iChannel.iAvailDesCount -= aCount;

 	iDesCount += aCount;

 	TDmac& c = *(iChannel.iController);

 	c.Wait();

 	if (iFirstHdr == NULL)

 		{

 		// handle empty list specially to simplify following loop

 		iFirstHdr = iLastHdr = c.iFreeHdr;

 		c.iFreeHdr = c.iFreeHdr->iNext;

 		--aCount;

 		}

 	else

 		iLastHdr->iNext = c.iFreeHdr;

 	// Remove as many descriptors and headers from free pool as necessary and

 	// ensure hardware descriptors are chained together.

 	while (aCount-- > 0)

 		{

 		__DMA_ASSERTD(c.iFreeHdr != NULL);

 		if (c.iCaps & TDmac::KCapsBitHwDes)

 			c.ChainHwDes(*iLastHdr, *(c.iFreeHdr));

 		iLastHdr = c.iFreeHdr;

 		c.iFreeHdr = c.iFreeHdr->iNext;

 		}

 	c.Signal();

 	iLastHdr->iNext = NULL;

 	__DMA_INVARIANT();

 	return KErrNone;

 	}

 EXPORT_C void DDmaRequest::FreeDesList()

 	{

 	__DMA_ASSERTD(!iQueued);

 	if (iDesCount > 0)

 		{

 		iChannel.iAvailDesCount += iDesCount;

 		TDmac& c = *(iChannel.iController);

 		c.Wait();

 		iLastHdr->iNext = c.iFreeHdr;

 		c.iFreeHdr = iFirstHdr;

 		c.Signal();

 		iFirstHdr = iLastHdr = NULL;

 		iDesCount = 0;

 		}

 	}

 #ifdef _DEBUG

 void DDmaRequest::Invariant()

 	{

 	iChannel.Wait();

 	__DMA_ASSERTD(iChannel.IsOpened());

 	__DMA_ASSERTD(0 <= iMaxTransferSize);

 	__DMA_ASSERTD(0 <= iDesCount && iDesCount <= iChannel.iMaxDesCount);

 	if (iDesCount == 0)

 		{

 		__DMA_ASSERTD(!iQueued);

 		__DMA_ASSERTD(!iFirstHdr && !iLastHdr);

 		}

 	else

 		{

 		__DMA_ASSERTD(iChannel.iController->IsValidHdr(iFirstHdr));

 		__DMA_ASSERTD(iChannel.iController->IsValidHdr(iLastHdr));

 		}

 	iChannel.Signal();

 	}

 #endif

 //////////////////////////////////////////////////////////////////////////////

 // TDmaChannel

 EXPORT_C TInt TDmaChannel::StaticExtension(TInt aCmd, TAny* aArg)

 	{

 	return DmaChannelMgr::StaticExtension(aCmd, aArg);

 	}

 TDmaChannel::TDmaChannel()

 	: iController(NULL),

 	  iPslId(0),

 	  iCurHdr(NULL),

 	  iNullPtr(&iCurHdr),

 	  iDfc(Dfc, NULL, 0),

 	  iMaxDesCount(0),

 	  iAvailDesCount(0),

 	  iIsrDfc(0),

 	  iReqQ(),

 	  iReqCount(0),

 	  iQueuedRequests(0),

 	  iCancelInfo(NULL)

 	{

 	__DMA_INVARIANT();

 	}

 EXPORT_C TInt TDmaChannel::Open(const SCreateInfo& aInfo, TDmaChannel*& aChannel)

 	{

 	__KTRACE_OPT(KDMA, Kern::Printf("TDmaChannel::Open thread %O", &Kern::CurrentThread()));

 	__DMA_ASSERTD(aInfo.iDfcQ != NULL);

 	__DMA_ASSERTD(aInfo.iDfcPriority < KNumDfcPriorities);

 	__DMA_ASSERTD(aInfo.iDesCount >= 1);

 	aChannel = NULL;

 	DmaChannelMgr::Wait();

 	TDmaChannel* pC = DmaChannelMgr::Open(aInfo.iCookie);

 	DmaChannelMgr::Signal();

 	if (!pC)

 		return KErrInUse;

 	TInt r = pC->iController->ReserveSetOfDes(aInfo.iDesCount);

 	if (r != KErrNone)

 		{

 		pC->Close();

 		return r;

 		}

 	pC->iAvailDesCount = pC->iMaxDesCount = aInfo.iDesCount;

 	new (&pC->iDfc) TDfc(&Dfc, pC, aInfo.iDfcQ, aInfo.iDfcPriority);

 	aChannel = pC;

 #ifdef _DEBUG

 	pC->Invariant();

 #endif

 	__KTRACE_OPT(KDMA, Kern::Printf("opened channel %d", pC->iPslId));

 	return KErrNone;

 	}

 EXPORT_C void TDmaChannel::Close()

 	{

 	__KTRACE_OPT(KDMA, Kern::Printf("TDmaChannel::Close %d", iPslId));

 	__DMA_ASSERTD(IsOpened());

 	__DMA_ASSERTD(IsQueueEmpty());

 	__DMA_ASSERTD(iReqCount == 0);

 	__DMA_ASSERTD(iQueuedRequests == 0);

 	// descriptor leak? bug in request code

 	__DMA_ASSERTD(iAvailDesCount == iMaxDesCount);

 	iController->ReleaseSetOfDes(iMaxDesCount);

 	iAvailDesCount = iMaxDesCount = 0;

 	DmaChannelMgr::Wait();

 	DmaChannelMgr::Close(this);

 	iController = NULL;

 	DmaChannelMgr::Signal();

 	__DMA_INVARIANT();

 	}

 EXPORT_C void TDmaChannel::CancelAll()

 	{

 	__KTRACE_OPT(KDMA, Kern::Printf("TDmaChannel::CancelAll thread %O channel - %d",

 									&Kern::CurrentThread(), iPslId));

 	__DMA_ASSERTD(IsOpened());

 	NThread* nt = NKern::CurrentThread();

 	TBool wait = FALSE;

 	TDmaCancelInfo c;

 	TDmaCancelInfo* waiters = 0;

 	NKern::ThreadEnterCS();

 	Wait();

 	const TUint32 req_count_before = iQueuedRequests;

 	NThreadBase* dfcnt = iDfc.Thread();

 	__e32_atomic_store_ord32(&iIsrDfc, (TUint32)KCancelFlagMask);

 	// ISRs after this point will not post a DFC, however a DFC may already be queued or running or both

 	if (!IsQueueEmpty())

 		{

 		// There is a transfer in progress.  It may complete before the DMAC

 		// has stopped, but the resulting ISR will not post a DFC.

 		// ISR should not happen after this function returns.

 		iController->StopTransfer(*this);

 		ResetStateMachine();

 		// Clean-up the request queue.

 		SDblQueLink* pL;

 		while ((pL = iReqQ.GetFirst()) != NULL)

 			{

 			iQueuedRequests--;

 			DDmaRequest* pR = _LOFF(pL, DDmaRequest, iLink);

 			pR->OnDeque();

 			}

 		}

 	if (!dfcnt || dfcnt==nt)

 		{

 		// no DFC queue or DFC runs in this thread, so just cancel it and we're finished

 		iDfc.Cancel();

 		// if other calls to CancelAll() are waiting for the DFC, release them here

 		waiters = iCancelInfo;

 		iCancelInfo = 0;

 		// reset the ISR count

 		__e32_atomic_store_rel32(&iIsrDfc, 0);

 		}

 	else

 		{

 		// DFC runs in another thread. Make sure it's queued and then wait for it to run.

 		if (iCancelInfo)

 			c.InsertBefore(iCancelInfo);

 		else

 			iCancelInfo = &c;

 		wait = TRUE;

 		iDfc.Enque();

 		}

 	const TUint32 req_count_after = iQueuedRequests;

 	Signal();

 	if (waiters)

 		waiters->Signal();

 	if (wait)

 		NKern::FSWait(&c.iSem);

  	NKern::ThreadLeaveCS();

 	// Only call PSL if there were requests queued when we entered AND there

 	// are now no requests left on the queue.

 	if ((req_count_before != 0) && (req_count_after == 0))

 		{

 		QueuedRequestCountChanged();

 		}

 	__DMA_INVARIANT();

 	}

 /**

  DFC callback function (static member).

  */

 void TDmaChannel::Dfc(TAny* aArg)

 	{

 	((TDmaChannel*)aArg)->DoDfc();

 	}

 void TDmaChannel::DoDfc()

 	{

 	Wait();

 	// Atomically fetch and reset the number of DFC queued by ISR and the error

 	// flag. Leave the cancel flag alone for now.

 	const TUint32 w = __e32_atomic_and_ord32(&iIsrDfc, (TUint32)KCancelFlagMask);

 	TUint32 count = w & KDfcCountMask;

 	const TBool error = w & (TUint32)KErrorFlagMask;

 	TBool stop = w & (TUint32)KCancelFlagMask;

 	__DMA_ASSERTD(count>0 || stop);

 	const TUint32 req_count_before = iQueuedRequests;

 	TUint32 req_count_after = 0;

 	while(count && !stop)

 		{

 		--count;

 		// If an error occurred it must have been reported on the last interrupt since transfers are

 		// suspended after an error.

 		DDmaRequest::TResult res = (count==0 && error) ? DDmaRequest::EError : DDmaRequest::EOk;

 		__DMA_ASSERTD(!iReqQ.IsEmpty());

 		DDmaRequest* pCompletedReq = NULL;

 		DDmaRequest* pCurReq = _LOFF(iReqQ.First(), DDmaRequest, iLink);

 		DDmaRequest::TCallback cb = 0;

 		TAny* arg = 0;

 		if (res == DDmaRequest::EOk)

 			{

 			// Update state machine, current fragment, completed fragment and

 			// tell DMAC to transfer next fragment if necessary.

 			SDmaDesHdr* pCompletedHdr = NULL;

 			DoDfc(*pCurReq, pCompletedHdr);

 			// If just completed last fragment from current request, switch to next

 			// request (if any).

 			if (pCompletedHdr == pCurReq->iLastHdr)

 				{

 				pCompletedReq = pCurReq;

 				pCurReq->iLink.Deque();

 				iQueuedRequests--;

 				if (iReqQ.IsEmpty())

 					iNullPtr = &iCurHdr;

 				pCompletedReq->OnDeque();

 				}

 			}

 		else if (res == DDmaRequest::EError)

 			pCompletedReq = pCurReq;

 		else

 			__DMA_CANT_HAPPEN();

 		if (pCompletedReq)

 			{

 			cb = pCompletedReq->iCb;

 			arg = pCompletedReq->iCbArg;

 			Signal();

 			__KTRACE_OPT(KDMA, Kern::Printf("notifying DMA client result=%d", res));

 			(*cb)(res,arg);

 			Wait();

 			}

 		if (pCompletedReq || Flash())

 			stop = __e32_atomic_load_acq32(&iIsrDfc) & (TUint32)KCancelFlagMask;

 		}

 	// Some interrupts may be missed (double-buffer and scatter-gather

 	// controllers only) if two or more transfers complete while interrupts are

 	// disabled in the CPU. If this happens, the framework will go out of sync

 	// and leave some orphaned requests in the queue.

 	//

 	// To ensure correctness we handle this case here by checking that the request

 	// queue is empty when all transfers have completed and, if not, cleaning up

 	// and notifying the client of the completion of the orphaned requests.

 	//

 	// Note that if some interrupts are missed and the controller raises an

 	// error while transferring a subsequent fragment, the error will be reported

 	// on a fragment which was successfully completed.  There is no easy solution

 	// to this problem, but this is okay as the only possible action following a

 	// failure is to flush the whole queue.

 	if (stop)

 		{

 		TDmaCancelInfo* waiters = iCancelInfo;

 		iCancelInfo = 0;

 		// make sure DFC doesn't run again until a new request completes

 		iDfc.Cancel();

 		// reset the ISR count - new requests can now be processed

 		__e32_atomic_store_rel32(&iIsrDfc, 0);

 		req_count_after = iQueuedRequests;

 		Signal();

 		// release threads doing CancelAll()

 		waiters->Signal();

 		}

 	else if (!error && !iDfc.Queued() && !iReqQ.IsEmpty() && iController->IsIdle(*this))

 		{

 		__KTRACE_OPT(KDMA, Kern::Printf("Missed interrupt(s) - draining request queue"));

 		ResetStateMachine();

 		// Move orphaned requests to temporary queue so channel queue can

 		// accept new requests.

 		SDblQue q;

 		q.MoveFrom(&iReqQ);

 		SDblQueLink* pL;

 		while ((pL = q.GetFirst()) != NULL)

 			{

 			iQueuedRequests--;

 			DDmaRequest* pR = _LOFF(pL, DDmaRequest, iLink);

 			__KTRACE_OPT(KDMA, Kern::Printf("Removing request from queue and notifying client"));

 			pR->OnDeque();

 			DDmaRequest::TCallback cb = pR->iCb;

 			TAny* arg = pR->iCbArg;

 			if (cb)

 				{

 				Signal();

 				(*cb)(DDmaRequest::EOk, arg);

 				Wait();

 				}

 			}

 		req_count_after = iQueuedRequests;

 		Signal();

 		}

 	else

 		{

 		req_count_after = iQueuedRequests;

 		Signal();

 		}

 	// Only call PSL if there were requests queued when we entered AND there

 	// are now no requests left on the queue (after also having executed all

 	// client callbacks).

 	if ((req_count_before != 0) && (req_count_after == 0))

 		{

 		QueuedRequestCountChanged();

 		}

 	__DMA_INVARIANT();

 	}

 /** Reset state machine only, request queue is unchanged */

 void TDmaChannel::ResetStateMachine()

 	{

 	DoCancelAll();

 	iCurHdr = NULL;

 	iNullPtr = &iCurHdr;

 	}

 /** Unlink the last item of a LLI chain from the next chain.

 	Default implementation does nothing. This is overridden by scatter-gather channels. */

 void TDmaChannel::DoUnlink(SDmaDesHdr& /*aHdr*/)

 	{

 	}

 /** PSL may override */

 void TDmaChannel::QueuedRequestCountChanged()

 	{

 #ifdef _DEBUG

 	Wait();

 	__KTRACE_OPT(KDMA,

 				 Kern::Printf("TDmaChannel::QueuedRequestCountChanged() %d",

 							  iQueuedRequests));

 	__DMA_ASSERTA(iQueuedRequests >= 0);

 	Signal();

 #endif

 	}

 #ifdef _DEBUG

 void TDmaChannel::Invariant()

 	{

 	Wait();

 	__DMA_ASSERTD(iReqCount >= 0);

 	// should always point to NULL pointer ending fragment queue

 	__DMA_ASSERTD(*iNullPtr == NULL);

 	__DMA_ASSERTD(0 <= iAvailDesCount && iAvailDesCount <= iMaxDesCount);

 	__DMA_ASSERTD(iCurHdr == NULL || iController->IsValidHdr(iCurHdr));

 	if (IsOpened())

 		{

 		__DMA_ASSERTD((iCurHdr && !IsQueueEmpty()) || (!iCurHdr && IsQueueEmpty()));

 		if (iCurHdr == NULL)

 			__DMA_ASSERTD(iNullPtr == &iCurHdr);

 		}

 	else

 		{

 		__DMA_ASSERTD(iCurHdr == NULL);

 		__DMA_ASSERTD(iNullPtr == &iCurHdr);

 		__DMA_ASSERTD(IsQueueEmpty());

 		}

 	Signal();

 	}

 #endif

 //////////////////////////////////////////////////////////////////////////////

 // TDmaSbChannel

 void TDmaSbChannel::DoQueue(DDmaRequest& /*aReq*/)

 	{

 	if (!iTransferring)

 		{

 		iController->Transfer(*this, *iCurHdr);

 		iTransferring = ETrue;

 		}

 	}

 void TDmaSbChannel::DoCancelAll()

 	{

 	__DMA_ASSERTD(iTransferring);

 	iTransferring = EFalse;

 	}

 void TDmaSgChannel::DoUnlink(SDmaDesHdr& aHdr)

 	{

 	iController->UnlinkHwDes(*this, aHdr);

 	}

 void TDmaSbChannel::DoDfc(DDmaRequest& /*aCurReq*/, SDmaDesHdr*& aCompletedHdr)

 	{

 	__DMA_ASSERTD(iTransferring);

 	aCompletedHdr = iCurHdr;

 	iCurHdr = iCurHdr->iNext;

 	if (iCurHdr != NULL)

 		iController->Transfer(*this, *iCurHdr);

 	else

 		iTransferring = EFalse;

 	}

 //////////////////////////////////////////////////////////////////////////////

 // TDmaDbChannel

 void TDmaDbChannel::DoQueue(DDmaRequest& aReq)

 	{

 	switch (iState)

 		{

 	case EIdle:

 		iController->Transfer(*this, *iCurHdr);

 		if (iCurHdr->iNext)

 			{

 			iController->Transfer(*this, *(iCurHdr->iNext));

 			iState = ETransferring;

 			}

 		else

 			iState = ETransferringLast;

 		break;

 	case ETransferring:

 		// nothing to do

 		break;

 	case ETransferringLast:

 		iController->Transfer(*this, *(aReq.iFirstHdr));

 		iState = ETransferring;

 		break;

 	default:

 		__DMA_CANT_HAPPEN();

 		}

 	}

 void TDmaDbChannel::DoCancelAll()

 	{

 	iState = EIdle;

 	}

 void TDmaDbChannel::DoDfc(DDmaRequest& /*aCurReq*/, SDmaDesHdr*& aCompletedHdr)

 	{

 	aCompletedHdr = iCurHdr;

 	iCurHdr = iCurHdr->iNext;

 	switch (iState)

 		{

 	case ETransferringLast:

 		iState = EIdle;

 		break;

 	case ETransferring:

 		if (iCurHdr->iNext == NULL)

 			iState = ETransferringLast;

 		else

 			iController->Transfer(*this, *(iCurHdr->iNext));

 		break;

 	default:

 		__DMA_CANT_HAPPEN();

 		}

 	}

 //////////////////////////////////////////////////////////////////////////////

 // TDmaSgChannel

 void TDmaSgChannel::DoQueue(DDmaRequest& aReq)

 	{

 	if (iTransferring)

 		{

 		__DMA_ASSERTD(!aReq.iLink.Alone());

 		DDmaRequest* pReqPrev = _LOFF(aReq.iLink.iPrev, DDmaRequest, iLink);

 		iController->AppendHwDes(*this, *(pReqPrev->iLastHdr), *(aReq.iFirstHdr));

 		}

 	else

 		{

 		iController->Transfer(*this, *(aReq.iFirstHdr));

 		iTransferring = ETrue;

 		}

 	}

 void TDmaSgChannel::DoCancelAll()

 	{

 	__DMA_ASSERTD(iTransferring);

 	iTransferring = EFalse;

 	}

 void TDmaSgChannel::DoDfc(DDmaRequest& aCurReq, SDmaDesHdr*& aCompletedHdr)

 	{

 	__DMA_ASSERTD(iTransferring);

 	aCompletedHdr = aCurReq.iLastHdr;

 	iCurHdr = aCompletedHdr->iNext;

 	iTransferring = (iCurHdr != NULL);

 	}