// Copyright (c) 2002-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

// e32\drivers\dmapil.cpp

// DMA Platform Independent Layer (PIL)

// 

//

#include <drivers/dma.h>

#include <kernel/kern_priv.h>

static const char KDmaPanicCat[] = "DMA";

NFastMutex DmaChannelMgr::Lock;

class TDmaCancelInfo : public SDblQueLink

	{

public:

	TDmaCancelInfo();

	void Signal();

public:

	NFastSemaphore iSem;

	};

TDmaCancelInfo::TDmaCancelInfo()

	:	iSem(0)

	{

	iNext = this;

	iPrev = this;

	}

void TDmaCancelInfo::Signal()

	{

	TDmaCancelInfo* p = this;

	FOREVER

		{

		TDmaCancelInfo* next = (TDmaCancelInfo*)p->iNext;

		if (p!=next)

			p->Deque();

		NKern::FSSignal(&p->iSem);	// Don't dereference p after this

		if (p==next)

			break;

		p = next;

		}

	}

//////////////////////////////////////////////////////////////////////////////

#ifdef __DMASIM__

#ifdef __WINS__

typedef TLinAddr TPhysAddr;

#endif

static inline TPhysAddr LinToPhys(TLinAddr aLin) {return aLin;}

#else

static inline TPhysAddr LinToPhys(TLinAddr aLin) {return Epoc::LinearToPhysical(aLin);}

#endif

//

// Return minimum of aMaxSize and size of largest physically contiguous block

// starting at aLinAddr.

//

static TInt MaxPhysSize(TLinAddr aLinAddr, const TInt aMaxSize)

	{

	const TPhysAddr physBase = LinToPhys(aLinAddr);

	TLinAddr lin = aLinAddr;

	TInt size = 0;

	for (;;)

		{

		// Round up the linear address to the next MMU page boundary

		const TLinAddr linBoundary = Kern::RoundToPageSize(lin + 1);

		size += linBoundary - lin;

		if (size >= aMaxSize)

			return aMaxSize;

		if ((physBase + size) != LinToPhys(linBoundary))

			return size;

		lin = linBoundary;

		}

	}

//////////////////////////////////////////////////////////////////////////////

// TDmac

TDmac::TDmac(const SCreateInfo& aInfo)

	: iMaxDesCount(aInfo.iDesCount),

	  iAvailDesCount(aInfo.iDesCount),

	  iDesSize(aInfo.iDesSize),

	  iCaps(aInfo.iCaps)

	{

	__DMA_ASSERTD(iMaxDesCount > 0);

	__DMA_ASSERTD((iCaps & ~KCapsBitHwDes) == 0); // undefined bits set?

	__DMA_ASSERTD(iDesSize > 0);

	}

//

// Second-phase c'tor

//

TInt TDmac::Create(const SCreateInfo& aInfo)

	{

	iHdrPool = new SDmaDesHdr[iMaxDesCount];

	if (iHdrPool == NULL)

		return KErrNoMemory;

	TInt r = AllocDesPool(aInfo.iDesChunkAttribs);

	if (r != KErrNone)

		return KErrNoMemory;

	// Link all descriptor headers together on the free list

	iFreeHdr = iHdrPool;

	TInt i;

	for (i = 0; i < iMaxDesCount - 1; i++)

		iHdrPool[i].iNext = iHdrPool + i + 1;

	iHdrPool[iMaxDesCount-1].iNext = NULL;

	__DMA_INVARIANT();

	return KErrNone;

	}

TDmac::~TDmac()

	{

	__DMA_INVARIANT();

	FreeDesPool();

	delete[] iHdrPool;

	}

// Calling thread must be in CS

TInt TDmac::AllocDesPool(TUint aAttribs)

	{

	TInt r;

	if (iCaps & KCapsBitHwDes)

		{

		TInt size = iMaxDesCount*iDesSize;

#ifdef __WINS__

		(void)aAttribs;

		iDesPool = new TUint8[size];

		r = iDesPool ? KErrNone : KErrNoMemory;

#else

		// Chunk not mapped as supervisor r/w user none? incorrect mask passed by PSL

		__DMA_ASSERTD((aAttribs & EMapAttrAccessMask) == EMapAttrSupRw);

		TPhysAddr phys;

		r = Epoc::AllocPhysicalRam(size, phys);

		if (r == KErrNone)

			{

			r = DPlatChunkHw::New(iHwDesChunk, phys, size, aAttribs);

			if (r == KErrNone)

				{

				iDesPool = (TAny*)iHwDesChunk->LinearAddress();

				__KTRACE_OPT(KDMA, Kern::Printf("descriptor hw chunk created lin=0x%08X phys=0x%08X, size=0x%X",

												iHwDesChunk->iLinAddr, iHwDesChunk->iPhysAddr, size));

				}

			else

				Epoc::FreePhysicalRam(phys, size);

			}

#endif

		}

	else

		{

		iDesPool = new SDmaPseudoDes[iMaxDesCount];

		r = iDesPool ? KErrNone : KErrNoMemory;

		}

	return r;

	}

// Calling thread must be in CS

void TDmac::FreeDesPool()

	{

	if (iCaps & KCapsBitHwDes)

		{

#ifdef __WINS__

		delete[] iDesPool;

#else

		if (iHwDesChunk)

			{

			TPhysAddr phys = iHwDesChunk->PhysicalAddress();

			TInt size = iHwDesChunk->iSize;

			iHwDesChunk->Close(NULL);

			Epoc::FreePhysicalRam(phys, size);

			}

#endif

		}

	else

		Kern::Free(iDesPool); 

	}

/**

 Prealloc the given number of descriptors.

 */

TInt TDmac::ReserveSetOfDes(TInt aCount)

	{

	__KTRACE_OPT(KDMA, Kern::Printf(">TDmac::ReserveSetOfDes count=%d", aCount));

	__DMA_ASSERTD(aCount > 0);

	TInt r = KErrTooBig;

	Wait();

	if (iAvailDesCount - aCount >= 0)

		{

		iAvailDesCount -= aCount;

		r = KErrNone;

		}

	Signal();

	__DMA_INVARIANT();

	__KTRACE_OPT(KDMA, Kern::Printf("<TDmac::ReserveSetOfDes r=%d", r));

	return r;

	}

/**

 Return the given number of preallocated descriptors to the free pool.

 */

void TDmac::ReleaseSetOfDes(TInt aCount)

	{

	__DMA_ASSERTD(aCount >= 0);

	Wait();

	iAvailDesCount += aCount;

	Signal();

	__DMA_INVARIANT();

	}

/**

 Queue DFC and update word used to communicate with DFC.

 Called in interrupt context by PSL.

 */

void TDmac::HandleIsr(TDmaChannel& aChannel, TBool aIsComplete)

	{

	//__KTRACE_OPT(KDMA, Kern::Printf("TDmac::HandleIsr channel=%d complete=%d", aChannelIdx, aIsComplete));

	// Queue DFC if necessary.  The possible scenarios are:

	// * no DFC queued --> need to queue DFC

	// * DFC queued (not running yet) --> just need to update iIsrDfc

	// * DFC running / iIsrDfc already reset --> need to requeue DFC

	// * DFC running /  iIsrDfc not reset yet --> just need to update iIsrDfc

	// Set error flag if necessary.

	TUint32 inc = aIsComplete ? 1u : TUint32(TDmaChannel::KErrorFlagMask)|1u;

	TUint32 orig = __e32_atomic_tau_ord32(&aChannel.iIsrDfc, TUint32(TDmaChannel::KCancelFlagMask), 0, inc);

	// As transfer should be suspended when an error occurs, we

	// should never get there with the error flag already set.

	__DMA_ASSERTD((orig & inc & (TUint32)TDmaChannel::KErrorFlagMask) == 0);

	if (orig == 0)

		aChannel.iDfc.Add();

	}

void TDmac::InitDes(const SDmaDesHdr& aHdr, TUint32 aSrc, TUint32 aDest, TInt aCount,

					TUint aFlags, TUint32 aPslInfo, TUint32 aCookie)

	{

 	if (iCaps & KCapsBitHwDes)

		InitHwDes(aHdr, aSrc, aDest, aCount, aFlags, aPslInfo, aCookie);

	else

		{

		SDmaPseudoDes& des = HdrToDes(aHdr);

		des.iSrc = aSrc;

		des.iDest = aDest;

		des.iCount = aCount;

		des.iFlags = aFlags;

		des.iPslInfo = aPslInfo;

		des.iCookie = aCookie;

		}

	}

void TDmac::InitHwDes(const SDmaDesHdr& /*aHdr*/, TUint32 /*aSrc*/, TUint32 /*aDest*/, TInt /*aCount*/,

					  TUint /*aFlags*/, TUint32 /*aPslInfo*/, TUint32 /*aCookie*/)

	{

	// concrete controller must override if KCapsBitHwDes set

	__DMA_CANT_HAPPEN();

	}

void TDmac::ChainHwDes(const SDmaDesHdr& /*aHdr*/, const SDmaDesHdr& /*aNextHdr*/)

	{

	// concrete controller must override if KCapsBitHwDes set

	__DMA_CANT_HAPPEN();

	}

void TDmac::AppendHwDes(const TDmaChannel& /*aChannel*/, const SDmaDesHdr& /*aLastHdr*/,

						const SDmaDesHdr& /*aNewHdr*/)

	{

 	// concrete controller must override if KCapsBitHwDes set

	__DMA_CANT_HAPPEN();

	}

void TDmac::UnlinkHwDes(const TDmaChannel& /*aChannel*/, SDmaDesHdr& /*aHdr*/)

	{

 	// concrete controller must override if KCapsBitHwDes set

	__DMA_CANT_HAPPEN();

	}

TInt TDmac::FailNext(const TDmaChannel& /*aChannel*/)

	{

	return KErrNotSupported;

	}

TInt TDmac::MissNextInterrupts(const TDmaChannel& /*aChannel*/, TInt /*aInterruptCount*/)

	{

	return KErrNotSupported;

	}

TInt TDmac::Extension(TDmaChannel& /*aChannel*/, TInt /*aCmd*/, TAny* /*aArg*/)

	{

	// default implementation - NOP

	return KErrNotSupported;

	}

#ifdef _DEBUG

void TDmac::Invariant()

	{

	Wait();

	__DMA_ASSERTD(0 <= iAvailDesCount && iAvailDesCount <= iMaxDesCount);

	__DMA_ASSERTD(! iFreeHdr || IsValidHdr(iFreeHdr));

	for (TInt i = 0; i < iMaxDesCount; i++)

		__DMA_ASSERTD(iHdrPool[i].iNext == NULL || IsValidHdr(iHdrPool[i].iNext));

	Signal();

	}

TBool TDmac::IsValidHdr(const SDmaDesHdr* aHdr)

	{

	return (iHdrPool <= aHdr) && (aHdr < iHdrPool + iMaxDesCount);

	}

#endif

//////////////////////////////////////////////////////////////////////////////

// DDmaRequest

EXPORT_C DDmaRequest::DDmaRequest(TDmaChannel& aChannel, TCallback aCb, TAny* aCbArg, TInt aMaxTransferSize)

	: iChannel(aChannel),

	  iCb(aCb),

	  iCbArg(aCbArg),

	  iMaxTransferSize(aMaxTransferSize)

	{

	// iDesCount = 0;

	// iFirstHdr = iLastHdr = NULL;

	// iQueued = EFalse;

	iChannel.iReqCount++;

	__DMA_INVARIANT();

	}

EXPORT_C DDmaRequest::~DDmaRequest()

	{

	__DMA_ASSERTD(!iQueued);

	__DMA_INVARIANT();

	FreeDesList();

	iChannel.iReqCount--;

	}

EXPORT_C TInt DDmaRequest::Fragment(TUint32 aSrc, TUint32 aDest, TInt aCount, TUint aFlags, TUint32 aPslInfo)

	{

	__KTRACE_OPT(KDMA, Kern::Printf("DDmaRequest::Fragment thread %O "

									"src=0x%08X dest=0x%08X count=%d flags=0x%X psl=0x%08X",

									&Kern::CurrentThread(), aSrc, aDest, aCount, aFlags, aPslInfo));

	__DMA_ASSERTD(aCount > 0);

	__DMA_ASSERTD(!iQueued);

	const TUint alignMask = iChannel.MemAlignMask(aFlags, aPslInfo);

	const TBool memSrc  = aFlags & KDmaMemSrc;

	const TBool memDest = aFlags & KDmaMemDest;

	// Memory buffers must satisfy alignment constraint

	__DMA_ASSERTD(!memSrc || ((aSrc & alignMask) == 0));

	__DMA_ASSERTD(!memDest || ((aDest & alignMask) == 0));

	// Ask the PSL what the maximum size possible for this transfer is

	TInt maxTransferSize = iChannel.MaxTransferSize(aFlags, aPslInfo);

	if (!maxTransferSize)

		{

		__KTRACE_OPT(KPANIC, Kern::Printf("Error: maxTransferSize == 0"));

		return KErrArgument;

		}

	if (iMaxTransferSize)

		{

		// User has set a size cap

		__DMA_ASSERTA((iMaxTransferSize <= maxTransferSize) || (maxTransferSize == -1));

		maxTransferSize = iMaxTransferSize;

		}

	else

		{

		// User doesn't care about max size

		if (maxTransferSize == -1)

			{

			// No maximum imposed by controller

			maxTransferSize = aCount;

			}

		}

	const TInt maxAlignedSize = (maxTransferSize & ~alignMask);

	__DMA_ASSERTD(maxAlignedSize > 0);						// bug in PSL if not true

	FreeDesList();

	TInt r = KErrNone;

	do

		{

		// Allocate fragment

		r = ExpandDesList();

		if (r != KErrNone)

			{

			FreeDesList();

			break;

			}

		// Compute fragment size

		TInt c = Min(maxTransferSize, aCount);

		if (memSrc && ((aFlags & KDmaPhysAddrSrc) == 0))

			c = MaxPhysSize(aSrc, c);

		if (memDest && ((aFlags & KDmaPhysAddrDest) == 0))

			c = MaxPhysSize(aDest, c);

		if ((memSrc || memDest) && (c < aCount) && (c > maxAlignedSize))

			{

			// This is not last fragment of transfer to/from memory. We must

			// round down fragment size so next one is correctly aligned.

			c = maxAlignedSize;

			}

		// Initialise fragment

		__KTRACE_OPT(KDMA, Kern::Printf("fragment: src=0x%08X dest=0x%08X count=%d", aSrc, aDest, c));

		iChannel.iController->InitDes(*iLastHdr, aSrc, aDest, c, aFlags, aPslInfo, iChannel.PslId());

		// Update for next iteration

		aCount -= c;

		if (memSrc)

			aSrc += c;

		if (memDest)

			aDest += c;

		}

	while (aCount > 0);

	__DMA_INVARIANT();

	return r;

	}

EXPORT_C void DDmaRequest::Queue()

	{

	__KTRACE_OPT(KDMA, Kern::Printf("DDmaRequest::Queue thread %O", &Kern::CurrentThread()));

	__DMA_ASSERTD(iDesCount > 0);	// Not configured? call Fragment() first !

	__DMA_ASSERTD(!iQueued);

	// append request to queue and link new descriptor list to existing one.

	iChannel.Wait();

	TUint32 req_count = iChannel.iQueuedRequests++;

	if (req_count == 0)

		{

		iChannel.Signal();

		iChannel.QueuedRequestCountChanged();

		iChannel.Wait();

		}

	if (!(iChannel.iIsrDfc & (TUint32)TDmaChannel::KCancelFlagMask))

		{

		iQueued = ETrue;

		iChannel.iReqQ.Add(&iLink);

		*iChannel.iNullPtr = iFirstHdr;

		iChannel.iNullPtr = &(iLastHdr->iNext);

		iChannel.DoQueue(*this);

		iChannel.Signal();

		}

	else

		{

		// Someone is cancelling all requests...

		req_count = --iChannel.iQueuedRequests;

		iChannel.Signal();

		if (req_count == 0)

			{

			iChannel.QueuedRequestCountChanged();

			}

		}

	__DMA_INVARIANT();

	}

EXPORT_C TInt DDmaRequest::ExpandDesList(TInt aCount)

	{

	__DMA_ASSERTD(!iQueued);

	__DMA_ASSERTD(aCount > 0);

	if (aCount > iChannel.iAvailDesCount)

		return KErrTooBig;

	iChannel.iAvailDesCount -= aCount;

	iDesCount += aCount;

	TDmac& c = *(iChannel.iController);

	c.Wait();

	if (iFirstHdr == NULL)

		{

		// handle empty list specially to simplify following loop

		iFirstHdr = iLastHdr = c.iFreeHdr;

		c.iFreeHdr = c.iFreeHdr->iNext;

		--aCount;

		}

	else

		iLastHdr->iNext = c.iFreeHdr;

	// Remove as many descriptors and headers from free pool as necessary and

	// ensure hardware descriptors are chained together.

	while (aCount-- > 0)

		{

		__DMA_ASSERTD(c.iFreeHdr != NULL);

		if (c.iCaps & TDmac::KCapsBitHwDes)

			c.ChainHwDes(*iLastHdr, *(c.iFreeHdr));

		iLastHdr = c.iFreeHdr;

		c.iFreeHdr = c.iFreeHdr->iNext;

		}

	c.Signal();

	iLastHdr->iNext = NULL;

	__DMA_INVARIANT();

	return KErrNone;

	}

EXPORT_C void DDmaRequest::FreeDesList()

	{

	__DMA_ASSERTD(!iQueued);

	if (iDesCount > 0)

		{

		iChannel.iAvailDesCount += iDesCount;

		TDmac& c = *(iChannel.iController);

		c.Wait();

		iLastHdr->iNext = c.iFreeHdr;

		c.iFreeHdr = iFirstHdr;

		c.Signal();

		iFirstHdr = iLastHdr = NULL;

		iDesCount = 0;

		}

	}

#ifdef _DEBUG

void DDmaRequest::Invariant()

	{

	iChannel.Wait();

	__DMA_ASSERTD(iChannel.IsOpened());

	__DMA_ASSERTD(0 <= iMaxTransferSize);

	__DMA_ASSERTD(0 <= iDesCount && iDesCount <= iChannel.iMaxDesCount);

	if (iDesCount == 0)

		{

		__DMA_ASSERTD(!iQueued);

		__DMA_ASSERTD(!iFirstHdr && !iLastHdr);

		}

	else

		{

		__DMA_ASSERTD(iChannel.iController->IsValidHdr(iFirstHdr));

		__DMA_ASSERTD(iChannel.iController->IsValidHdr(iLastHdr));

		}

	iChannel.Signal();

	}

#endif

//////////////////////////////////////////////////////////////////////////////

// TDmaChannel

EXPORT_C TInt TDmaChannel::StaticExtension(TInt aCmd, TAny* aArg)

	{

	return DmaChannelMgr::StaticExtension(aCmd, aArg);

	}

TDmaChannel::TDmaChannel()

	: iController(NULL),

	  iPslId(0),

	  iCurHdr(NULL),

	  iNullPtr(&iCurHdr),

	  iDfc(Dfc, NULL, 0),

	  iMaxDesCount(0),

	  iAvailDesCount(0),

	  iIsrDfc(0),

	  iReqQ(),

	  iReqCount(0),

	  iQueuedRequests(0),

	  iCancelInfo(NULL)

	{

	__DMA_INVARIANT();

	}

EXPORT_C TInt TDmaChannel::Open(const SCreateInfo& aInfo, TDmaChannel*& aChannel)

	{

	__KTRACE_OPT(KDMA, Kern::Printf("TDmaChannel::Open thread %O", &Kern::CurrentThread()));

	__DMA_ASSERTD(aInfo.iDfcQ != NULL);

	__DMA_ASSERTD(aInfo.iDfcPriority < KNumDfcPriorities);

	__DMA_ASSERTD(aInfo.iDesCount >= 1);

	aChannel = NULL;

	DmaChannelMgr::Wait();

	TDmaChannel* pC = DmaChannelMgr::Open(aInfo.iCookie);

	DmaChannelMgr::Signal();

	if (!pC)

		return KErrInUse;

	TInt r = pC->iController->ReserveSetOfDes(aInfo.iDesCount);

	if (r != KErrNone)

		{

		pC->Close();

		return r;

		}

	pC->iAvailDesCount = pC->iMaxDesCount = aInfo.iDesCount;

	new (&pC->iDfc) TDfc(&Dfc, pC, aInfo.iDfcQ, aInfo.iDfcPriority);

	aChannel = pC;

#ifdef _DEBUG

	pC->Invariant();

#endif

	__KTRACE_OPT(KDMA, Kern::Printf("opened channel %d", pC->iPslId));

	return KErrNone;

	}

EXPORT_C void TDmaChannel::Close()

	{

	__KTRACE_OPT(KDMA, Kern::Printf("TDmaChannel::Close %d", iPslId));

	__DMA_ASSERTD(IsOpened());

	__DMA_ASSERTD(IsQueueEmpty());

	__DMA_ASSERTD(iReqCount == 0);

	__DMA_ASSERTD(iQueuedRequests == 0);

	// descriptor leak? bug in request code

	__DMA_ASSERTD(iAvailDesCount == iMaxDesCount);

	iController->ReleaseSetOfDes(iMaxDesCount);

	iAvailDesCount = iMaxDesCount = 0;

	DmaChannelMgr::Wait();

	DmaChannelMgr::Close(this);

	iController = NULL;

	DmaChannelMgr::Signal();

	__DMA_INVARIANT();

	}

EXPORT_C void TDmaChannel::CancelAll()

	{

	__KTRACE_OPT(KDMA, Kern::Printf("TDmaChannel::CancelAll thread %O channel - %d",

									&Kern::CurrentThread(), iPslId));

	__DMA_ASSERTD(IsOpened());

	NThread* nt = NKern::CurrentThread();

	TBool wait = FALSE;

	TDmaCancelInfo c;

	TDmaCancelInfo* waiters = 0;

	NKern::ThreadEnterCS();

	Wait();

	const TUint32 req_count_before = iQueuedRequests;

	NThreadBase* dfcnt = iDfc.Thread();

	__e32_atomic_store_ord32(&iIsrDfc, (TUint32)KCancelFlagMask);

	// ISRs after this point will not post a DFC, however a DFC may already be queued or running or both

	if (!IsQueueEmpty())

		{

		// There is a transfer in progress.  It may complete before the DMAC

		// has stopped, but the resulting ISR will not post a DFC.

		// ISR should not happen after this function returns.

		iController->StopTransfer(*this);

		ResetStateMachine();

		// Clean-up the request queue.

		SDblQueLink* pL;

		while ((pL = iReqQ.GetFirst()) != NULL)

			{

			iQueuedRequests--;

			DDmaRequest* pR = _LOFF(pL, DDmaRequest, iLink);

			pR->OnDeque();

			}

		}

	if (!dfcnt || dfcnt==nt)

		{

		// no DFC queue or DFC runs in this thread, so just cancel it and we're finished

		iDfc.Cancel();

		// if other calls to CancelAll() are waiting for the DFC, release them here

		waiters = iCancelInfo;

		iCancelInfo = 0;

		// reset the ISR count

		__e32_atomic_store_rel32(&iIsrDfc, 0);

		}

	else

		{

		// DFC runs in another thread. Make sure it's queued and then wait for it to run.

		if (iCancelInfo)

			c.InsertBefore(iCancelInfo);

		else

			iCancelInfo = &c;

		wait = TRUE;

		iDfc.Enque();

		}

	const TUint32 req_count_after = iQueuedRequests;

	Signal();

	if (waiters)

		waiters->Signal();

	if (wait)

		NKern::FSWait(&c.iSem);

 	NKern::ThreadLeaveCS();

	// Only call PSL if there were requests queued when we entered AND there

	// are now no requests left on the queue.

	if ((req_count_before != 0) && (req_count_after == 0))

		{

		QueuedRequestCountChanged();

		}

	__DMA_INVARIANT();

	}

/**

 DFC callback function (static member).

 */

void TDmaChannel::Dfc(TAny* aArg)

	{

	((TDmaChannel*)aArg)->DoDfc();

	}

void TDmaChannel::DoDfc()

	{

	Wait();

	// Atomically fetch and reset the number of DFC queued by ISR and the error

	// flag. Leave the cancel flag alone for now.

	const TUint32 w = __e32_atomic_and_ord32(&iIsrDfc, (TUint32)KCancelFlagMask);

	TUint32 count = w & KDfcCountMask;

	const TBool error = w & (TUint32)KErrorFlagMask;

	TBool stop = w & (TUint32)KCancelFlagMask;

	__DMA_ASSERTD(count>0 || stop);

	const TUint32 req_count_before = iQueuedRequests;

	TUint32 req_count_after = 0;

	while(count && !stop)

		{

		--count;

		// If an error occurred it must have been reported on the last interrupt since transfers are

		// suspended after an error.

		DDmaRequest::TResult res = (count==0 && error) ? DDmaRequest::EError : DDmaRequest::EOk;

		__DMA_ASSERTD(!iReqQ.IsEmpty());

		DDmaRequest* pCompletedReq = NULL;

		DDmaRequest* pCurReq = _LOFF(iReqQ.First(), DDmaRequest, iLink);

		DDmaRequest::TCallback cb = 0;

		TAny* arg = 0;

		if (res == DDmaRequest::EOk)

			{

			// Update state machine, current fragment, completed fragment and

			// tell DMAC to transfer next fragment if necessary.

			SDmaDesHdr* pCompletedHdr = NULL;

			DoDfc(*pCurReq, pCompletedHdr);

			// If just completed last fragment from current request, switch to next

			// request (if any).

			if (pCompletedHdr == pCurReq->iLastHdr)

				{

				pCompletedReq = pCurReq;

				pCurReq->iLink.Deque();

				iQueuedRequests--;

				if (iReqQ.IsEmpty())

					iNullPtr = &iCurHdr;

				pCompletedReq->OnDeque();

				}

			}

		else if (res == DDmaRequest::EError)

			pCompletedReq = pCurReq;

		else

			__DMA_CANT_HAPPEN();

		if (pCompletedReq)

			{

			cb = pCompletedReq->iCb;

			arg = pCompletedReq->iCbArg;

			Signal();

			__KTRACE_OPT(KDMA, Kern::Printf("notifying DMA client result=%d", res));

			(*cb)(res,arg);

			Wait();

			}

		if (pCompletedReq || Flash())

			stop = __e32_atomic_load_acq32(&iIsrDfc) & (TUint32)KCancelFlagMask;

		}

	// Some interrupts may be missed (double-buffer and scatter-gather

	// controllers only) if two or more transfers complete while interrupts are

	// disabled in the CPU. If this happens, the framework will go out of sync

	// and leave some orphaned requests in the queue.

	//

	// To ensure correctness we handle this case here by checking that the request

	// queue is empty when all transfers have completed and, if not, cleaning up

	// and notifying the client of the completion of the orphaned requests.

	//

	// Note that if some interrupts are missed and the controller raises an

	// error while transferring a subsequent fragment, the error will be reported

	// on a fragment which was successfully completed.  There is no easy solution

	// to this problem, but this is okay as the only possible action following a

	// failure is to flush the whole queue.

	if (stop)

		{

		TDmaCancelInfo* waiters = iCancelInfo;

		iCancelInfo = 0;

		// make sure DFC doesn't run again until a new request completes

		iDfc.Cancel();

		// reset the ISR count - new requests can now be processed

		__e32_atomic_store_rel32(&iIsrDfc, 0);

		req_count_after = iQueuedRequests;

		Signal();

		// release threads doing CancelAll()

		waiters->Signal();

		}

	else if (!error && !iDfc.Queued() && !iReqQ.IsEmpty() && iController->IsIdle(*this))

		{

		__KTRACE_OPT(KDMA, Kern::Printf("Missed interrupt(s) - draining request queue"));

		ResetStateMachine();

		// Move orphaned requests to temporary queue so channel queue can

		// accept new requests.

		SDblQue q;

		q.MoveFrom(&iReqQ);

		SDblQueLink* pL;

		while ((pL = q.GetFirst()) != NULL)

			{

			iQueuedRequests--;

			DDmaRequest* pR = _LOFF(pL, DDmaRequest, iLink);

			__KTRACE_OPT(KDMA, Kern::Printf("Removing request from queue and notifying client"));

			pR->OnDeque();

			DDmaRequest::TCallback cb = pR->iCb;

			TAny* arg = pR->iCbArg;

			if (cb)

				{

				Signal();

				(*cb)(DDmaRequest::EOk, arg);

				Wait();

				}

			}

		req_count_after = iQueuedRequests;

		Signal();

		}

	else

		{

		req_count_after = iQueuedRequests;

		Signal();

		}

	// Only call PSL if there were requests queued when we entered AND there

	// are now no requests left on the queue (after also having executed all

	// client callbacks).

	if ((req_count_before != 0) && (req_count_after == 0))

		{

		QueuedRequestCountChanged();

		}

	__DMA_INVARIANT();

	}

/** Reset state machine only, request queue is unchanged */

void TDmaChannel::ResetStateMachine()

	{

	DoCancelAll();

	iCurHdr = NULL;

	iNullPtr = &iCurHdr;

	}

/** Unlink the last item of a LLI chain from the next chain.

	Default implementation does nothing. This is overridden by scatter-gather channels. */

void TDmaChannel::DoUnlink(SDmaDesHdr& /*aHdr*/)

	{

	}

/** PSL may override */

void TDmaChannel::QueuedRequestCountChanged()

	{

#ifdef _DEBUG

	Wait();

	__KTRACE_OPT(KDMA,

				 Kern::Printf("TDmaChannel::QueuedRequestCountChanged() %d",

							  iQueuedRequests));

	__DMA_ASSERTA(iQueuedRequests >= 0);

	Signal();

#endif

	}

#ifdef _DEBUG

void TDmaChannel::Invariant()

	{

	Wait();

	__DMA_ASSERTD(iReqCount >= 0);

	// should always point to NULL pointer ending fragment queue

	__DMA_ASSERTD(*iNullPtr == NULL);

	__DMA_ASSERTD(0 <= iAvailDesCount && iAvailDesCount <= iMaxDesCount);

	__DMA_ASSERTD(iCurHdr == NULL || iController->IsValidHdr(iCurHdr));

	if (IsOpened())

		{

		__DMA_ASSERTD((iCurHdr && !IsQueueEmpty()) || (!iCurHdr && IsQueueEmpty()));

		if (iCurHdr == NULL)

			__DMA_ASSERTD(iNullPtr == &iCurHdr);

		}

	else

		{

		__DMA_ASSERTD(iCurHdr == NULL);

		__DMA_ASSERTD(iNullPtr == &iCurHdr);

		__DMA_ASSERTD(IsQueueEmpty());

		}

	Signal();

	}

#endif

//////////////////////////////////////////////////////////////////////////////

// TDmaSbChannel

void TDmaSbChannel::DoQueue(DDmaRequest& /*aReq*/)

	{

	if (!iTransferring)

		{

		iController->Transfer(*this, *iCurHdr);

		iTransferring = ETrue;

		}

	}

void TDmaSbChannel::DoCancelAll()

	{

	__DMA_ASSERTD(iTransferring);

	iTransferring = EFalse;

	}