Update contrib.
1 // Copyright (c) 2004-2009 Nokia Corporation and/or its subsidiary(-ies).
2 // All rights reserved.
3 // This component and the accompanying materials are made available
4 // under the terms of the License "Eclipse Public License v1.0"
5 // which accompanies this distribution, and is available
6 // at the URL "http://www.eclipse.org/legal/epl-v10.html".
8 // Initial Contributors:
9 // Nokia Corporation - initial contribution.
14 // e32test/buffer/t_huff.cpp
16 // Test methods of the Huffman, TBitInput and TBitOutput classes.
18 // Huffman, TBitInput, TBitOutput
20 // - Test and verify the results of TBitInput bit reading:
21 // - test and verify single bit reads, multiple bit reads and 32-bit reads
22 // - test and verify single bit reads and multiple bit reads from a
24 // - test and verify overrun reads
25 // - Test and verify the results of TBitOutput bit writing:
26 // - test and verify bitstream padding
27 // - test and verify single bit and multiple bit writes
28 // - test and verify overflow writes
29 // - Test and verify the results of a Huffman decoder using Huffman class
30 // static methods, TBitOutput and TBitInput objects.
31 // - Test and verify the results of a Huffman generator for known distributions:
32 // flat, power-of-2 and Fibonacci.
33 // - Test and verify the results of a Huffman generator for random distributions:
34 // - generate random frequency distributions and verify:
35 // (a) the Huffman generator creates a mathematically 'optimal code'
36 // (b) the canonical encoding is canonical
37 // (c) the decoding tree correctly decodes each code
38 // (d) the encoding can be correctly externalised and internalised
39 // Platforms/Drives/Compatibility:
41 // Assumptions/Requirement/Pre-requisites:
42 // Failures and causes:
43 // Base Port information:
49 #include <e32huffman.h>
51 RTest test(RProcess().FileName());
53 const Uint64 KTestData=UI64LIT(0x6f1b09a7e8c523d4);
54 const TUint8 KTestBuffer[] = {0x6f,0x1b,0x09,0xa7,0xe8,0xc5,0x23,0xd4};
55 const TInt KTestBytes=sizeof(KTestBuffer);
56 const TInt KTestBits=KTestBytes*8;
58 // Input stream: bit and multi-bit read tests with exhsautive buffer reload testing
60 typedef TBool (*TestFn)(TBitInput& aIn, Uint64 aBits, TInt aCount);
62 class TAlignedBitInput : public TBitInput
65 TAlignedBitInput(const TUint8*,TInt,TInt);
69 const TUint8* iRemainder;
73 TAlignedBitInput::TAlignedBitInput(const TUint8* aPtr,TInt aCount,TInt aOffset)
74 :TBitInput(aPtr,32-aOffset,aOffset), iRemainder(aPtr+4), iCount(aOffset+aCount-32)
77 void TAlignedBitInput::UnderflowL()
80 User::Leave(KErrUnderflow);
83 Set(iRemainder,iCount);
88 class TSplitBitInput : public TBitInput
91 TSplitBitInput(const TUint8*,TInt,TInt,TInt);
101 TSplitBitInput::TSplitBitInput(const TUint8* aPtr,TInt aLength,TInt aOffset,TInt aSize)
102 :TBitInput(aPtr,aSize,aOffset), iBase(aPtr), iBlockSize(aSize), iOffset(aOffset+aSize), iAvail(aLength-aSize)
105 void TSplitBitInput::UnderflowL()
107 TInt len=Min(iBlockSize,iAvail);
109 User::Leave(KErrUnderflow);
110 Set(iBase,len,iOffset);
115 class TAlternateBitInput : public TBitInput
118 TAlternateBitInput(const TUint8*,TInt,TInt);
127 TAlternateBitInput::TAlternateBitInput(const TUint8* aPtr,TInt aLength,TInt aOffset)
128 :TBitInput(aPtr,1,aOffset), iBase(aPtr), iOffset(aOffset+2), iAvail(aLength-2)
131 void TAlternateBitInput::UnderflowL()
134 User::Leave(KErrUnderflow);
135 Set(iBase,1,iOffset);
140 void TestReader(TBitInput& aIn, TestFn aFunc, Uint64 aBits, TInt aCount)
143 TRAPD(r,eof=aFunc(aIn,aBits,aCount));
148 test (r==KErrUnderflow);
152 void TestBits(TInt aOffset, TInt aCount, TestFn aFunc)
154 Uint64 bits=KTestData;
158 bits&=~((Uint64(1)<<(64-aCount))-1);
159 // test with direct input
160 TBitInput in1(KTestBuffer,aCount,aOffset);
161 TestReader(in1,aFunc,bits,aCount);
162 // test with aligned input
163 if (aOffset<32 && aOffset+aCount>32)
165 TAlignedBitInput in2(KTestBuffer,aCount,aOffset);
166 TestReader(in2,aFunc,bits,aCount);
168 // test with blocked input
169 for (TInt block=aCount;--block>0;)
171 TSplitBitInput in3(KTestBuffer,aCount,aOffset,block);
172 TestReader(in3,aFunc,bits,aCount);
176 void TestAlternateBits(TInt aOffset, TInt aCount, TestFn aFunc)
180 for (TInt ix=aOffset;ix<aOffset+aCount;ix+=2)
182 if (KTestData<<ix>>63)
183 bits|=Uint64(1)<<(63-c);
186 // test with alternate input
187 TAlternateBitInput in1(KTestBuffer,aCount,aOffset);
188 TestReader(in1,aFunc,bits,c);
191 void PermBits(TestFn aFunc, TInt aMinCount=1, TInt aMaxCount=64)
193 for (TInt offset=0;offset<KTestBits;++offset)
194 for (TInt count=Min(KTestBits-offset,aMaxCount);count>=aMinCount;--count)
195 TestBits(offset,count,aFunc);
198 void AlternateBits(TestFn aFunc, TInt aMinCount=1)
200 for (TInt offset=0;offset<KTestBits;++offset)
201 for (TInt count=KTestBits-offset;count>=aMinCount;--count)
202 TestAlternateBits(offset,count,aFunc);
205 TBool SingleBitRead(TBitInput& aIn, Uint64 aBits, TInt aCount)
209 test (aIn.ReadL() == (aBits>>63));
215 TBool MultiBitRead(TBitInput& aIn, Uint64 aBits, TInt aCount)
218 TUint v=aIn.ReadL(c);
223 test (v==TUint(aBits>>(64-c)));
231 test (v==TUint(aBits>>(64-c)));
235 TBool LongShortRead(TBitInput& aIn, Uint64 aBits, TInt aCount)
237 TUint v=aIn.ReadL(32);
238 test (v==TUint(aBits>>32));
245 test (v==TUint(aBits>>(64-c)));
249 TBool ShortLongRead(TBitInput& aIn, Uint64 aBits, TInt aCount)
252 TUint v=aIn.ReadL(c);
257 test (v==TUint(aBits>>(64-c)));
261 test (v==TUint(aBits>>32));
265 TBool EofRead(TBitInput& aIn, Uint64, TInt aCount)
267 TRAPD(r,aIn.ReadL(aCount+1));
268 test(r==KErrUnderflow);
272 void TestBitReading()
274 test.Start(_L("Test single bit reads"));
275 PermBits(&SingleBitRead);
276 test.Next(_L("Test multi bit reads"));
277 PermBits(&MultiBitRead);
278 test.Next(_L("Test 32-bit reads"));
279 PermBits(&LongShortRead,32);
280 PermBits(&ShortLongRead,32);
281 test.Next(_L("Test single bit reads (fractured input)"));
282 AlternateBits(&SingleBitRead);
283 test.Next(_L("Test multi bit reads (fractured input)"));
284 AlternateBits(&MultiBitRead);
285 test.Next(_L("Test overrun reads"));
286 PermBits(&EofRead,1,31);
290 // Bit output testing (assumes bit input is correct)
295 TBitOutput out(buffer,4);
296 test(out.Ptr()==buffer);
297 test(out.BufferedBits()==0);
299 test(out.Ptr()==buffer);
300 test(out.BufferedBits()==0);
303 test(out.Ptr()==buffer);
304 test(out.BufferedBits()==0);
311 test(out.BufferedBits()==(i%8));
313 test(out.BufferedBits()==0);
315 test(out.BufferedBits()==(i%8));
317 test(out.BufferedBits()==0);
318 test (out.Ptr()==buffer+2);
319 test (buffer[0]==(0xff>>i));
320 test (buffer[1]==(0xff>>i));
328 test (out.Ptr()==buffer+1);
329 test (buffer[0]==(0xff^(0xff>>i)));
335 TUint8 buffer[KTestBytes];
336 TBitOutput out(buffer,KTestBytes);
337 TBitInput in(KTestBuffer,KTestBits);
339 for (i=KTestBits;--i>=0;)
340 out.WriteL(in.ReadL(),1);
341 test (Mem::Compare(buffer,KTestBytes,KTestBuffer,KTestBytes)==0);
343 Mem::FillZ(buffer,KTestBytes);
344 out.Set(buffer,KTestBytes);
345 Uint64 bits=KTestData;
346 for (i=KTestBits;--i>=0;)
347 out.WriteL(TUint(bits>>i),1);
348 test (Mem::Compare(buffer,KTestBytes,KTestBuffer,KTestBytes)==0);
351 void TestMultiBitWrites()
354 for (TInt j=0;j<32;++j)
355 for (TInt k=0;k<32;++k)
360 TUint8 buffer[KTestBytes];
361 TBitInput in(KTestBuffer,KTestBits);
362 TBitOutput out(buffer,KTestBytes);
364 out.WriteL(in.ReadL(j),j);
365 out.WriteL(in.ReadL(k),k);
367 const TUint8* p=out.Ptr();
368 test (p-buffer == (j+k+7)/8);
371 v=(v>>8) | Uint64(*--p)<<56;
372 Uint64 res=KTestData;
374 res>>=KTestBits-i-j-k;
381 void TestAlternatingWrites()
383 const TInt KBufferSize=(1+32)*32;
384 TUint8 buffer[(7+KBufferSize)/8];
385 TBitOutput out(buffer,sizeof(buffer));
388 out.WriteL(i&1?0xffffffff:0,i);
390 out.WriteL(i&1?0:0xffffffff,i);
392 TBitInput in(buffer,KBufferSize);
397 test (v == (1u<<i)-1u);
407 test (v == 0xffffffffu);
409 test (v == (1u<<i)-1u);
413 class TOverflowOutput : public TBitOutput
424 TOverflowOutput::TOverflowOutput()
428 void TOverflowOutput::OverflowL()
432 test (Ptr()-iBuf == 1);
433 test (iBuf[0] == KTestBuffer[iIx]);
434 if (++iIx==KTestBytes)
435 User::Leave(KErrOverflow);
440 void OverflowTestL(TBitOutput& out, TInt j)
442 for (;;) out.WriteL(0xffffffffu,j);
447 test.Start(_L("Test default constructed output"));
452 TRAPD(r,out.WriteL(1,1));
455 test (out.BufferedBits() == i);
456 test (r == KErrNone);
459 test (r == KErrOverflow);
462 test.Next(_L("Test overflow does not overrun the buffer"));
464 for (TInt j=1;j<=32;++j)
468 TUint8 buffer[KTestBytes+1];
469 Mem::FillZ(buffer,sizeof(buffer));
471 TRAPD(r,OverflowTestL(out,j));
472 test (r == KErrOverflow);
474 while (buffer[k]==0xff)
477 test (k<TInt(sizeof(buffer)));
481 while (k<TInt(sizeof(buffer)))
488 test.Next(_L("Test overflow handler"));
489 TOverflowOutput vout;
490 TBitInput in(KTestBuffer,KTestBits);
491 for (i=KTestBits;--i>=0;)
492 vout.WriteL(in.ReadL(),1);
493 test(vout.BufferedBits() == 0);
494 TRAPD(r,vout.WriteL(0,1));
495 test (r == KErrNone);
496 TRAP(r,vout.PadL(0));
497 test (r == KErrOverflow);
501 void TestBitWriting()
503 test.Start(_L("Test padding"));
505 test.Next(_L("Test bit writes"));
507 test.Next(_L("Test multi-bit writes"));
508 TestMultiBitWrites();
509 TestAlternatingWrites();
510 test.Next(_L("Test overflow writes"));
515 // Huffman decode testing
519 void Dummy(volatile TInt & /*x*/)
525 const TInt KTestBits=32*32;
527 // build the huffman decoding tree for
531 TUint32 huffman[Huffman::KMaxCodeLength+1];
533 for (i=0;i<Huffman::KMaxCodeLength;++i)
535 huffman[Huffman::KMaxCodeLength]=Huffman::KMaxCodeLength;
536 Huffman::Decoding(huffman,Huffman::KMaxCodeLength+1,huffman);
538 TUint8 buffer[KTestBits/8];
539 for (TInt sz=0;sz<Huffman::KMaxCodeLength;++sz)
541 const TInt rep=KTestBits/(sz+1);
542 TBitOutput out(buffer,sizeof(buffer));
545 out.WriteL(0xffffffff,sz);
549 for (TInt blk=1;blk<=64;++blk)
551 TSplitBitInput in(buffer,rep*(sz+1)-1,0,blk);
552 for (i=0;i<rep-1;++i)
555 TRAPD(r,v=in.HuffmanL(huffman));
561 TRAPD(r, v=in.HuffmanL(huffman));
563 test (r==KErrUnderflow);
568 // Huffman generator testing with known but atypical distributions
570 void FlatHuffman(TInt aMaxCount)
572 TUint32* tab=new TUint32[aMaxCount];
575 // test empty distribution
576 Mem::FillZ(tab,sizeof(TUint32)*aMaxCount);
577 TRAPD(r, Huffman::HuffmanL(tab,aMaxCount,tab));
580 for (i=0;i<aMaxCount;++i)
582 Huffman::Decoding(tab,aMaxCount,tab);
584 // test single-symbol distribution
585 Mem::FillZ(tab,sizeof(TUint32)*aMaxCount);
587 TRAP(r, Huffman::HuffmanL(tab,aMaxCount,tab));
590 for (i=1;i<aMaxCount;++i)
592 Huffman::Decoding(tab,aMaxCount,tab,200);
594 TBitInput in(&bits,1);
595 test (in.HuffmanL(tab)==200);
597 // test flat distributions with 2..aMaxCount symbols
599 for (TInt c=2;c<aMaxCount;++c)
603 Mem::FillZ(tab,sizeof(TUint32)*aMaxCount);
606 TRAP(r, Huffman::HuffmanL(tab,aMaxCount,tab));
611 if (TInt(tab[i])==len)
614 test (TInt(tab[i])==len+1);
616 for (;i<aMaxCount;++i)
618 test (small == (2<<len)-c);
626 // Test Huffman generator for the distribution 2^0,2^0,2^1,2^2,2^3,...
629 TUint32 tab[Huffman::KMaxCodeLength+2];
631 for (TInt c=1;c<=Huffman::KMaxCodeLength+1;c++)
638 TRAPD(r,Huffman::HuffmanL(tab,c+1,tab));
639 if (c>Huffman::KMaxCodeLength)
641 test (r==KErrOverflow);
645 test (TInt(tab[c]) == c);
647 test (TInt(tab[i]) == i+1);
649 Huffman::Decoding(tab,c+1,tab);
653 TBitOutput out(buf,4);
654 out.WriteL(0xffffffff,i);
657 TBitInput in(buf,Min(i+1,c));
659 TRAP(r, ix=in.HuffmanL(tab));
662 TRAP(r, in.HuffmanL(tab));
663 test (r==KErrUnderflow);
668 void FibonacciHuffman()
670 // Test Huffman generator for the distribution 1,1,2,3,5,8,13,21,...
673 TUint32 tab[Huffman::KMaxCodeLength+2];
675 for (TInt c=1;c<=Huffman::KMaxCodeLength+1;c++)
680 tab[i]=tab[i+1]+tab[i+2];
682 TRAPD(r,Huffman::HuffmanL(tab,c+1,tab));
683 if (c>Huffman::KMaxCodeLength)
685 test (r==KErrOverflow);
689 test (TInt(tab[c]) == c);
691 test (TInt(tab[i]) == i+1);
693 Huffman::Decoding(tab,c+1,tab);
697 TBitOutput out(buf,4);
698 out.WriteL(0xffffffff,i);
701 TBitInput in(buf,Min(i+1,c));
703 TRAP(r, ix=in.HuffmanL(tab));
706 TRAP(r, in.HuffmanL(tab));
707 test (r==KErrUnderflow);
712 void SpecificHuffman(TInt aMaxCount)
714 test.Start(_L("Flat distributions"));
715 FlatHuffman(aMaxCount);
716 test.Next(_L("Power-of-2 distributions"));
718 test.Next(_L("Fibonacci distributions"));
723 // Huffman generator validity testing. Checking code properties for a sequence of random
724 // frequency distributions.
726 TInt64 RSeed(KTestData);
728 inline TInt Random(TInt aLimit)
729 {return aLimit>0 ? (Math::Rand(RSeed)%aLimit) : 0;}
731 void GenerateFreq(TUint32* aTable, TInt aCount, TInt aTotalFreq, TInt aVariance, TInt aZeros)
733 // Generate a random frequency table
736 for (TInt i=0;i<aCount;++i)
738 if (aZeros && Random(aCount-i)<aZeros)
743 else if (aCount-aZeros-i == 1)
745 aTable[i]=aTotalFreq;
750 TInt ave=aTotalFreq/(aCount-aZeros-i);
758 TInt var=I64INT(TInt64(ave)<<aVariance>>8);
759 TInt min=Max(1,ave-var);
760 TInt max=Min(1+aTotalFreq-(aCount-aZeros-i),ave+var);
761 TInt f = max<=min ? ave : min+Random(max-min);
769 TInt NumericalSort(const TUint32& aLeft, const TUint32& aRight)
774 TInt64 VerifyOptimalCode(const TUint32* aFreq, const TUint32* aCode, TInt aCount, TInt aTotalFreqLog2)
776 // We can show tht the expected code length is at least as short as a Shannon-Fano encoding
782 for (i=0;i<aCount;++i)
793 while ((f<<s>>aTotalFreqLog2)!=1)
797 test (totalHuff<=totalSF);
799 RPointerArray<TUint32> index(aCount);
800 CleanupClosePushL(index);
801 for (i=0;i<aCount;++i)
804 User::LeaveIfError(index.InsertInOrderAllowRepeats(aFreq+i,&NumericalSort));
808 smin=smax=aCode[index[0]-aFreq];
809 for (i=1;i<index.Count();++i)
811 TInt pix=index[i-1]-aFreq;
812 TInt nix=index[i]-aFreq;
822 test (smin==smax || smin+1==smax);
831 CleanupStack::PopAndDestroy();
836 TInt LexicalSort(const TUint32& aLeft, const TUint32& aRight)
838 const TUint32 KCodeMask=(1<<Huffman::KMaxCodeLength)-1;
839 return (aLeft&KCodeMask)-(aRight&KCodeMask);
842 void VerifyCanonicalEncodingL(const TUint32* aCode, const TUint32* aEncode, TInt aCount)
844 // A canonical encoding assigns codes from '0' in increasing code length order, and
845 // in increasing index in the table for equal code length.
847 // Huffman is also a 'prefix-free' code, so we check this property of the encoding
851 for (i=0;i<aCount;++i)
852 test (aCode[i] == aEncode[i]>>Huffman::KMaxCodeLength);
854 RPointerArray<TUint32> index(aCount);
855 CleanupClosePushL(index);
856 for (i=0;i<aCount;++i)
859 User::LeaveIfError(index.InsertInOrder(aEncode+i,&LexicalSort));
862 for (i=1;i<index.Count();++i)
864 TInt pix=index[i-1]-aEncode;
865 TInt nix=index[i]-aEncode;
866 test (aCode[pix]<=aCode[nix]); // code lengths are always increasing
867 test (aCode[pix]<aCode[nix] || pix<nix); // same code length => index order preserved
869 // check that a code is not a prefix of the next one. This is sufficent for checking the
870 // prefix condition as we have already sorted the codes in lexicographical order
871 TUint32 pc=aEncode[pix]<<(32-Huffman::KMaxCodeLength);
872 TUint32 nc=aEncode[nix]<<(32-Huffman::KMaxCodeLength);
873 TInt plen=aCode[pix];
874 test ((pc>>(32-plen)) != (nc>>(32-plen))); // pc is not a prefix for nc
876 CleanupStack::PopAndDestroy(&index);
879 void VerifyCanonicalDecoding(const TUint32* aEncode, const TUint32* aDecode, TInt aCount, TInt aBase)
881 // We've checked the encoding is valid, so now we check that the decoding can correctly
885 TUint8 buffer[(Huffman::KMaxCodeLength+7)/8];
893 out.Set(buffer,sizeof(buffer));
894 out.HuffmanL(aEncode[aCount]);
896 in.Set(buffer,aEncode[aCount]>>Huffman::KMaxCodeLength);
898 TRAPD(r,v=in.HuffmanL(aDecode));
900 test (v==aCount+aBase);
902 test (r==KErrUnderflow);
907 TInt TestExternalizeL(const TUint32* aCode, TUint8* aExtern, TUint32* aIntern, TInt aCount)
909 TBitOutput out(aExtern,aCount*4);
910 Huffman::ExternalizeL(out,aCode,aCount);
911 TInt bits=out.BufferedBits()+8*(out.Ptr()-aExtern);
913 TBitInput in(aExtern,bits);
914 TRAPD(r,Huffman::InternalizeL(in,aIntern,aCount));
915 test (r == KErrNone);
916 test (Mem::Compare((TUint8*)aCode,aCount*sizeof(TUint32),(TUint8*)aIntern,aCount*sizeof(TUint32)) == 0);
918 test (r == KErrUnderflow);
922 void RandomHuffmanL(TInt aIter, TInt aMaxSymbols)
924 // generate random frequency distributions and verify
925 // (a) the Huffman generator creates a mathematically 'optimal code'
926 // (b) the canonical encoding is the canonical encoding
927 // (c) the decoding tree correctly decodes each code.
928 // (d) the encoding can be correctly externalised and internalised
933 const TInt KTotalFreqLog2=24;
934 const TInt KTotalFreq=1<<KTotalFreqLog2;
938 TInt num=2+Random(aMaxSymbols-1);
940 TUint32* const freq = new(ELeave) TUint32[num*3];
941 CleanupArrayDeletePushL(freq);
942 TUint32* const code = freq+num;
943 TUint32* const encoding = code+num;
944 TUint32* const decoding = freq;
945 TUint8* const exter = (TUint8*)encoding;
946 TUint32* const intern = freq;
949 TInt zero=Random(num-2);
950 GenerateFreq(freq,num,KTotalFreq,var,zero);
952 Huffman::HuffmanL(freq,num,code);
953 VerifyOptimalCode(freq,code,num,KTotalFreqLog2);
955 Huffman::Encoding(code,num,encoding);
956 VerifyCanonicalEncodingL(code,encoding,num);
958 TInt base=Random(Huffman::KMaxCodes-num);
959 Huffman::Decoding(code,num,decoding,base);
960 VerifyCanonicalDecoding(encoding,decoding,num,base);
962 TestExternalizeL(code,exter,intern,num);
963 CleanupStack::PopAndDestroy();
971 test.Start(_L("Test Bit reader"));
973 test.Next(_L("Test Bit writer"));
975 test.Next(_L("Test Huffman decoder"));
977 test.Next(_L("Test Huffman generator for known distributions"));
978 SpecificHuffman(800);
979 test.Next(_L("Test Huffman generator for random distributions"));
980 TRAPD(r,RandomHuffmanL(1000,800));
988 CTrapCleanup* c=CTrapCleanup::New();