sl@0: # Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
sl@0: # All rights reserved.
sl@0: # This component and the accompanying materials are made available
sl@0: # under the terms of "Eclipse Public License v1.0"
sl@0: # which accompanies this distribution, and is available
sl@0: # at the URL "http://www.eclipse.org/legal/epl-v10.html".
sl@0: #
sl@0: # Initial Contributors:
sl@0: # Nokia Corporation - initial contribution.
sl@0: #
sl@0: # Contributors:
sl@0: #
sl@0: # Description:
sl@0: #
sl@0: 
sl@0: use strict;
sl@0: use LWP::Simple;
sl@0: use TestScriptResults;
sl@0: use Getopt::Std;
sl@0: use File::Find;
sl@0: 
sl@0: # Mistral tags the result files from test scripts in a particular format, depending on the test harness used to execute them. 
sl@0: # We use these names to identify which files are results files.
sl@0: my $KMistralCoreConfFileName = "-coreconf-";
sl@0: my $KMistralTEFFileName = "-testexecute-";
sl@0: my $KMistralTestFrameworkFileName = "-testframework"; # No trailing hyphen due to different security variants
sl@0: 
sl@0: #  It appears that Mistral uses a completely different file format for results from HW runs
sl@0: my $KMistrealTEFFileNameHW = "testexecute";
sl@0: my $KMistrealTFFileNameHW1 = "testframework";
sl@0: my $KMistrealTFFileNameHW2 = "testframeworkmmddcap";
sl@0: my $KMistrealTFFileNameHW3 = "testframeworkuecap";
sl@0: my $KMistrealTFFileNameHW4 = "testframeworknone";
sl@0: 
sl@0: # Work in progress script outputs.
sl@0: my $iKnownFails;
sl@0: my $iUnknownFailsButKnownInRef;
sl@0: my $iUnknownFailsButKnownInOtherCodelines;
sl@0: my $iUnknownFails;
sl@0: my $iMissingTests; 	
sl@0: 
sl@0: my $iVerbose;
sl@0: 
sl@0: ###############################################
sl@0: ##	Source test domain specific functionality
sl@0: ###############################################
sl@0: # Scans the test results summary page from the Mistral build and compiles a collection of the 
sl@0: # URL paths for the log files.
sl@0: # param1 - the URL to the Test Results HTML Summary page of the Mistral build
sl@0: # returns - the array of logs' URLS
sl@0: sub GetResultFilesFromMistral
sl@0: 	{
sl@0: 	my $aMistralTestResultsSummaryUrl = shift;
sl@0: 
sl@0: 	my @fileNames;
sl@0: 	my $nextFileNamesIndex = 0;
sl@0: 
sl@0: 	my $summaryContents = get "$aMistralTestResultsSummaryUrl"; # Returns undef if failure
sl@0: 	unless ($summaryContents)
sl@0: 		{
sl@0: 		print "\nERROR: Unable to retrieve source summary file from $aMistralTestResultsSummaryUrl\n";
sl@0: 		exit;
sl@0: 		}
sl@0: 	
sl@0: 	#Could add an optimisation to this to search for the <td class="error"> tag BUT doesn't take into account the crashed tests and would need to check here the pass/fail count
sl@0: 	#In addition this might lead to migration problems with Sirroco and certainly isn't appliable to developer's personal builds where we must scan each file
sl@0: 	#So it's simpler to do the brute force approach and keep all implementations alike, it doesn't take long anyway
sl@0: 	while ($summaryContents =~ s/href="(.*?($KMistralCoreConfFileName|$KMistralTEFFileName|$KMistralTestFrameworkFileName).*?\.html?)"//i)
sl@0: 		{
sl@0: 		$fileNames[$nextFileNamesIndex] = $1;
sl@0: 		$nextFileNamesIndex++;
sl@0: 		}
sl@0: 
sl@0: 	# Scan this second so we don't accidentally cut off the filenames if they are matched against Winscw runs
sl@0: 	while ($summaryContents =~ s/href="(.*?($KMistrealTEFFileNameHW|$KMistrealTFFileNameHW1|$KMistrealTFFileNameHW2|$KMistrealTFFileNameHW3|$KMistrealTFFileNameHW4).*?\.html?)"//i)
sl@0: 		{
sl@0: 		$fileNames[$nextFileNamesIndex] = $1;
sl@0: 		$nextFileNamesIndex++;
sl@0: 		}
sl@0: 	
sl@0: 	unless ($fileNames[0])
sl@0: 		{
sl@0: 		print "\nERROR: No test result log files found, please ensure the -s parameter points to Mistral's TestResults.html summary page\n";	
sl@0: 		exit;
sl@0: 		}
sl@0: 		
sl@0: 	return @fileNames;
sl@0: 	}
sl@0: 
sl@0: # Parse the Mistral test results summary page and generate an array of TestScriptResults
sl@0: # objects which contain all the required information such as script names, test counts
sl@0: # and failures.
sl@0: # param1 - the URL of the Test Results HTML Summary page for the Mistral build 
sl@0: # returns - the array of parsed log data
sl@0: sub PopulateResultsArrayFromMistral
sl@0: 	{
sl@0: 	my $aTestResultsSummaryPageUrl = shift;
sl@0: 	my @aResultsArray = ();
sl@0: 	my $nextFreeIndexInResultsArray = 0;
sl@0: 	
sl@0: 	my @fileNames = GetResultFilesFromMistral($aTestResultsSummaryPageUrl);
sl@0: 	foreach my $fileName (@fileNames)
sl@0: 		{
sl@0: 		my $testFileContents = get "$fileName";
sl@0: 		unless ($testFileContents)
sl@0: 			{
sl@0: 			print "\nERROR: Unable to open logfile $fileName\n";
sl@0: 			next;
sl@0: 			}
sl@0: 			
sl@0: 		my $scriptResults = TestScriptResults->TestScriptResults();
sl@0: 		$scriptResults->SetFilePath($fileName);		
sl@0: 		if (($fileName =~ m/.*?$KMistralTEFFileName(.*?)\.html?/i) || ($fileName =~ m/.*?$KMistrealTEFFileNameHW(.*?)\.html?/i))
sl@0: 			{
sl@0: 			$scriptResults->SetScriptName($1);
sl@0: 			if (!($testFileContents =~ m/TEST CASE SUMMARY:/i) && ($testFileContents =~ m/SUMMARY:/i))
sl@0: 				{
sl@0: 				# One of the old MM TEF tests which didn't use testcases
sl@0: 				$scriptResults->TEFNoTestcasesTest();
sl@0: 				$aResultsArray[$nextFreeIndexInResultsArray] = ScanTEFLogFileForFailures($scriptResults, \$testFileContents, 0);
sl@0: 				}
sl@0: 			else
sl@0: 				{
sl@0: 				$scriptResults->TEFTest();
sl@0: 				$aResultsArray[$nextFreeIndexInResultsArray] = ScanTEFLogFileForFailures($scriptResults, \$testFileContents, 1);
sl@0: 				}	
sl@0: 			}
sl@0: 		elsif ($fileName =~ m/.*?$KMistralCoreConfFileName(.*?)\.txt\.html?/i)
sl@0: 			{
sl@0: 			$scriptResults->SetScriptName($1);
sl@0: 			$scriptResults->CoreConfTest();
sl@0: 			$aResultsArray[$nextFreeIndexInResultsArray] = ScanCoreConfLogFileForFailures($scriptResults, \$testFileContents);
sl@0: 			}
sl@0: 		elsif (($fileName =~ m/.*?$KMistralTestFrameworkFileName[^\-]*?\-(.*?)\.html?/i) || ($fileName =~ m/.*?$KMistrealTFFileNameHW2(.*?)\.html?/i) || ($fileName =~ m/.*?$KMistrealTFFileNameHW3(.*?)\.html?/i) || ($fileName =~ m/.*?$KMistrealTFFileNameHW4(.*?)\.html?/i) || ($fileName =~ m/.*?$KMistrealTFFileNameHW1(.*?)\.html?/i))
sl@0: 			{
sl@0: 			$scriptResults->SetScriptName($1);
sl@0: 			$scriptResults->TestFrameworkTest();
sl@0: 			
sl@0: 			$aResultsArray[$nextFreeIndexInResultsArray] = ScanTestFrameworkLogFileForFailures($scriptResults, \$testFileContents);
sl@0: 			}
sl@0: 		else
sl@0: 			{
sl@0: 			print "\nWARNING: Results file has unrecognised format - $fileName.\n";			
sl@0: 			}
sl@0: 		$nextFreeIndexInResultsArray++;	
sl@0: 		}
sl@0: 	
sl@0: 	return \@aResultsArray;
sl@0: 	}
sl@0: 	
sl@0: 
sl@0: # Walk the file path provided by the developer for his local machine, parse test logs
sl@0: #  and generate an array of TestScriptResults objects which contain all the required 
sl@0: # information such as script names, test counts and failures.
sl@0: # param1 - the pathname of a folder containing the test results
sl@0: # returns - the array of parsed log data
sl@0: sub PopulateResultsArrayFromLocalMachine
sl@0: 	{
sl@0: 	my $aTestResultsPath = shift;
sl@0: 	my @aResultsArray = ();
sl@0: 	my $nextFreeIndexInResultsArray = 0;
sl@0: 
sl@0: 	my @directories = ("$aTestResultsPath");
sl@0: 	my @fileNames;
sl@0: 	find( sub { push @fileNames, $File::Find::name if /\.html?$/ }, @directories );
sl@0: 
sl@0: 	foreach my $fileName (@fileNames)
sl@0: 		{
sl@0: 		my $testFileContents = do { local $/; open(I,"$fileName"); <I> };
sl@0: 		my $scriptResults = TestScriptResults->TestScriptResults();
sl@0: 		$scriptResults->SetFilePath($fileName);
sl@0: 		$fileName =~ m/([^\\\/\.]*)\.[^\\\/]*\Z/;
sl@0: 		$scriptResults->SetScriptName($1);
sl@0: 		
sl@0: 		if ($testFileContents =~ m/Core Loader Conformance Suite/)
sl@0: 			{
sl@0: 			$scriptResults->CoreConfTest();
sl@0: 			$aResultsArray[$nextFreeIndexInResultsArray] = ScanCoreConfLogFileForFailures($scriptResults, \$testFileContents);
sl@0: 			}
sl@0: 		elsif ($testFileContents =~ m/TestFrameworkMain.cpp/)
sl@0: 			{
sl@0: 			$scriptResults->TestFrameworkTest();
sl@0: 			$aResultsArray[$nextFreeIndexInResultsArray] = ScanTestFrameworkLogFileForFailures($scriptResults, \$testFileContents);
sl@0: 			}
sl@0: 		elsif ($testFileContents =~ m/TEF Version/)
sl@0: 			{
sl@0: 			if ($testFileContents =~ m/TEST CASE SUMMARY:/i)
sl@0: 				{
sl@0: 				$scriptResults->TEFTest();
sl@0: 				$aResultsArray[$nextFreeIndexInResultsArray] = ScanTEFLogFileForFailures($scriptResults, \$testFileContents, 1);
sl@0: 				}
sl@0: 			else
sl@0: 				{
sl@0: 				# One of the old MM TEF tests which didn't use testcases
sl@0: 				$scriptResults->TEFNoTestcasesTest();
sl@0: 				$aResultsArray[$nextFreeIndexInResultsArray] = ScanTEFLogFileForFailures($scriptResults, \$testFileContents, 0);
sl@0: 				}			
sl@0: 			}
sl@0: 		else
sl@0: 			{
sl@0: 			print "\nWARNING: Results file has unrecognised format - $fileName.\n";	
sl@0: 			}
sl@0: 					
sl@0: 		$nextFreeIndexInResultsArray++;	
sl@0: 		}
sl@0: 	
sl@0: 	return \@aResultsArray;
sl@0: 	}
sl@0: 	
sl@0: ###############################################
sl@0: ##	Test harness specific functionality
sl@0: ###############################################	
sl@0: 
sl@0: # Parses a TF log file for failures, inconclusives and crashes
sl@0: # param1 - the TestScriptResults object to populate
sl@0: # param2 - reference to the contents of the log file
sl@0: # returns - the TestScriptResults object containing the parsed data
sl@0: sub ScanTestFrameworkLogFileForFailures
sl@0: 	{
sl@0: 	my ($aScriptResults, $aLogFileContents) = @_;
sl@0: 	
sl@0: 	my $testFileContents = $$aLogFileContents;
sl@0: 	my $numberOfTests = 0;
sl@0: 	# Search for test case results, which take the following form:
sl@0: 	#04/09/2002 15:51:39:772    V   Log.cpp 736 Test Result for TSI_ICL_BTRANS_01:MM_ICL_BTRANS_I_0202_CP is PASS
sl@0: 	while ($testFileContents =~ s/Test[\s]+Result[\s]+for[\s]+[\S]+:([\_|\-|\S]+)[\s]+is[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)
sl@0: 		{
sl@0: 		my $testId  = $1;
sl@0: 	    my $result = $2;
sl@0: 	    $numberOfTests++;
sl@0: 		if ($result =~ /PASS/)
sl@0: 			{
sl@0: 			# Do nothing
sl@0: 			}
sl@0: 		elsif ($result =~ /INCONCLUSIVE/)
sl@0: 			{
sl@0: 			$aScriptResults->AddInconclusiveResult($testId);
sl@0: 			}
sl@0: 		else
sl@0: 			{
sl@0: 			# Treat all other results as failure
sl@0: 			$aScriptResults->AddFailureResult($testId);
sl@0: 			}
sl@0: 		}
sl@0: 	$aScriptResults->SetTotalTestCount($numberOfTests);
sl@0: 	
sl@0: 	unless ($testFileContents =~ m/Test Results Summary/)
sl@0: 		{
sl@0: 		# Test file summary not in the log file - the test has crashed
sl@0: 		$aScriptResults->TestCrashed();
sl@0: 		}
sl@0: 	
sl@0: 	return $aScriptResults;
sl@0: 	}
sl@0: 
sl@0: # Parses a TEF log file for failures, inconclusives and crashes
sl@0: # param1 - the TestScriptResults object to populate
sl@0: # param2 - reference to the contents of the log file
sl@0: # param3 - boolean whether the test is using testcases (older MM tests didn't, in which case we use the test steps)
sl@0: # returns - the TestScriptResults object containing the parsed data
sl@0: sub ScanTEFLogFileForFailures
sl@0: 	{
sl@0: 	my ($aScriptResults, $aLogFileContents, $aTestcasesInUse) = @_;
sl@0: 	
sl@0: 	my $testFileContents = $$aLogFileContents;
sl@0: 	my $numberOfTests = 0;
sl@0: 
sl@0: 	# Search for test case results, which take the following form:
sl@0: 	# 02:56:42:145 c:\mm\tsu_3gp_compose_api_te.script Line = 58 Command = END_TESTCASE MM-3GP-COMP-U-0003-CP ***TestCaseResult = PASS 
sl@0: 	# Or if testcases not in use in this script file test step results, which take the following form:
sl@0: 	# 14:20:51:459 c:\mm\tsu_mmf_aclnt_securedrmtestdata.script Line = 36 Command = RUN_TEST_STEP 1000 RTAUtils ImportDrmArchive c:\mm\tsu_mmf_aclnt_securedrmtestdata.ini ImportSmallWavDrmArchiveAudio ***Result = UNEXECUTED 
sl@0: 	while (($testFileContents =~ s/Command[\s]+=[\s]+END\_TESTCASE[\s]+([\_|\-|\S]+)[\s]+\*\*\*TestCaseResult[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)
sl@0: 			|| (!$aTestcasesInUse && ($testFileContents =~ s/Command[\s]+=[\s]+RUN\_TEST\_STEP[\s]+\d*[\s]+.*?[\s]+([\_|\-|\S]+)[\s]+.*?\*\*\*Result[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)))
sl@0: 		{
sl@0: 		my $testId  = $1;
sl@0: 	    my $result = $2;
sl@0: 	    $numberOfTests++;
sl@0: 		if ($result =~ /PASS/)
sl@0: 			{
sl@0: 			# Do nothing
sl@0: 			}
sl@0: 		elsif ($result =~ /INCONCLUSIVE/)
sl@0: 			{
sl@0: 			$aScriptResults->AddInconclusiveResult($testId);
sl@0: 			}
sl@0: 		else
sl@0: 			{
sl@0: 			# Treat all other results as failure
sl@0: 			$aScriptResults->AddFailureResult($testId);
sl@0: 			}
sl@0: 		}
sl@0: 	$aScriptResults->SetTotalTestCount($numberOfTests);
sl@0: 	
sl@0: 	# Testcase scripts use TEST CASE SUMMARY, non-testcase scripts just use SUMMARY
sl@0: 	unless ($testFileContents =~ m/SUMMARY:/)
sl@0: 		{
sl@0: 		# Test file summary not in the log file - the test has crashed
sl@0: 		$aScriptResults->TestCrashed();
sl@0: 		}
sl@0: 	
sl@0: 	return $aScriptResults;	
sl@0: 	}
sl@0: 
sl@0: # Parses a CoreConf log file for failures, inconclusives and crashes
sl@0: # param1 - the TestScriptResults object to populate
sl@0: # param2 - reference to the contents of the log file
sl@0: # returns - the TestScriptResults object containing the parsed data	
sl@0: sub ScanCoreConfLogFileForFailures
sl@0: 	{
sl@0: 	my ($aScriptResults, $aLogFileContents) = @_;
sl@0: 	my $testFileContents = $$aLogFileContents;
sl@0: 	
sl@0: 	if ($testFileContents =~ s/\*\*\*\s*?Summary of tests executed(.*\n)*//)
sl@0: 		{
sl@0: 		my $numberOfTests = 0;
sl@0: 
sl@0: 		# Parse the summary listings
sl@0: 		while($testFileContents =~ s/\*\*\*\s*?Passed tests:((.*\n)*?)\*\*\*\s*?Failed tests:.*\n((\*\*\*.*\n)*)//)
sl@0: 			{
sl@0: 			my $passingTests = $1;
sl@0: 			my $failingTests = $3;
sl@0: 			
sl@0: 			# Passing tests
sl@0: 			while ($passingTests =~ s/\*\*\*[\s]*?\S+[\s]*\n//)
sl@0: 				{
sl@0: 				$numberOfTests++;
sl@0: 				}
sl@0: 			
sl@0: 			# Failing tests
sl@0: 			while ($failingTests =~ s/\*\*\*[\s]*?(\S+)[\s]*\n//)
sl@0: 				{
sl@0: 				$aScriptResults->AddFailureResult($1);
sl@0: 				$numberOfTests++;
sl@0: 				}			
sl@0: 			}
sl@0: 
sl@0: 		$aScriptResults->SetTotalTestCount($numberOfTests);		
sl@0: 		}
sl@0: 	else
sl@0: 		{
sl@0: 		# Test file summary not in the log file - the test has crashed
sl@0: 		$aScriptResults->TestCrashed();		
sl@0: 		}
sl@0: 	
sl@0: 	return $aScriptResults;		
sl@0: 	}
sl@0: 
sl@0: # Gets the test case count from a Test Execute Framework log file
sl@0: # param1 - a reference to the contents of the TEF log file
sl@0: #returns - The number of test cases in the script or -1 if the summary could not be found
sl@0: sub GetTEFTestCount
sl@0: 	{
sl@0: 	my $aReferenceLogContents = shift;
sl@0: 	my $refContents = $$aReferenceLogContents;
sl@0: 
sl@0: 	if ($refContents =~ m/TEST CASE SUMMARY:(.*\n)*?.*?PASS =\s*(\d*)(.*\n)*?.*?FAIL =\s*(\d*)(.*\n)*?.*?INCONCLUSIVE =\s*(\d*)/)
sl@0: 		{
sl@0: 		my $result = $2 + $4 + $6;
sl@0: 		return $result;
sl@0: 		}
sl@0: 	
sl@0: 	if ($refContents =~ m/SUMMARY:(.*\n)*?.*?PASS =\s*(\d*)(.*\n)*?.*?FAIL =\s*(\d*)(.*\n)*?.*?ABORT =\s*(\d*)(.*\n)*?.*?PANIC =\s*(\d*)(.*\n)*?.*?INCONCLUSIVE =\s*(\d*)(.*\n)*?.*?UNKNOWN =\s*(\d*)(.*\n)*?.*?UNEXECUTED =\s*(\d*)/)	
sl@0: 		{
sl@0: 		# One of the MM tests that doesn't use testcases
sl@0: 		my $result = $2 + $4 + $6 + $8 + $10 + $12 + $14;
sl@0: 		return $result;
sl@0: 		}									
sl@0: 	
sl@0: 	# Summary not found, we crashed
sl@0: 	return -1;
sl@0: 	}
sl@0: 
sl@0: # Gets the test count from a Test Framework log file
sl@0: # param1 - a reference to the contents of the Test Framework log file
sl@0: #returns - The number of test cases in the script or -1 if the summary could not be found	
sl@0: sub GetTestFrameworkTestCount
sl@0: 	{
sl@0: 	my $aReferenceLogContents = shift;
sl@0: 	my $refContents = $$aReferenceLogContents;
sl@0: 	
sl@0: 	unless ($refContents =~ m/Test Results Summary(.*\n)*?.*?Total\s*:(\d*)\s*\n/)
sl@0: 		{
sl@0: 		# Summary not found, we crashed
sl@0: 		return -1;
sl@0: 		}
sl@0: 	return $2;
sl@0: 	}
sl@0: 
sl@0: # Gets the test count from an OpenMAX IL Core Conformance log file
sl@0: # param1 - a reference to the contents of the Core Conformance log file
sl@0: #returns - The number of test cases in the script or -1 if the summary could not be found	
sl@0: sub GetCoreConfTestCount
sl@0: 	{
sl@0: 	my $aReferenceLogContents = shift;
sl@0: 	my $refContents = $$aReferenceLogContents;
sl@0: 	
sl@0: 	unless ($refContents =~ m/\*\*\*\s*Summary of tests executed(.*\n)*?\*\*\*\s*Total :\s*(\d*)\s*\n/)
sl@0: 		{
sl@0: 		# Summary not found, we crashed
sl@0: 		return -1;
sl@0: 		}
sl@0: 	return $2;
sl@0: 	}
sl@0: 
sl@0: # Scans a TEF log looking for the test specified and checks if it returned inconclusive
sl@0: # param1 - the test name
sl@0: # param2 - reference to the contents of the log file, the test case result is removed from the argument if passed by ref
sl@0: # param3 - whether the TEF script uses testcases
sl@0: # returns - 1 if it returned inconclusive, 0 otherwise	
sl@0: sub MatchingTEFInconclusiveResult	
sl@0: 	{
sl@0: 	my $aInconclusive = shift;
sl@0: 	my $aReferenceLogContents = shift;
sl@0: 	my $aTestcasesInUse = shift;
sl@0: 	my $refContents = $$aReferenceLogContents;
sl@0: 	if ($aTestcasesInUse)
sl@0: 		{
sl@0: 		if ($refContents =~ s/Command[\s]+=[\s]+END\_TESTCASE[\s]+$aInconclusive[\s]+\*\*\*TestCaseResult[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)
sl@0: 			{
sl@0: 			if ($1 eq "INCONCLUSIVE")
sl@0: 				{
sl@0: 				return 1;
sl@0: 				}
sl@0: 			}
sl@0: 		}
sl@0: 	else
sl@0: 		{
sl@0: 		if ($refContents =~ s/Command[\s]+=[\s]+RUN\_TEST\_STEP[\s]+\d*[\s]+.*?[\s]+$aInconclusive[\s]+.*?\*\*\*Result[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)
sl@0: 			{
sl@0: 			if ($1 eq "INCONCLUSIVE")
sl@0: 				{
sl@0: 				return 1;
sl@0: 				}
sl@0: 			}
sl@0: 		}
sl@0: 	return 0;
sl@0: 	}
sl@0: 
sl@0: # Scans a TestFramework log looking for the test specified and checks if it returned inconclusive
sl@0: # param1 - the test name
sl@0: # param2 - reference to the contents of the log file, the test case result is removed from the argument if passed by ref
sl@0: # returns - 1 if it returned inconclusive, 0 otherwise		
sl@0: sub MatchingTFInconclusiveResult 
sl@0: 	{
sl@0: 	my $aInconclusive = shift;
sl@0: 	my $aReferenceLogContents = shift;
sl@0: 	my $refContents = $$aReferenceLogContents;
sl@0: 	if ($refContents =~ s/Test[\s]+Result[\s]+for[\s]+[\S]+:$aInconclusive[\s]+is[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)
sl@0: 		{
sl@0: 		if ($1 eq "INCONCLUSIVE")
sl@0: 			{
sl@0: 			return 1;
sl@0: 			}
sl@0: 		}
sl@0: 	return 0;
sl@0: 	}
sl@0: 
sl@0: # Scans a TEF log looking for the test specified and checks if it returned an error
sl@0: # param1 - the test name
sl@0: # param2 - reference to the contents of the log file, the test case result is removed from the argument if passed by ref
sl@0: # param3 - whether the TEF script uses testcases
sl@0: # returns - 1 if it returned an error, 0 otherwise		
sl@0: sub MatchingTEFErrorResult 
sl@0: 	{
sl@0: 	my $aFail = shift;
sl@0: 	my $aReferenceLogContents = shift;
sl@0: 	my $aTestcasesInUse = shift;
sl@0: 	my $refContents = $$aReferenceLogContents;
sl@0: 	
sl@0: 	if ($aTestcasesInUse)
sl@0: 		{
sl@0: 		if (not $refContents =~ s/Command[\s]+=[\s]+END\_TESTCASE[\s]+$aFail[\s]+\*\*\*TestCaseResult[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)
sl@0: 			{
sl@0: 			return 0;
sl@0: 			}
sl@0: 	
sl@0: 		if (($1 eq "PASS") || ($1 eq "INCONCLUSIVE"))
sl@0: 			{
sl@0: 			return 0;
sl@0: 			}
sl@0: 		}
sl@0: 	else
sl@0: 		{
sl@0: 		if (not $refContents =~ s/Command[\s]+=[\s]+RUN\_TEST\_STEP[\s]+\d*[\s]+.*?[\s]+$aFail[\s]+.*?\*\*\*Result[\s]+=[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)
sl@0: 			{
sl@0: 			return 0;
sl@0: 			}
sl@0: 	
sl@0: 		if (($1 eq "PASS") || ($1 eq "INCONCLUSIVE"))
sl@0: 			{
sl@0: 			return 0;
sl@0: 			}
sl@0: 		}
sl@0: 		
sl@0: 	return 1;
sl@0: 	}	
sl@0: 	
sl@0: # Scans a TestFramework log looking for the test specified and checks if it returned an error
sl@0: # param1 - the test name
sl@0: # param2 - reference to the contents of the log file, the test case result is removed from the argument if passed by ref
sl@0: # returns - 1 if it returned an error, 0 otherwise		
sl@0: sub MatchingTFErrorResult 
sl@0: 	{
sl@0: 	my $aFail = shift;
sl@0: 	my $aReferenceLogContents = shift;
sl@0: 	my $refContents = $$aReferenceLogContents;
sl@0: 	if (not $refContents =~ s/Test[\s]+Result[\s]+for[\s]+[\S]+:$aFail[\s]+is[\s]+([\_|\-|\S]+)[\s]*\<\/font\>//)
sl@0: 		{
sl@0: 		return 0;
sl@0: 		}
sl@0: 	
sl@0: 	if (($1 eq "PASS") || ($1 eq "INCONCLUSIVE"))
sl@0: 		{
sl@0: 		return 0;
sl@0: 		}
sl@0: 		
sl@0: 	return 1;
sl@0: 	}	
sl@0: 	
sl@0: # N.B. VERY IMPORTANT that the log file is passed by reference.  Core conf scripts can include numerous
sl@0: # different tests that use the same test name so to avoid false positives from this function the 
sl@0: # substitution done in the comparison MUST affect the caller's copy of the log file.
sl@0: sub MatchingCoreConfErrorResult
sl@0: 	{	
sl@0: 	my $aFail = shift;
sl@0: 	my $aReferenceLogContents = shift;
sl@0: 	my $refContents = $$aReferenceLogContents;
sl@0: 	
sl@0: 	if ($refContents =~ s/\*\*\*[\s]*$aFail[\s]*FAILED//)
sl@0: 		{
sl@0: 		return 1;
sl@0: 		}
sl@0: 
sl@0: 	return 0;	
sl@0: 	}
sl@0: 	
sl@0: ###############################################		
sl@0: ##	Failure comparison functions
sl@0: ###############################################		
sl@0: 
sl@0: # The analysis takes an array of TestScriptResults, each storing the results of a test script's log file.
sl@0: # It processes this list in stages looking for any scripts with errors and trying to match them against
sl@0: # entries in a Known Failure sheet and/or comparison reference build.  Whenever a match is found
sl@0: # the error is removed from the TestScriptResults object, these essentially act as an object containing
sl@0: # unresolved errors.  When errors are matched their details are added to the corresponding text 
sl@0: # variables at the top of this file (e.g. iKnownFails).
sl@0: sub AnalyseFailures
sl@0: 	{
sl@0: 	my ($aKFSheetLocation, $aReferenceMistralURL, $aIgnoreFailsNotInRef, $aResultsRef, $aKfPlatforms) = @_;
sl@0: 
sl@0: 	my @results = @$aResultsRef;
sl@0: 	
sl@0: 	my $refContents = undef;
sl@0: 	if ($aReferenceMistralURL)
sl@0: 		{
sl@0: 		$refContents = get "$aReferenceMistralURL"; # Returns undef if failure
sl@0: 		unless ($refContents)
sl@0: 			{
sl@0: 			print "\nERROR: Unable to retrieve reference summary file from $aReferenceMistralURL\n";
sl@0: 			exit;
sl@0: 			}		
sl@0: 		}
sl@0: 	
sl@0: 	my $kfSheetContents = undef;
sl@0: 	if ($aKFSheetLocation)
sl@0: 		{
sl@0: 		unless ($aKFSheetLocation =~ m/\.xml\Z/i)
sl@0: 			{
sl@0: 			print "\nERROR: KF sheet ($aKFSheetLocation) does not appear to be in XML format, you should save the Excel worksheet as XML Spreadsheet 2003 format for input to this script.\n";	
sl@0: 			exit;
sl@0: 			}		
sl@0: 		$kfSheetContents = do { local $/; open(I,"$aKFSheetLocation"); <I> };
sl@0: 		unless ($kfSheetContents)
sl@0: 			{
sl@0: 			print "\nERROR: Unable to open KF sheet at $aKFSheetLocation\n";	
sl@0: 			exit;			
sl@0: 			}
sl@0: 		}
sl@0: 		
sl@0: 	foreach my $scriptResults (@results)
sl@0: 		{
sl@0: 		my $scriptName = $scriptResults->ScriptName();
sl@0: 		
sl@0: 		my $referenceResults = undef;
sl@0: 		
sl@0: 		# If a reference build was provided, find the corresponding log file
sl@0: 		if ($refContents)
sl@0: 			{
sl@0: 			# We use substitution here because some of the coreconf test scripts
sl@0: 			# have the same script name.  This ensures we don't keep referencing
sl@0: 			# against the same copy.  We cannot do the comparison on the script 
sl@0: 			# path because this isn't resilient to adding new tests or cross domain 
sl@0: 			# comparisons.
sl@0: 			# N.B. This assumes the test list ordering remains the same.
sl@0: 			# N.B. The optional .txt is for Core Conf tests
sl@0: 			$refContents =~ s/href="(.*?$scriptName(\.txt)?(\.script)?\.html?)"//;
sl@0: 			my $file = $1;
sl@0: 			if ($file eq undef)
sl@0: 				{
sl@0: 				if ($aIgnoreFailsNotInRef)
sl@0: 					{
sl@0: 					next;
sl@0: 					}
sl@0: 				print "\nWARNING: Unable to find $scriptName in Reference Build\n";	
sl@0: 				}
sl@0: 			else
sl@0: 				{
sl@0: 				$referenceResults = get "$file";	# returns undef on failure
sl@0: 				unless ($referenceResults)
sl@0: 					{
sl@0: 					print "\nWARNING: Unable to open $scriptName ($file) in Reference Build\n";			
sl@0: 					}
sl@0: 				}
sl@0: 			}
sl@0: 		
sl@0: 		my $refTestCrashed;
sl@0: 		# Check the test count hasn't decreased
sl@0: 		if ($referenceResults)
sl@0: 			{
sl@0: 			$refTestCrashed = CheckTestCount($scriptResults, \$referenceResults);
sl@0: 			}
sl@0: 			
sl@0: 		if (not $scriptResults->AnyFailures())
sl@0: 			{
sl@0: 			# No errors so move onto the next
sl@0: 			next;
sl@0: 			}
sl@0: 		
sl@0: 		if ($kfSheetContents)
sl@0: 			{
sl@0: 			ResolveFailuresAgainstKFSheet($scriptResults, \$kfSheetContents, $aKfPlatforms); 
sl@0: 		
sl@0: 			if (not $scriptResults->AnyFailures())
sl@0: 				{
sl@0: 				# All errors resolved against KF sheet so move onto the next
sl@0: 				next;
sl@0: 				}
sl@0: 			}	
sl@0: 		
sl@0: 		if ($referenceResults)
sl@0: 			{
sl@0: 			ResolveFailuresAgainstReferenceRun($scriptResults, $refTestCrashed, \$referenceResults);
sl@0: 			
sl@0: 			if (not $scriptResults->AnyFailures())
sl@0: 				{
sl@0: 				# All errors resolved against reference run so move onto the next
sl@0: 				next;
sl@0: 				}			
sl@0: 			}
sl@0: 		
sl@0: 		# Unknown failures
sl@0: 		$iUnknownFails = $iUnknownFails . "\n$scriptName\n";
sl@0: 		
sl@0: 		my $inconText;
sl@0: 		my @inconclusives = $scriptResults->Inconclusives();
sl@0: 		foreach my $inconclusive (@inconclusives)
sl@0: 			{
sl@0: 			$inconText = $inconText . "$inconclusive\n";
sl@0: 			}
sl@0: 		my $failText;
sl@0: 		my @failures = $scriptResults->Failures();
sl@0: 		foreach my $failure (@failures)
sl@0: 			{
sl@0: 			$failText = $failText . "$failure\n";
sl@0: 			}	
sl@0: 			
sl@0: 		if ($inconText)
sl@0: 			{
sl@0: 			$iUnknownFails = $iUnknownFails . "INCONCLUSIVES:\n$inconText";
sl@0: 			}
sl@0: 		if ($failText)
sl@0: 			{
sl@0: 			$iUnknownFails = $iUnknownFails . "FAILS:\n$failText";
sl@0: 			}
sl@0: 		if ($scriptResults->DidItCrash())	
sl@0: 			{
sl@0: 			$iUnknownFails = $iUnknownFails . "CRASHED\n";
sl@0: 			}
sl@0: 		}	
sl@0: 	
sl@0: 	}
sl@0: 
sl@0: # Will compare the test count of the test in question against the equivalent result from a reference build's log
sl@0: # to detect if there has been a reduction in the total number of tests run as part of the script, unless that script 
sl@0: # has crashed.  As a side effect the return value indicates whether the test in question crashed or not (saves
sl@0: # having to scan it twice).
sl@0: # param1 - the log results to compare against the reference build's run
sl@0: # param2 - the contents of the reference build's corresponding test log
sl@0: # returns = 1 if the reference script crashed and we were unable to compare the counts, 0 otherwise
sl@0: sub CheckTestCount
sl@0: 	{
sl@0: 	my ($aTestResultsObject, $aReferenceLogContents) = @_;
sl@0: 	
sl@0: 	my $testHarness = $aTestResultsObject->TestHarness();
sl@0: 	my $refTestCount;
sl@0: 	if (($testHarness eq $TestScriptResults::KTEFTest) || ($testHarness eq $TestScriptResults::KTEFNoTestcasesTest))
sl@0: 		{
sl@0: 		$refTestCount = GetTEFTestCount($aReferenceLogContents);
sl@0: 		}
sl@0: 	elsif ($testHarness eq $TestScriptResults::KTestFrameworkTest)
sl@0: 		{
sl@0: 		$refTestCount = GetTestFrameworkTestCount($aReferenceLogContents);
sl@0: 		}
sl@0: 	else
sl@0: 		{
sl@0: 		$refTestCount = GetCoreConfTestCount($aReferenceLogContents);
sl@0: 		}
sl@0: 	
sl@0: 	if ($refTestCount < 0)
sl@0: 		{
sl@0: 		# Reference test crashed
sl@0: 		unless ($aTestResultsObject->DidItCrash())
sl@0: 			{
sl@0: 			my $scriptName = $aTestResultsObject->ScriptName();
sl@0: 			print "\nWARNING: $scriptName crashed in the reference build, unable to compare test counts\n";
sl@0: 			}
sl@0: 		return 1;
sl@0: 		}
sl@0: 	
sl@0: 	my $testCount = $aTestResultsObject->TotalTestCount;
sl@0: 	if ($testCount < $refTestCount)
sl@0: 		{
sl@0: 		unless ($aTestResultsObject->DidItCrash())
sl@0: 			{
sl@0: 			my $testName = $aTestResultsObject->ScriptName();
sl@0: 			$iMissingTests = $iMissingTests . "$testName  Previous = $refTestCount, Current = $testCount\n"; 	
sl@0: 			}
sl@0: 		}
sl@0: 	return 0;	
sl@0: 	}	
sl@0: 
sl@0: # Will scan the Known Failure sheet for any entries indicating if the failures,inconclusives and/or crash
sl@0: # witnessed in the script in question are already known. If so these will be appended to the known failures list
sl@0: # and removed from the TestScriptResults object in question. If matching entries were found but existed for
sl@0: # other codelines then this information is stored but the failures are not removed from the TestScriptResults 
sl@0: # object.
sl@0: # param1 - TestScriptResults object for the script with failures
sl@0: # param2 - the contents of the known failure sheet
sl@0: # param3 - the array of platform entries in the known failure sheet that are applicable to this run
sl@0: sub ResolveFailuresAgainstKFSheet
sl@0: 	{
sl@0: 	my ($aScriptWithProblems, $aKfSheetContents, $aKfPlatforms) = @_;
sl@0: 	my $kfSheetContents = $$aKfSheetContents;
sl@0: 	my $scriptName = $aScriptWithProblems->ScriptName();
sl@0: 	
sl@0: 	my @kFApplicablePlatforms;
sl@0: 	if ($aKfPlatforms)
sl@0: 		{
sl@0: 		# Will error if undef
sl@0: 		@kFApplicablePlatforms = @$aKfPlatforms;
sl@0: 		}
sl@0: 
sl@0: 	# Modified version of the KF Sheet contents, at the end of this function it will contain the Rows from the KF Sheet with any that correspond to the current test script stripped out.
sl@0: 	# This is a by product of the fact that we inspect each row one at a time looking for any that are relevant, trying to do a global search results in potentially error prone and difficult to maintain
sl@0: 	# regexs.  And in reality underneath Perl shouldn't be doing that much in the way of optimisation versus this approach.
sl@0: 	my $wipKfSheet;
sl@0: 	my $candidatesForOtherPlatforms;
sl@0: 	my $foundSamePlatFails;
sl@0: 	my $foundSamePlatInconcs;
sl@0: 	my $foundSamePlatCrash;
sl@0: 	while ($kfSheetContents =~ s/<Row((.|\n)*?>)((.|\n)*?)<\/Row>//i)
sl@0: 		{
sl@0: 		my $row = $3;
sl@0: 		# Optimisation
sl@0: 		unless ($row =~ m/$scriptName/i)
sl@0: 			{
sl@0: 			$wipKfSheet = $wipKfSheet . "<Row>$row<\/Row>";
sl@0: 			next;
sl@0: 			}
sl@0: 		
sl@0: 		# None of the cells prior to the 'Scriptname' named cell are of any interest, now verify this row is actually for 
sl@0: 		# this test script and we didn't pick the hit up off the entries for one of the other fields, e.g. Remarks
sl@0: 		my $applicable;
sl@0: 		while ($row =~ s/(<Cell(.|\n)*?<\/Cell>)//i)
sl@0: 			{
sl@0: 			my $cell = $1;
sl@0: 			if ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Scriptname\d*"\//i)
sl@0: 				{
sl@0: 				# Named field we're looking for
sl@0: 				if ($cell =~ m/$scriptName/i)
sl@0: 					{
sl@0: 					$applicable = 1;
sl@0: 					last;
sl@0: 					}
sl@0: 				else
sl@0: 					{
sl@0: 					$wipKfSheet = $wipKfSheet . "<Row>$cell\n$row<\/Row>";	# This entry isn't applicable to us, reinsert it into the KF sheet
sl@0: 					last;
sl@0: 					}
sl@0: 				}
sl@0: 			}
sl@0: 		unless ($applicable)
sl@0: 			{
sl@0: 			next;
sl@0: 			}
sl@0: 		
sl@0: 		# We are now dealing with a Row corresponding to the script in question
sl@0: 		# So pull out the relevant named cells contents
sl@0: 		my $crash;
sl@0: 		my $fails;
sl@0: 		my $inconclusives;
sl@0: 		my $targets;
sl@0: 		while ($row =~ s/(<Cell(.|\n)*?<\/Cell>)//i)
sl@0: 			{
sl@0: 			my $cell = $1;
sl@0: 			if ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="CrashedStatus\d*"\//i)
sl@0: 				{
sl@0: 				$crash = $cell;
sl@0: 				}
sl@0: 			elsif ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Fails\d*"\//i)
sl@0: 				{
sl@0: 				$fails = $cell;			
sl@0: 				}
sl@0: 			elsif ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Inconclusives\d*"\//i)
sl@0: 				{
sl@0: 				$inconclusives = $cell;			
sl@0: 				}
sl@0: 			elsif ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Platforms\d*"\//i)
sl@0: 				{
sl@0: 				$targets = $cell;			
sl@0: 				}				
sl@0: 			}
sl@0: 			
sl@0: 		my $platformMatched;
sl@0: 		foreach my $platform (@kFApplicablePlatforms)
sl@0: 			{
sl@0: 			unless (($targets =~ m/$platform&#10;/i) || ($targets =~ m/$platform<\/Data>/i))	# To ensure for example that All isn't picked up by an All Winscw entry
sl@0: 				{
sl@0: 				next;	# Platform not found in the entry
sl@0: 				}		
sl@0: 			$platformMatched = 1;
sl@0: 
sl@0: 			if (($aScriptWithProblems->DidItCrash()) && ($crash =~ m/Crashed/i))
sl@0: 				{
sl@0: 				$foundSamePlatCrash = 1;
sl@0: 				$aScriptWithProblems->ResetCrashed();
sl@0: 				}
sl@0: 				
sl@0: 			if ($fails)
sl@0: 				{
sl@0: 				my @failures = $aScriptWithProblems->Failures();
sl@0: 				$aScriptWithProblems->ResetFailures();
sl@0: 				foreach my $failure (@failures)
sl@0: 					{
sl@0: 					if (($fails =~ m/$failure/i) || ($fails =~ m/>ALL<\/Data>/))
sl@0: 						{
sl@0: 						$foundSamePlatFails = $foundSamePlatFails . "$failure\n";
sl@0: 						}
sl@0: 					else
sl@0: 						{
sl@0: 						
sl@0: 						$aScriptWithProblems->AddFailureResult($failure);
sl@0: 						}
sl@0: 					}
sl@0: 				}
sl@0: 
sl@0: 			if ($inconclusives)
sl@0: 				{
sl@0: 				my @incons = $aScriptWithProblems->Inconclusives();
sl@0: 				$aScriptWithProblems->ResetInconclusives();
sl@0: 				foreach my $incon (@incons)
sl@0: 					{
sl@0: 					if (($inconclusives =~ m/$incon/i) || ($inconclusives =~ m/>ALL<\/Data>/))
sl@0: 						{
sl@0: 						$foundSamePlatInconcs = $foundSamePlatInconcs . "$incon\n";
sl@0: 						}
sl@0: 					else
sl@0: 						{
sl@0: 						$aScriptWithProblems->AddInconclusiveResult($incon);
sl@0: 						}
sl@0: 					}
sl@0: 				}
sl@0: 			}	# End of platform matching loop
sl@0: 		
sl@0: 		unless ($platformMatched)
sl@0: 			{
sl@0: 			# The row entry did not match any of the applicable platforms.  We need to keep checking for more appropriate rows.
sl@0: 			# However, if after the KF Sheet has been parsed we still have unknown errors, we need to compare against these rows as well
sl@0: 			# in case the issue has been seen on other platforms.
sl@0: 	
sl@0: 			$candidatesForOtherPlatforms = $candidatesForOtherPlatforms . "<Row>$crash$fails$inconclusives<\/Row>";
sl@0: 			}
sl@0: 			
sl@0: 		}	# End of row scanning
sl@0: 	
sl@0: 
sl@0: 	my $foundOtherPlatFails;
sl@0: 	my $foundOtherPlatInconcs;
sl@0: 	my $foundOtherPlatCrash;	
sl@0: 	if ($aScriptWithProblems->AnyFailures())	
sl@0: 		{
sl@0: 		# Failures remain, potentially matched against the rows for other platforms
sl@0: 		
sl@0: 		while ($candidatesForOtherPlatforms =~ s/<Row>((.|\n)*?)<\/Row>//i)
sl@0: 			{
sl@0: 			my $row = $1;
sl@0: 			my $crash;
sl@0: 			my $fails;
sl@0: 			my $inconclusives;
sl@0: 			while ($row =~ s/(<Cell(.|\n)*?<\/Cell>)//i)
sl@0: 				{
sl@0: 				my $cell = $1;
sl@0: 				if ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="CrashedStatus\d*"\//i)
sl@0: 					{
sl@0: 					$crash = $cell;
sl@0: 					}
sl@0: 				elsif ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Fails\d*"\//i)
sl@0: 					{
sl@0: 					$fails = $cell;
sl@0: 					}
sl@0: 				elsif ($cell =~ m/<NamedCell\s*\n?\s*ss:Name="Inconclusives\d*"\//i)
sl@0: 					{
sl@0: 					$inconclusives = $cell;
sl@0: 					}					
sl@0: 				}	# End of stripping out content details
sl@0: 			
sl@0: 			if (($aScriptWithProblems->DidItCrash()) && ($crash =~ m/Crashed<\/Data>/i))
sl@0: 				{
sl@0: 				$foundOtherPlatCrash = 1;
sl@0: 				}
sl@0: 				
sl@0: 			if ($fails)
sl@0: 				{
sl@0: 				my @failures = $aScriptWithProblems->Failures();
sl@0: 				foreach my $failure (@failures)
sl@0: 					{
sl@0: 					if (($fails =~ m/$failure/i) || ($fails =~ m/>ALL<\/Data>/))
sl@0: 						{
sl@0: 						$foundOtherPlatFails = $foundOtherPlatFails . "$failure\n";
sl@0: 						}
sl@0: 					}
sl@0: 				}
sl@0: 
sl@0: 			if ($inconclusives)
sl@0: 				{
sl@0: 				my @incons = $aScriptWithProblems->Inconclusives();
sl@0: 				foreach my $incon (@incons)
sl@0: 					{
sl@0: 					if (($inconclusives =~ m/$incon/i) || ($inconclusives =~ m/>ALL<\/Data>/))
sl@0: 						{
sl@0: 						$foundOtherPlatInconcs = $foundOtherPlatInconcs . "$incon\n";
sl@0: 						}
sl@0: 					}
sl@0: 				}
sl@0: 				
sl@0: 			}	# End of cross-platform candidate matchine
sl@0: 		}	# End of dealing with potential cross-platform matches
sl@0: 
sl@0: 	# Output known failures
sl@0: 	if ($foundSamePlatFails || $foundSamePlatInconcs || $foundSamePlatCrash)
sl@0: 		{
sl@0: 		$iKnownFails = $iKnownFails . "\n$scriptName\n";
sl@0: 		}
sl@0: 	if ($foundSamePlatInconcs)
sl@0: 		{
sl@0: 		$iKnownFails = $iKnownFails . "INCONCLUSIVES:\n$foundSamePlatInconcs";
sl@0: 		}
sl@0: 	if ($foundSamePlatFails)
sl@0: 		{
sl@0: 		$iKnownFails = $iKnownFails . "FAILS:\n$foundSamePlatFails";
sl@0: 		}
sl@0: 	if ($foundSamePlatCrash)
sl@0: 		{
sl@0: 		$iKnownFails = $iKnownFails . "CRASHED\n";
sl@0: 		}		
sl@0: 	
sl@0: 	# Output matches found only in other test platforms
sl@0: 	if ($foundOtherPlatFails || $foundOtherPlatInconcs || $foundOtherPlatCrash)
sl@0: 		{
sl@0: 		$iUnknownFailsButKnownInOtherCodelines = $iUnknownFailsButKnownInOtherCodelines .  "\n$scriptName\n";
sl@0: 		}
sl@0: 	if ($foundOtherPlatInconcs)
sl@0: 		{
sl@0: 		$iUnknownFailsButKnownInOtherCodelines = $iUnknownFailsButKnownInOtherCodelines . "INCONCLUSIVES:\n$foundOtherPlatInconcs";
sl@0: 		}
sl@0: 	if ($foundOtherPlatFails)
sl@0: 		{
sl@0: 		$iUnknownFailsButKnownInOtherCodelines = $iUnknownFailsButKnownInOtherCodelines . "FAILS:\n$foundOtherPlatFails";
sl@0: 		}
sl@0: 	if ($foundOtherPlatCrash)
sl@0: 		{
sl@0: 		$iUnknownFailsButKnownInOtherCodelines = $iUnknownFailsButKnownInOtherCodelines . "CRASHED\n";
sl@0: 		}
sl@0: 		
sl@0: 	$aKfSheetContents = $wipKfSheet;
sl@0: 	}
sl@0: 
sl@0: 
sl@0: # Takes the corresponding log file from the reference build and test for a failing script and
sl@0: # compare it to see if the failures match.  If so this information is stored and the matching
sl@0: # failures removed from the TestScriptResults object.
sl@0: # param1 - TestScriptResults object for the script with failures
sl@0: # param2 - boolean whether the test crashed in the reference build and test
sl@0: # param3 - the contents of the corresponding log file from the reference build
sl@0: sub ResolveFailuresAgainstReferenceRun
sl@0: 	{
sl@0: 	my $aReferenceTestResults = shift;
sl@0: 	my $aRefTestCrashed = shift;
sl@0: 	my $aReferenceLogContents = shift;
sl@0: 	my $refContents = $$aReferenceLogContents;	
sl@0: 
sl@0: 	my $scriptName = $aReferenceTestResults->ScriptName();
sl@0: 	my $testHarness = $aReferenceTestResults->TestHarness();
sl@0: 	
sl@0: 	my $inconText;
sl@0: 	my @inconclusives = $aReferenceTestResults->Inconclusives();
sl@0: 	$aReferenceTestResults->ResetInconclusives();
sl@0: 	foreach my $inconclusive (@inconclusives)
sl@0: 		{
sl@0: 		if (($testHarness eq $TestScriptResults::KTEFTest) || ($testHarness eq $TestScriptResults::KTEFNoTestcasesTest))
sl@0: 			{
sl@0: 			my $testcasesInUse = 0;
sl@0: 			if ($testHarness eq $TestScriptResults::KTEFTest)
sl@0: 				{
sl@0: 				$testcasesInUse = 1;
sl@0: 				}
sl@0: 			if (MatchingTEFInconclusiveResult($inconclusive, \$refContents, $testcasesInUse))
sl@0: 				{
sl@0: 				$inconText = $inconText . "$inconclusive\n";
sl@0: 				}
sl@0: 			else
sl@0: 				{
sl@0: 				$aReferenceTestResults->AddInconclusiveResult($inconclusive);
sl@0: 				}
sl@0: 			}
sl@0: 		elsif ($testHarness eq $TestScriptResults::KTestFrameworkTest)
sl@0: 			{
sl@0: 			if (MatchingTFInconclusiveResult($inconclusive, \$refContents))
sl@0: 				{
sl@0: 				$inconText = $inconText . "$inconclusive\n";
sl@0: 				}
sl@0: 			else
sl@0: 				{
sl@0: 				$aReferenceTestResults->AddInconclusiveResult($inconclusive);			
sl@0: 				}			
sl@0: 			}
sl@0: 		# Core Conf tests have no comprehension of inconclusive
sl@0: 		}
sl@0: 			
sl@0: 	my $failText;
sl@0: 	my @failures = $aReferenceTestResults->Failures();
sl@0: 	$aReferenceTestResults->ResetFailures();
sl@0: 	foreach my $failure (@failures)
sl@0: 		{
sl@0: 		if (($testHarness eq $TestScriptResults::KTEFTest) || ($testHarness eq $TestScriptResults::KTEFNoTestcasesTest))
sl@0: 			{
sl@0: 			my $testcasesInUse = 0;
sl@0: 			if ($testHarness eq $TestScriptResults::KTEFTest)
sl@0: 				{
sl@0: 				$testcasesInUse = 1;
sl@0: 				}
sl@0: 			if (MatchingTEFErrorResult($failure, \$refContents, $testcasesInUse))
sl@0: 				{
sl@0: 				$failText = $failText . "$failure\n";
sl@0: 				}
sl@0: 			else
sl@0: 				{
sl@0: 				$aReferenceTestResults->AddFailureResult($failure);
sl@0: 				}
sl@0: 			}
sl@0: 		elsif ($testHarness eq $TestScriptResults::KTestFrameworkTest)
sl@0: 			{
sl@0: 			if (MatchingTFErrorResult($failure, \$refContents))
sl@0: 				{
sl@0: 				$failText = $failText . "$failure\n";
sl@0: 				}
sl@0: 			else
sl@0: 				{
sl@0: 				$aReferenceTestResults->AddFailureResult($failure);
sl@0: 				}			
sl@0: 			}
sl@0: 		else
sl@0: 			{
sl@0: 			# Core Conf 
sl@0: 			if (MatchingCoreConfErrorResult($failure, \$refContents))
sl@0: 				{
sl@0: 				$failText = $failText . "$failure\n";
sl@0: 				}
sl@0: 			else
sl@0: 				{
sl@0: 				$aReferenceTestResults->AddFailureResult($failure);
sl@0: 				}			
sl@0: 			}
sl@0: 		}
sl@0: 
sl@0: 	my $bothCrashed;
sl@0: 	if (($aReferenceTestResults->DidItCrash()) && $aRefTestCrashed)
sl@0: 		{
sl@0: 		$bothCrashed = 1;
sl@0: 		$aReferenceTestResults->ResetCrashed();
sl@0: 		}
sl@0: 	
sl@0: 	if ($inconText || $failText || $bothCrashed)
sl@0: 		{
sl@0: 		$iUnknownFailsButKnownInRef = $iUnknownFailsButKnownInRef . "\n$scriptName\n";
sl@0: 		}
sl@0: 	
sl@0: 	if 	($inconText)
sl@0: 		{
sl@0: 		$iUnknownFailsButKnownInRef = $iUnknownFailsButKnownInRef . "INCONCLUSIVES:\n$inconText";
sl@0: 		}
sl@0: 	if ($failText)
sl@0: 		{
sl@0: 		$iUnknownFailsButKnownInRef = $iUnknownFailsButKnownInRef . "FAILS:\n$failText";
sl@0: 		}
sl@0: 	if ($bothCrashed)
sl@0: 		{
sl@0: 		$iUnknownFailsButKnownInRef = $iUnknownFailsButKnownInRef . "CRASHED\n";
sl@0: 		}
sl@0: 	}
sl@0: 	
sl@0: ###############################################
sl@0: ##	Utility functions
sl@0: ###############################################	
sl@0: 
sl@0: sub PrintHelp
sl@0: 	{
sl@0: 	print "\n\nResultsComparison.pl -m Mode -s Results_source [-k Known_failure_sheet_path -c Codeline -p Platform] [-r Mistral_comparison_job_url] [-i] [-v]\n\n";
sl@0: 	print "  -m = Mode of operation, specify the source of the results that you are\n";
sl@0: 	print "    comparing. Currently supported modes: Mistral, Local\n\n";
sl@0: 	print "  -s = Source of the results, mode dependant:\n";
sl@0: 	print "      (i) Mistral = The URL of the test results HTML summary page\n";
sl@0: 	print "      (ii) Local = The pathname of a folder containing the test log files\n\n";
sl@0: 	print "  -k = Pathname of the KF sheet, this should be exported as an XML spreadsheet\n";
sl@0: 	print "    from Excel\n\n";
sl@0: 	print "  -r = Reference Mistral build, the URL of the test results HTML summary page.\n";
sl@0: 	print "    This can be used in addition or in place of the Known Failure sheet\n\n";
sl@0: 	print "  -i = Ignore test failures in scripts not found in the Reference Mistral build\n\n";
sl@0: 	print "  -c = An index indicating the codeline used in the test run, only used in\n";
sl@0: 	print "    conjunction with a KF sheet. Possible values: TB92, TB101, TB102\n\n";	
sl@0: 	print "  -p = An index indicating the platform the test run is from, only used in\n";
sl@0: 	print "    conjunction with a KF sheet. Possible values: Winscw, H6, H4 WDP,\n";
sl@0: 	print "    NaviEngine\n\n";
sl@0: 	print "  -v = Verbose output\n\n";
sl@0: 	}
sl@0: 	
sl@0: ###############################################
sl@0: ##	Main processing loop
sl@0: ###############################################		
sl@0: 
sl@0: # Read in parameters
sl@0: my %aParams;
sl@0: getopts('ihvm:s:k:r:c:p:', \%aParams);
sl@0: if ($aParams{h})
sl@0: 	{
sl@0: 	PrintHelp();
sl@0: 	exit;
sl@0: 	}
sl@0: 
sl@0: $iVerbose = $aParams{v};
sl@0: 	
sl@0: # The mode affects where we get the source files from and nothing else.
sl@0: my @runResults;
sl@0: if ($aParams{m} =~ m/\AMistral\Z/i)
sl@0: 	{
sl@0: 	@runResults = @{PopulateResultsArrayFromMistral($aParams{s})};	
sl@0: 	}
sl@0: elsif ($aParams{m} =~ m/\ALocal\Z/i)
sl@0: 	{
sl@0: 	@runResults = @{PopulateResultsArrayFromLocalMachine($aParams{s})};	
sl@0: 	}
sl@0: else
sl@0: 	{
sl@0: 	print "\nERROR: Operation mode absent or not recognised.\n";
sl@0: 	PrintHelp();
sl@0: 	exit;
sl@0: 	}
sl@0: 	
sl@0: my @kfPlatforms;
sl@0: if ($aParams{k})
sl@0: 	{
sl@0: 	$kfPlatforms[0] = "All";
sl@0: 	
sl@0: 	# KF sheet in use, codeline and platform arguments are required in order to search for matching failures
sl@0: 	my $codeline = $aParams{c};
sl@0: 	my $platform = $aParams{p};
sl@0: 	unless (($codeline eq "TB92") || ($codeline eq "TB101") || ($codeline eq "TB102"))
sl@0: 		{
sl@0: 		print "\nERROR: The codeline specified was not recognised, supported options: TB92, TB101, TB102.\n";
sl@0: 		exit;		
sl@0: 		}
sl@0: 
sl@0: 	if ($platform eq "Winscw")
sl@0: 		{
sl@0: 		$kfPlatforms[1] = "All Winscw";
sl@0: 		}
sl@0: 	elsif (($platform eq "H6") || ($platform eq "H4 WDP") || ($platform eq "NaviEngine"))
sl@0: 		{
sl@0: 		$kfPlatforms[1] = "All HW";
sl@0: 		}
sl@0: 	else
sl@0: 		{
sl@0: 		print "\nERROR: The platform specified was not recognised, supported options: Winscw, H6, H4 WDP, NaviEngine.\n";
sl@0: 		exit;		
sl@0: 		}
sl@0: 	
sl@0: 	$kfPlatforms[2] = "$codeline $platform";
sl@0: 	}
sl@0: 
sl@0: if ($iVerbose)
sl@0: 	{
sl@0: 	print "\nLog files found:\n";
sl@0: 	foreach my $file (@runResults)
sl@0: 		{
sl@0: 		my $scriptName = $file->ScriptName();
sl@0: 		print "$scriptName\n";
sl@0: 		}
sl@0: 	}
sl@0: 	
sl@0: AnalyseFailures($aParams{k}, $aParams{r}, $aParams{i}, \@runResults, \@kfPlatforms);
sl@0: 
sl@0: print "\n\nKnown failures:\n";
sl@0: print "------------------\n";
sl@0: print "$iKnownFails\n\n";
sl@0: 
sl@0: print "Unknown failures but found in the reference build:\n";
sl@0: print "----------------------------------------------------\n";
sl@0: print "$iUnknownFailsButKnownInRef\n\n";
sl@0: 
sl@0: print "Unknown failures but known in other codelines:\n";
sl@0: print "------------------------------------------------\n";
sl@0: print "$iUnknownFailsButKnownInOtherCodelines\n\n";
sl@0: 
sl@0: print "Unknown failures:\n";
sl@0: print "-------------------\n";
sl@0: print "$iUnknownFails\n\n";
sl@0: 
sl@0: print "Tests with reduced test counts:\n";
sl@0: print "---------------------------------\n";
sl@0: print "$iMissingTests\n\n";
sl@0: 
sl@0: