First public contribution.
2 # Copyright (c) 2008-2010 Nokia Corporation and/or its subsidiary(-ies).
4 # This component and the accompanying materials are made available
5 # under the terms of the License "Eclipse Public License v1.0"
6 # which accompanies this distribution, and is available
7 # at the URL "http://www.eclipse.org/legal/epl-v10.html".
9 # Initial Contributors:
10 # Nokia Corporation - initial contribution.
15 # This script runs security tests for chosen components on a local machine without DABS scripts and ONB. Follow the steps below
16 # in order to launch tests for a security component:
17 # 1. Choose a local drive to be used, e.g. p:
18 # 2. Download a CBR onto the chosen drive.
19 # 3. Download the security source tree onto the chosen drive, and ensure that your environment variable
20 # SECURITYSOURCEDIR points to the relevant root directory. It helps to use a drive independant value here
21 # as this will allow concurrent builds and test runs on seperate drives (assuming the source code has the same path in each case).
22 # 4. Download ExecTimeOut.exe, which is stored at //PR/share/DABSRelease/buildscripts/, onto the relevant location
23 # (by default this is the chosen drive's root).
24 # 5. Build the security code. Note that builds require the SECURITYSOURCEDIR environment variable to be set.
25 # 6. Run or modify the relevant wrapper batch file for your component, which will be stored in the root of the test directory.
27 # This script works fine with the following components: Asnpkcs, Caf2, Certman, Common, Crypto, Cryptospi, CryptoTokens,
28 # FileTokens, Sistools, Streamingcaf, Swi, Switools, Ups. Other components must be tested and therefore
29 # further extensions may be needed.
35 ########################### Start of Input Reading Block ###########################
37 # Check environment variable SECURITYSOURCEDIR is correctly set
38 if ($ENV{'SECURITYSOURCEDIR'} eq "")
40 die "ERROR: The SECURITYSOURCEDIR environment variable has not been set. Set this to point to the root of the security source tree on your drive and rebuild the components you want to test. If you want to perform concurrent builds/tests use a drive relative value and use the same directory structure on each drive.";
44 my $current_dir = getcwd();
45 $current_dir =~ /^(\D):/; # $1 = driveletter
47 my $current_drive = lc $1;
49 $ENV{'SECURITYSOURCEDIR'} =~ /^(\D):/; # $1 = driveletter if it exists
50 my $sourcedir_drive = lc $1;
52 if (($sourcedir_drive ne "") && ($sourcedir_drive ne $current_drive))
54 print "WARNING: Your SECURITYSOURCEDIR environment variable is set to an absolute path and points to a different drive than the current one\n" . "SECURITYSOURCEDIR= " . $ENV{'SECURITYSOURCEDIR'} . "\nAre you sure you want to continue (Y/N)?\n";
55 my $user_decision = <STDIN>;
56 $user_decision = lc $user_decision;
57 $user_decision =~ /^(.*)\n/;
58 if (($1 ne "y") && ($1 ne "yes"))
60 die "User decided not to continue, quiting...";
65 # Read in command line arguments
66 my $testedComponentsRaw;
68 my $testedReleasesRaw;
72 my $skipCleanAfterFinalComponent;
73 my $suppressBrowserPopup;
76 GetOptions("components=s" => \$testedComponentsRaw,
77 "testReleases=s" => \$testedReleasesRaw,
78 "resultsoutput=s" => \$resultsOutputDir,
79 "exectimeout=s" => \$ExecTimeOut,
80 "suppressbrowser" => \$suppressBrowserPopup,
81 "skipfinalclean" => \$skipCleanAfterFinalComponent,
82 "help" => \$displayHelp);
89 # Ensure necessary arguments are present
92 while ($testedComponentsRaw =~ s/([^,]*),//)
94 push(@testedComponents, $1);
96 if ($testedComponentsRaw =~ /\S/)
98 push(@testedComponents, $testedComponentsRaw);
102 print "ERROR: Components to be tested not listed (e.g. -components caf2,streamingcaf).\n";
106 while ($testedReleasesRaw =~ s/([^,]*),//)
108 push(@testedReleases, $1);
110 if ($testedReleasesRaw =~ /\S/)
112 push(@testedReleases, $testedReleasesRaw);
116 print "ERROR: Releases to be tested not listed (e.g. -testReleases udeb,urel).\n";
121 if (!defined $ExecTimeOut)
123 print "ERROR: Location of ExecTimeOut.exe not specified (e.g. -exectimeout \\ExecTimeOut.exe).\n";
126 if (!defined $resultsOutputDir)
128 print "ERROR: Directory to place test results not specified (e.g. -resultsoutput \\logs\\winscw).\n";
134 die "Input parameters incorrect, exiting.";
139 my $buildTarget = 'winscw'; # At the moment the script only works for winscw
140 my $testSpec = "$ENV{SECURITYSOURCEDIR}\\testframework\\test\\autotesting\\test_spec.txt";
141 my $testResultsPath = "$resultsOutputDir\\TestResults_Summary.html";
142 ############################ End of Input Reading Block ############################
145 use Text::ParseWords;
146 use File::Path 'rmtree';
151 # helper functions taken from the TestUtilities.pm perl module
153 sub FullPathToCDrive;
154 sub FullPathToZDrive;
155 sub DigestSecurityTestLog;
156 sub DigestTestExecuteTestLog;
158 # create/empty directory for epoc window output files
159 ExecCmd("rmdir /s /q $resultsOutputDir");
160 ExecCmd("mkdir $resultsOutputDir");
162 # set the location of epoc window output
163 my $tempDir=$ENV{'TMP'};
164 my $epocWndPath = $tempDir.'\epocwind.out';
166 ExecCmd('mkdir \epoc32\drive_d');
167 ExecCmd('mkdir \epoc32\drive_e');
168 ExecCmd('mkdir \epoc32\drive_r');
170 if (!grep(/PlatSecDisabledCaps SwEvent/, ReadFile('\epoc32\data\epoc.ini')))
172 # set up drives for emulator
173 ExecCmd('echo _epoc_drive_d \epoc32\drive_d >> \epoc32\data\epoc.ini');
174 ExecCmd('echo _epoc_drive_e \epoc32\drive_e >> \epoc32\data\epoc.ini');
175 ExecCmd('echo _epoc_drive_r \epoc32\drive_r >> \epoc32\data\epoc.ini');
177 # disable the SwEvent to test SWI's handling of loader grantable capabilities
178 ExecCmd('echo PlatSecDisabledCaps SwEvent >> \epoc32\data\epoc.ini');
180 # turn off panic dialogs requiring user interaction
181 ExecCmd('echo JustInTime none >> \epoc32\data\epoc.ini');
185 ExecCmd('del /Q /F /S \epoc32\release\wins\*watcher.dll');
186 ExecCmd('del /Q /F /S \epoc32\release\winscw\*watcher.dll');
188 # replace custom Java Installer with test version
189 my $cmiPath = "\\epoc32\\release\\$buildTarget\\$testedRelease";
190 if ( -f "$cmiPath\\custommidletinstall.dll" && -f "$cmiPath\\tcustommidletinstall.dll" )
192 ExecCmd("ren $path\\custommidletinstall.dll custommidletinstall.std");
193 ExecCmd("copy $path\\tcustommidletinstall.dll $path\\custommidletinstall.dll");
196 rmtree("$ENV{'EPOCROOT'}backup\\wins\\c");
197 rmtree("$ENV{'EPOCROOT'}backup\\winscw\\c");
198 rmtree("$ENV{'EPOCROOT'}backup\\drive_d");
199 rmtree("$ENV{'EPOCROOT'}backup\\drive_e");
201 # create backups of the current drive states
202 ExecCmd("xcopy /y/e/i/q $ENV{'EPOCROOT'}epoc32\\wins\\c $ENV{'EPOCROOT'}backup\\wins\\c");
203 ExecCmd("xcopy /y/e/i/q $ENV{'EPOCROOT'}epoc32\\winscw\\c $ENV{'EPOCROOT'}backup\\winscw\\c");
204 ExecCmd("xcopy /y/e/i/q $ENV{'EPOCROOT'}epoc32\\drive_d $ENV{'EPOCROOT'}backup\\drive_d");
205 ExecCmd("xcopy /y/e/i/q $ENV{'EPOCROOT'}epoc32\\drive_e $ENV{'EPOCROOT'}backup\\drive_e");
207 # clean the results folder
208 rmtree("$resultsOutputDir");
209 ExecCmd("mkdir $resultsOutputDir");
211 # open test result file
212 if (!open(TESTRESULTS, ">$testResultsPath"))
214 die "Can't open output file $testResultsPath";
218 print TESTRESULTS "<html>\n<head>\n<title>Title of page</title>\n</head>";
219 print TESTRESULTS "\n<body>\n<h2 align=\"center\">Test Results</h2>\n";
220 print TESTRESULTS "<p><b>target:</b> $buildTarget<br>\n";
221 print TESTRESULTS "<b>security sources:</b> $ENV{SECURITYSOURCEDIR}<br>\n";
222 print TESTRESULTS "<b>test spec:</b> $testSpec<br>\n";
223 print TESTRESULTS "<b>tested components:</b> @testedComponents</p>\n";
224 print TESTRESULTS "<table border=\"1\">\n<tr bgcolor=\"gray\"><th>#</th><th>Component</th><th>Release</th><th>Program</th><th>Script</th><th>Results</th><th>Log file</th><th>Epoc window</th></tr>";
226 # for the first component being tested there is no need to perform a restore on the drives, this variable is used to make this optimisation
227 my $firstTestedComponent = 1;
231 foreach my $testedRelease (@testedReleases)
234 if ($testedRelease eq 'urel')
236 $logSuffix = '1'; # so that the udeb results aren't overwritten by urel ones, matches ONB labelling
239 # open test spec file
240 if (!open(TESTSPEC, "<$testSpec"))
242 die "Can't open input file $testSpec";
245 # set current directory
246 chdir("$ENV{'EPOCROOT'}epoc32\\release\\$buildTarget\\$testedRelease");
248 # walk through the test spec file line-by-line and execute matching tests
249 my $curComponent = '';
251 my $testingCurrentComponent = 0;
257 if ($line =~ /^\s*#/)
262 # set current component
263 if ($line =~ /^\s*\[\s*(\w+)\s*\]/) #find [component name] line
266 print "\n[$curComponent]\n";
268 # determine if we are testing this component.
269 if (!grep(/^$curComponent$/i,@testedComponents))
271 $testingCurrentComponent = 0;
275 $testingCurrentComponent = 1;
276 # clean the system and mmc drives if necessary
277 if ($firstTestedComponent)
279 $firstTestedComponent = 0;
283 ExecCmd("$ENV{SECURITYSOURCEDIR}\\testframework\\test\\autotesting\\restoredrivesstates.bat"); # For some reason when these commands are executed directly from Perl the call to xcopy to backup winscw\c causes xcopy to halt after copying one of the kanji SIS files, as a call to a batch file the commands work correctly.
285 # setup ethernet support on the emulator
286 ExecCmd("$ENV{SECURITYSOURCEDIR}\\testframework\\test\\autotesting\\useautocfg.bat");
290 # execute the test if we are testing this component
291 if ($testingCurrentComponent)
300 my $postCommand = '';
303 # parse line from the test spec file
304 ($program, $log, $commdb, $script, $timeout, $release, $preCommand, $postCommand) = quotewords(',', 1, $line);
308 $program =~ s/^\s*//;
309 $program =~ s/\s*$//;
312 $preCommand =~ s/^\s*//;
313 $preCommand =~ s/\s*$//;
314 $preCommand =~ s/%SECURITYSOURCEDIR%/$ENV{SECURITYSOURCEDIR}/;
315 $postCommand =~ s/^\s*//;
316 $postCommand =~ s/\s*$//;
317 $postCommand =~ s/%SECURITYSOURCEDIR%/$ENV{SECURITYSOURCEDIR}/;
323 if($release =! /$testedRelease/i)
329 # execute the pre-test command
330 if($preCommand ne "")
332 ExecCmd($preCommand);
335 # execute the test command
336 ExecCmd("$ExecTimeOut \"$program $script\" $timeout");
337 print "\n\nExecution of $program $script";
340 # store test results and epoc window output files
344 $trimmedLog =~ s/\\.*\\//; # perl does greedy pattern matching so strips off all directories in the path of the form \....\....\
345 $trimmedLog =~ s/.*\\//; # in case the path is relative strip of the leading directory
346 $trimmedLog =~ /(.*)\.([^\.]*)/; # extract the filename and the extension
348 $epocwindName = $1 . $logSuffix . "_epocwind.txt";
349 $targetLogName = $1 . $logSuffix. "\." . $2;
351 ExecCmd("copy $epocWndPath $resultsOutputDir\\$epocwindName");
352 ExecCmd("del $epocWndPath");
360 my $iPass_case = 0; # not used
361 my $iFail_case = 0; # not used
363 my $logPath = FullWin32Path($buildTarget, $release, $log);
367 ExecCmd("copy $logPath $resultsOutputDir\\$targetLogName");
371 ($sText, $iPass, $iFail, $iPass_case, $iFail_case) = DigestTestExecuteTestLog($logPath);
377 # Check to see if this is an Security RTest test log
378 ($sText, $iPass, $iFail) = DigestSecurityTestLog($logPath);
381 # get rid of surrounding html tags
384 if ($sText =~ /<td.*>(.+)<\/td>/)
391 # generate HTML results
392 my $htmlEpocWnd = "<a href=\"$resultsOutputDir\\$epocwindName\">here</a>";
395 $htmlEpocWnd = "not found";
397 my $htmlColor = "white";
398 my $htmlLog = "<a href=\"$resultsOutputDir\\$targetLogName\">$log</a>";
405 $sText = 'log file not parsed';
407 elsif(!$iPass or $iFail)
414 $htmlColor = 'yellow';
415 $sText = 'log file not found';
419 print TESTRESULTS "\n<tr bgcolor=\"$htmlColor\">";
420 print TESTRESULTS "<td>$i</td><td>$curComponent</td><td>$testedRelease</td><td>$program</td><td>$script</td>";
421 print TESTRESULTS "<td>$sText</td><td>$htmlLog</td><td>$htmlEpocWnd</td></tr>";
423 print "\nResults: $sText";
425 # execute the post-test command
426 if($postCommand ne "")
428 ExecCmd($postCommand);
435 # restore the drives' states
436 if (!$skipCleanAfterFinalComponent)
438 ExecCmd("$ENV{SECURITYSOURCEDIR}\\testframework\\test\\autotesting\\restoredrivesstates.bat");
442 print TESTRESULTS "\n</table></body></html>";
446 if (!$suppressBrowserPopup)
448 # open test results in browser, this will halt script execution until the browser is closed
452 ######################################################################################
454 ######################################################################################
462 print "\n\n\nHELP DIALOG:";
463 print "\nThe following arguments are required:\n\n";
464 print "-components: A comma separated (no spaces) list of component names.\n\n";
465 print "-testreleases: A comma separated (no spaces) list of the releases tested.\n\n";
466 print "-exectimeout: The path location of ExecTimeOut.exe relative to this drive. This is downloaded via Perforce from //PR/share/DABSRelease/buildscripts/ExecTimeOut.exe.\n\n";
467 print "-resultsoutput: Directory used to store the test results and epocwind.out files from running the tests. Current contents will be deleted. Use a value of \\logs\\winscw if you want it to be compatible with the panicscan test.\n\n";
468 print "-skipfinalclean: The script will restore the drives to their original state after each component has been tested. If you do not want the drives to be restored after the final component has been tested (typically used when debugging just one component's tests) then add this flag and before re-running the tests manually restore the drives from the contents of the \\backup\\ folder.\n\n";
469 print "-suppressbrowser: Add this flag to prevent the halting of execution and presentation of the results in a browser window after the tests have been run.";
470 print "Example usage:\n";
471 print "test_launcher.pl -components caf2,streamingcaf,swi -testreleases udeb,urel -exectimeout \\ExecTimeOut.exe resultsoutput \\logs\\winscw -skipfinalclean -suppressbrowser\n\n";
475 ######################################################################################
476 # helper functions taken from the TestUtilities.pm perl module
477 ######################################################################################
479 sub DigestTestExecuteTestLog
480 # Arg: [0] output log from a TestExecute run
481 # Returns: small HTML summary string of the run
483 my $filename = $_[0];
484 my $TestConfirmed = $_[1];
487 $filename =~ s/\s+$//;
488 return "" unless( -f $filename and open( TEST, $filename ));
495 my $bTestExecute = 0;
496 my $TEST_STEP_SECTION = 0;
500 my $TEST_CASE_SECTION = 0;
507 if ( $sLine =~ /TEST STEP SUMMARY:<\/font>/i)
509 $TEST_STEP_SECTION = 1;
510 $TEST_CASE_SECTION = 0;
512 elsif ( $sLine =~ /RUN PROGRAM SUMMARY:<\/font>/i)
514 $TEST_STEP_SECTION = 0;
516 elsif( $sLine =~ /TEST CASE SUMMARY:<\/font>/i)
518 $TEST_CASE_SECTION = 1;
519 $TEST_STEP_SECTION = 0;
521 elsif ( $sLine =~ /SUMMARY:<\/font>/i ) # To Keep Last
523 $TEST_STEP_SECTION = 1;
525 if ( $TEST_STEP_SECTION )
527 $iPass += $1 if( $sLine =~ /<font.*color=00AF00> PASS = (\d+)<\/font>/i);
528 $iPass += $1 if( $sLine =~ /<font.*color=00AF00>PASS = (\d+)<\/font>/i);
530 $iFail += $1 if( $sLine =~ /<font.*color=FF0000>FAIL = (\d+)<\/font>/i);
531 $iFail += $1 if( $sLine =~ /<font.*color=0000FF>ABORT = (\d+)<\/font>/i);
532 $iFail += $1 if( $sLine =~ /<font.*color=0000FF>PANIC = (\d+)<\/font>/i);
533 $iFail += $1 if( $sLine =~ /<font.*color=0000FF>INCONCLUSIVE = (\d+)<\/font>/i);
534 $iFail += $1 if( $sLine =~ /<font.*color=0000FF>UNKNOWN = (\d+)<\/font>/i);
535 $iFail += $1 if( $sLine =~ /<font.*color=0000FF>UNEXECUTED = (\d+)<\/font>/i);
537 if ( $TEST_CASE_SECTION )
540 $iPass_case += $1 if( $sLine =~ /<font.*color=00AF00> PASS = (\d+)<\/font>/i);
541 $iPass_case += $1 if( $sLine =~ /<font.*color=00AF00>PASS = (\d+)<\/font>/i);
542 $iFail_case += $1 if( $sLine =~ /<font.*color=FF0000>FAIL = (\d+)<\/font>/i);
543 $iFail_case += $1 if( $sLine =~ /<font.*color=0000FF>ABORT = (\d+)<\/font>/i);
544 $iFail_case += $1 if( $sLine =~ /<font.*color=0000FF>PANIC = (\d+)<\/font>/i);
545 $iFail_case += $1 if( $sLine =~ /<font.*color=0000FF>INCONCLUSIVE = (\d+)<\/font>/i);
546 $iFail_case += $1 if( $sLine =~ /<font.*color=0000FF>UNKNOWN = (\d+)<\/font>/i);
547 $iFail_case += $1 if( $sLine =~ /<font.*color=0000FF>UNEXECUTED = (\d+)<\/font>/i);
550 $bTestExecute = 1 if( $sLine =~ /\*\*\*\s+TestExecute\s+Started/i);
557 if(( $iPass == 0 and $iFail == 0) and ($TestConfirmed == 1))
559 $sHTML = "<td>$iPass passed, $iFail failed</td>";
561 elsif ( $iPass == 0 and $iFail == 0)
563 $sHTML = "<td $sFailedBGColor>test crashed</td>";
567 $sHTML = "<td $sFailedBGColor>$iPass passed, $iFail failed</td>";
571 $sHTML = "<td>$iPass passed, $iFail failed</td>";
574 elsif( $iPass > 0 or $iFail > 0 )
577 # Almost certainly a TEF log, but missing the "TestExecute Started" line.
578 # Possibly because of a loss of part of the log from the hardware.
582 $sHTML = "<td $sFailedBGColor>$iPass passed, $iFail failed</td>";
586 $sHTML = "<td $sWarningBCColor>$iPass passed, $iFail failed</td>";
590 return ($sHTML, $iPass, $iFail, $iPass_case, $iFail_case);
594 sub DigestSecurityTestLog
595 # Arg: [0] output log from an automated testcase run
596 # Returns: small HTML summary string of the run indicating test result extracted from log.
598 my $filename = $_[0];
602 my $sCmd = "type $filename |";
603 $filename =~ s/\s+$//;
604 return "" unless( -f $filename and open( TEST, $sCmd ));
608 # For these tests the result x tests failed out of y is displayed in the 2nd last line of the log file.
619 $sLine =~ s/$null|\s//g;
620 if( $sLine =~ /(\d+)testsfailedoutof(\d+)/i )
631 $sHTML = "<td $sFailedBGColor>" if( $iFail > 0 );
633 $iPass = $iTotal - $iFail;
634 $sHTML .= "$iPass passed, $iFail failed</td>";
637 return ($sHTML, $iPass, $iFail);
640 ######################################################################################
641 # helper functions taken from the BuildSpecifications.pm perl module
642 ######################################################################################
644 # Read the contents of a file into a string and return it
648 open FILE, "<$file" or LogDie("Can't read file: $file", @StdLogs);
656 ###########################################################################
657 # Return Path to z Drive
658 # Arguments: [0] Build Target (e.g. arm4, wins, etc)
659 # [1] Build Release (Optional) (e.g. UDEB/UREL)
660 # Returns Correspoonding Effective Z Drive
663 my ($buildTarget, $buildRelease) = @_;
664 $buildRelease="UDEB" unless( $buildRelease );
666 return "$ENV{'EPOCROOT'}epoc32\\release\\$buildTarget\\$buildRelease\\z" if( $buildTarget =~ /wins/i );
667 return "$ENV{'EPOCROOT'}epoc32\\data\\z";
671 ###########################################################################
672 # Return Path to C Drive
673 # Arguments: [0] Build Target (e.g. arm4, wins, etc)
674 # Returns Correspoonding Effective C Drive
677 my ($buildTarget) = @_;
679 return "$ENV{'EPOCROOT'}epoc32\\$buildTarget\\c" if( $buildTarget =~ /wins/i );
680 return "$ENV{'EPOCROOT'}epoc32\\data\\z";
684 ###########################################################################
685 # EPOC paths can be drive-relative ("\scripts\bob") or absolute ("c:\scripts\bob").
686 # Return an appropriate Win32 path, where - an additional complication
687 # is that the EPOC c: and z: aren't sibling Win32 paths
688 # Arguments: [0] Build Target (e.g. arm4, wins, etc)
689 # [1] Build Release (Optional) (e.g. UDEB/UREL)
691 # Returns: full Win32 path
694 my ($buildTarget, $buildRelease, $sPath) = @_;
697 if($sPath =~ /^(\w):(.*)/)
704 return FullPathToCDrive($buildTarget) . "\\$sDir";
706 elsif($sDrive eq "Z")
708 return FullPathToZDrive($buildTarget,$buildRelease) . "\\$sDir";
712 # Maybe an actual pathname
717 # Presumption is that path is relative to EPOC C:\
719 return FullPathToCDrive($buildTarget) . "\\$sPath";