3 #############################################################################
5 # Copyright (C) 1999-2001 Jason Evans <jasone@freebsd.org>.
8 # Redistribution and use in source and binary forms, with or without
9 # modification, are permitted provided that the following conditions
11 # 1. Redistributions of source code must retain the above copyright
12 # notice(s), this list of conditions and the following disclaimer as
13 # the first lines of this file unmodified other than the possible
14 # addition of one or more copyright notices.
15 # 2. Redistributions in binary form must reproduce the above copyright
16 # notice(s), this list of conditions and the following disclaimer in
17 # the documentation and/or other materials provided with the
20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
21 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
24 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
27 # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
28 # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
29 # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
30 # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 #############################################################################
36 # $FreeBSD: src/lib/libc_r/test/verify,v 1.1.2.2 2001/06/22 21:44:27 jasone Exp $
37 # $DragonFly: src/lib/libc_r/test/verify,v 1.2 2003/06/17 04:26:48 dillon Exp $
39 #############################################################################
46 # Parse command-line arguments.
49 Getopt::Long::config("bundling"); # Allow -hv rather than forcing -h -v.
51 # Set option defaults for optional arguments.
61 &GetOptions("h|help" => \$opt_help,
62 "v|verbose" => \$opt_verbose,
63 "q|quiet" => \$opt_quiet,
64 "s|srcdir=s" => \$opt_srcdir,
65 "o|objdir=s" => \$opt_objdir,
66 "u|ustats" => \$opt_ustats,
67 "z|zero" => \$opt_zero
82 if ($opt_verbose && $opt_quiet)
84 print STDERR "-v and -q are incompatible\n";
91 print STDERR "No tests specified\n";
98 print STDERR "Option values: h:$opt_help, v:$opt_verbose, "
99 . "s:\"$opt_srcdir\", o:\"$opt_objdir\" "
100 . "q:$opt_quiet, u:$opt_ustats, z:$opt_zero\n";
101 printf STDERR "Tests (%d total): @ARGV\n", $#ARGV + 1;
105 # Create and print header.
109 "--------------------------------------------------------------------------\n",
110 "Test c_user c_system c_total chng\n",
111 " passed/FAILED h_user h_system h_total %% chng\n"
116 foreach $line (@TSTATS)
118 printf STDOUT "$line";
123 # Run sequence test(s).
125 $total_utime = 0.0; # Total user time.
126 $total_stime = 0.0; # Total system time.
127 $total_hutime = 0.0; # Total historical user time.
128 $total_hstime = 0.0; # Total historical system time.
129 $total_ntime = 0.0; # Total time for tests that have historical data.
131 foreach $test (@ARGV)
133 # Strip out any whitespace in $test.
134 $test =~ s/^\s*(.*)\s*$/$1/;
138 if (-e "$opt_srcdir/$test.exp")
142 ($okay, $utime, $stime) = &run_test($test);
144 if (-e "$opt_objdir/$test.out")
146 `diff $opt_srcdir/$test.exp $opt_objdir/$test.out > $opt_objdir/$test.diff 2>&1`;
149 # diff returns non-zero if there is a difference.
159 "Nonexistent output file \"$opt_objdir/$test.out\"\n";
163 ($hutime, $hstime) = &print_stats($test, $okay, 0, 0, $utime, $stime);
169 ($okay, $utime, $stime) = &run_test($test);
171 if (open (STEST_OUT, "<$opt_objdir/$test.out"))
174 $num_failed_subtests = 0;
176 while (defined($line = <STEST_OUT>))
178 if ($line =~ /1\.\.(\d+)/)
184 if ($num_subtests == 0)
189 print STDERR "Malformed or missing 1..n line\n";
194 for ($subtest = 1; $subtest <= $num_subtests; $subtest++)
196 while (defined($line = <STEST_OUT>))
198 if ($line =~ /^not\s+ok\s+(\d+)?/)
204 elsif ($line =~ /^ok\s+(\d+)?/)
213 if (defined($test_num) && ($test_num != $subtest))
215 # There was no output printed for one or more tests.
216 for (; $subtest < $test_num; $subtest++)
218 $num_failed_subtests++;
223 $num_failed_subtests++;
228 for (; $subtest <= $num_subtests; $subtest++)
230 $num_failed_subtests++;
235 if (0 < $num_failed_subtests)
245 print STDERR "Cannot open output file \"$opt_objdir/$test.out\"\n";
250 ($hutime, $hstime) = &print_stats($test, $okay,
251 $num_failed_subtests, $num_subtests,
255 $total_hutime += $hutime;
256 $total_hstime += $hstime;
260 $total_utime += $utime;
261 $total_stime += $stime;
265 @FAILED_TESTS = (@FAILED_TESTS, $test);
268 # If there were historical data, add the run time to the total time to
269 # compare against the historical run time.
270 if (0 < ($hutime + $hstime))
272 $total_ntime += $utime + $stime;
276 # Print summary stats.
277 $tt_str = sprintf ("%d / %d passed (%5.2f%%%%)",
278 ($#ARGV + 1) - ($#FAILED_TESTS + 1),
280 (($#ARGV + 1) - ($#FAILED_TESTS + 1))
281 / ($#ARGV + 1) * 100);
283 $t_str = sprintf ("Totals %7.2f %7.2f %7.2f"
285 . " %s %7.2f %7.2f %7.2f %7.2f%%%%\n",
286 $total_utime, $total_stime, $total_utime + $total_stime,
287 ($total_ntime - ($total_hutime + $total_hstime)),
288 $tt_str . ' ' x (40 - length($tt_str)),
289 $total_hutime, $total_hstime, $total_hutime + $total_hstime,
290 ($total_hutime + $total_hstime == 0.0) ? 0.0 :
292 - ($total_hutime + $total_hstime))
293 / ($total_hutime + $total_hstime) * 100));
295 @TSTATS = ("--------------------------------------------------------------------------\n",
297 "--------------------------------------------------------------------------\n"
301 foreach $line (@TSTATS)
303 printf STDOUT "$line";
307 if ($#FAILED_TESTS >= 0)
309 # One or more tests failed, so return an error.
312 # End of main execution.
318 my ($tutime, $tstime);
319 my ($utime, $stime, $cutime, $cstime);
320 my (@TSTATS, @TPATH);
322 my ($srcdir, $objdir);
324 # Get the path component of $test, if any.
325 @TPATH = split(/\//, $test);
327 $srcdir = join('/', ($opt_srcdir, @TPATH));
328 $objdir = join('/', ($opt_objdir, @TPATH));
330 @TSTATS = ("--------------------------------------------------------------------------\n");
332 $t_str = sprintf ("%s%s", $test, ' ' x (40 - length($test)));
333 @TSTATS = (@TSTATS, $t_str);
334 @STATS = (@STATS, @TSTATS);
337 foreach $line (@TSTATS)
339 printf STDOUT "$line";
343 ($utime, $stime, $cutime, $cstime) = times;
344 `$opt_objdir/$test $srcdir $objdir > $opt_objdir/$test.out 2>&1`;
345 ($utime, $stime, $tutime, $tstime) = times;
347 # Subtract the before time from the after time.
359 "\"$opt_objdir/$test > $opt_objdir/$test.out 2>&1\" returned $?\n";
364 return ($okay, $tutime, $tstime);
369 my ($test, $okay, $failed_subtests, $subtests, $utime, $stime) = @_;
370 my ($hutime, $hstime);
373 my ($t_str, $pass_str);
375 $pass_str = $okay ? "passed" : "*** FAILED ***";
376 if ((0 != $subtests) && (!$okay))
378 $pass_str = $pass_str . " ($failed_subtests/$subtests failed)";
380 $pass_str = $pass_str . ' ' x (39 - length($pass_str));
384 if (!open (TEST_PERF, "<$opt_objdir/$test.perf"))
386 print STDERR "Unable to open \"$opt_objdir/$test.perf\"\n";
391 ($hutime, $hstime) = split;
394 $t_str = sprintf (" %7.2f %7.2f %7.2f %7.2f\n"
395 . " %s %7.2f %7.2f %7.2f %7.2f%%%%\n",
396 $utime, $stime, $utime + $stime,
397 ($utime + $stime) - ($hutime + $hstime),
399 $hutime, $hstime, $hutime + $hstime,
400 (($hutime + $hstime) == 0.0) ? 0.0 :
401 ((($utime + $stime) - ($hutime + $hstime))
402 / ($hutime + $hstime) * 100));
409 $t_str = sprintf (" %7.2f %7.2f %7.2f \n"
411 $utime, $stime, $utime + $stime,
417 foreach $line (@TSTATS)
419 printf STDOUT "$line";
423 if ($okay && $opt_ustats)
425 if (!open (TEST_PERF, ">$opt_objdir/$test.perf"))
429 print STDERR "Unable to update \"$opt_objdir/$test.perf\"\n";
434 print TEST_PERF "$utime $stime\n";
439 return ($hutime, $hstime);
446 $0 [<options>] <test>+
449 --------------+-------------------------------------------------------------
450 -h --help | Print usage and exit.
451 -v --verbose | Verbose (incompatible with quiet).
452 -q --quiet | Quiet (incompatible with verbose).
453 -s --srcdir | Path to source tree (default is ".").
454 -o --objdir | Path to object tree (default is ".").
455 -u --ustats | Update historical statistics (stored in "<test>.perf".
456 -z --zero | Consider non-zero exit code to be an error.
457 --------------+-------------------------------------------------------------
459 If <test>.exp exists, <test>'s output is diff'ed with <test>.exp. Any
460 difference is considered failure.
462 If <test>.exp does not exist, output to stdout of the following form is
473 Lines which do not match the patterns shown above are ignored.