[linux-block.git] / scripts / dtc / of_unittest_expect

#!/usr/bin/perl
# SPDX-License-Identifier: GPL-2.0
#
# Copyright 2020, 2022 Sony Corporation
#
# Author: Frank Rowand

# This program is meant to be an aid to reading the verbose output of
# on the console log that results from executing the Linux kernel
# devicetree unittest (drivers/of/unitest.c).

$VUFX = "220201a";

use strict 'refs';
use strict subs;

use Getopt::Long;
use Text::Wrap;

# strip off everything before final "/"
(undef, $script_name) = split(/^.*\//, $0);

# following /usr/include/sysexits.h
$EX_OK=0;
$EX_USAGE=64;


#______________________________________________________________________________
sub compare {
	my ($expect, $got) = @_;
	my $expect_next;
	my $expect_next_lit;
	my $got_next;
	my $type;

	while ($expect) {

		($expect_next, $type) = split(/<</, $expect);
		($type) = split(/>>/, $type);
		$expect =~ s/^.*?>>//;	# '?' is non-greedy, minimal match

		# literal, ignore all metacharacters when used in a regex
		$expect_next_lit = quotemeta($expect_next);

		$got_next = $got;
		$got_next =~ s/^($expect_next_lit).*/\1/;
		$got       =~ s/^$expect_next_lit//;

		if ($expect_next ne $got_next) {
			return 0;
		}

		if ($type eq "int") {
			if ($got =~ /^[+-]*[0-9]+/) {
				$got =~ s/^[+-]*[0-9]+//;
			} else {
				return 0;
			}
		} elsif ($type eq "hex") {
			if ($got =~ /^(0x)*[0-9a-f]+/) {
				$got =~ s/^(0x)*[0-9a-f]+//;
			} else {
				return 0;
			}
		} elsif ($type eq "") {
			if ($expect_next ne $got_next) {
				return 0;
			} else {
				return 1;
			}
		} else {
			$internal_err++;
			print "** ERROR: special pattern not recognized: <<$type>>, CONSOLE_LOG line: $.\n";
			return 0;
		}

	}

	# should not get here
	$internal_err++;
	print "** ERROR: $script_name internal error, at end of compare(), CONSOLE_LOG line: $.\n";

	return 0;
}


#______________________________________________________________________________
sub usage {

# ***** when editing, be careful to not put tabs in the string printed:

	print STDERR
"
usage:

  $script_name CONSOLE_LOG

     -h                print program usage
    --help             print program usage
    --hide-expect      suppress output of EXPECTed lines
    --line-num         report line number of CONSOLE_LOG
    --no-expect-stats  do not report EXPECT statistics
    --no-strip-ts      do not strip leading console timestamps
    --verbose          do not suppress EXPECT begin and end lines
    --version          print program version and exit


  Process a console log for EXPECTed test related messages to either
  highlight expected devicetree unittest related messages or suppress
  the messages.  Leading console timestamps will be stripped.

  Various unittests may trigger kernel messages from outside the
  unittest code.  The unittest annotates that it expects the message
  to occur with an 'EXPECT \\ : text' (begin) before triggering the
  message, and an 'EXPECT / : text' (end) after triggering the message.

  If an expected message does not occur, that will be reported.

  For each expected message, the 'EXPECT \\ : text' (begin) and
  'EXPECT / : text' (end), 'text' will contain the message text.

  If 'EXPECT \\' (begin) and 'EXPECT /' (end) lines do not contain
  matching 'text', that will be reported.

  If EXPECT lines are nested, 'EXPECT /' (end) lines must be in the
  reverse order of the corresponding 'EXPECT \\' (begin) lines.

  'EXPECT \\ : text' (begin) and 'EXPECT / : text' (end) lines can
  contain special patterns in 'text':

     <<int>> matches: [+-]*[0-9]+
     <<hex>> matches: (0x)*[0-9a-f]+

  'EXPECT \\' (begin) and 'EXPECT /' (end) lines are suppressed.

  A prefix is added to every line of output:

    'ok ' Line matches an enclosing EXPECT begin/end pair

    '** ' Line reports $script_name warning or error

    '-> ' Line reports start or end of the unittests

    '>> ' Line reports a unittest test FAIL

    '   ' Lines that are not otherwise prefixed

  Issues detected in CONSOLE_LOG are reported to STDOUT, not to STDERR.

  Known Issues:

    --line-num causes the CONSOLE_LOG line number to be printed in 4 columns.
       If CONSOLE_LOG contains more than 9999 lines then more columns will be
       used to report the line number for lines greater than 9999 (eg for
       lines 10000 - 99999, 5 columns will be used).
";

	return {};
}

#______________________________________________________________________________
#______________________________________________________________________________

if (!GetOptions(
	"h"               => \$help,
	"help"            => \$help,
	"hide-expect"     => \$hide_expect,
	"line-num"        => \$print_line_num,
	"no-expect-stats" => \$no_expect_stats,
	"no-strip-ts"     => \$no_strip_ts,
	"verbose"         => \$verbose,
	"version"         => \$version,
	)) {
	print STDERR "\n";
	print STDERR "ERROR processing command line options\n";
	print STDERR "\n";
	print STDERR "For help, type '$script_name --help'\n";
	print STDERR "\n";

	exit $EX_OK;
}


if ($no_strip_ts) {
	$strip_ts = 1;
	$no_strip_ts = 0;
} else {
	$strip_ts = 0;
	$no_strip_ts = 1;
}


# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if ($help){

	&usage;

	exit $EX_OK;
}


# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

if ($version) {
	print STDERR "\n$script_name  $VUFX\n\n";
	print STDERR "\n";

	exit $EX_OK;
}


# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if ($#ARGV != 0) {

	# Limit input files to exactly one.
	#
	# 'while ($line = <ARGV>) {' in the code below supports multiple file
	# names on the command line, but the EXPECT statistics are reported
	# once for all input - it is not an expected use case to generate one
	# set of statistics for multiple input files.

	print STDERR "\n";
	print STDERR "Required arguments: CONSOLE_LOG\n";
	print STDERR "\n";

	exit $EX_USAGE;
}


#______________________________________________________________________________

# Patterns to match 'EXPECT \ : ' (begin) and 'EXPECT / : ' (end)
#
# $exp_* are used as regex match patterns,
# so '\\\\' in $exp_begin matches a single '\'
# quotemeta() does not do the right thing in this case
#
# $pr_fmt is the prefix that unittest prints for every message

$pr_fmt = "### dt-test ### ";
$exp_begin = "${pr_fmt}EXPECT \\\\ : ";
$exp_end   = "${pr_fmt}EXPECT / : ";


$line_num = "";
$timestamp = "";

LINE:
while ($line = <ARGV>) {

	chomp $line;

	$prefix = "  ";  ## 2 characters


	if ($strip_ts) {

		$timestamp = $line;

		if ($timestamp =~ /^\[\s*[0-9]+\.[0-9]*\] /) {
			($timestamp, $null) = split(/]/, $line);
			$timestamp = $timestamp . "] ";

		} else {
			$timestamp = "";
		}
	}

	$line =~ s/^\[\s*[0-9]+\.[0-9]*\] //;


	# -----  find EXPECT begin

	if ($line =~ /^\s*$exp_begin/) {
		$data = $line;
		$data =~ s/^\s*$exp_begin//;
		push @begin, $data;

		if ($verbose) {
			if ($print_line_num) {
				$line_num = sprintf("%4s ", $.);
			}
			printf "%s %s%s%s\n", $prefix, $line_num,  $timestamp, $line;
		}

		next LINE;
	}


	# -----  find EXPECT end

	if ($line =~ /^\s*$exp_end/) {
		$data = $line;
		$data =~ s/^\s*$exp_end//;

		if ($verbose) {
			if ($print_line_num) {
				$line_num = sprintf("%4s ", $.);
			}
			printf "%s %s%s%s\n", $prefix, $line_num,  $timestamp, $line;
		}

		$found = 0;
		$no_begin = 0;
		if (@found_or_begin > 0) {
			$begin = pop @found_or_begin;
			if (compare($data, $begin)) {
				$found = 1;
			}
		} elsif (@begin > 0) {
			$begin = pop @begin;
		} else {
			$no_begin = 1;
		}

		if ($no_begin) {

			$expect_missing_begin++;
			print "** ERROR: EXPECT end without any EXPECT begin:\n";
			print "       end ---> $line\n";

		} elsif (! $found) {

			if ($print_line_num) {
				$line_num = sprintf("%4s ", $.);
			}

			$expect_not_found++;
			printf "** %s%s$script_name WARNING - not found ---> %s\n",
					$line_num,  $timestamp, $data;

		} elsif (! compare($data, $begin)) {

			$expect_missing_end++;
			print "** ERROR: EXPECT end does not match EXPECT begin:\n";
			print "       begin -> $begin\n";
			print "       end ---> $line\n";

		} else {

			$expect_found++;

		}

		next LINE;
	}


	# -----  not an EXPECT line

	if (($line =~ /^${pr_fmt}start of unittest - you will see error messages$/) ||
	    ($line =~ /^${pr_fmt}end of unittest - [0-9]+ passed, [0-9]+ failed$/ )   ) {
		$prefix = "->"; # 2 characters
	} elsif ($line =~ /^${pr_fmt}FAIL /) {
		$unittest_fail++;
		$prefix = ">>"; # 2 characters
	}

	$found = 0;
	foreach $begin (@begin) {
		if (compare($begin, $line)) {
			$found = 1;
			last;
		}
	}

	if ($found) {
		$begin = shift @begin;
		while (! compare($begin, $line)) {
			push @found_or_begin, $begin;
			$begin = shift @begin;
		}
		push @found_or_begin, $line;

		if ($hide_expect) {
			$suppress_line = 1;
			next LINE;
		}
		$prefix = "ok"; # 2 characters
	}


	if ($print_line_num) {
		$line_num = sprintf("%4s ", $.);
	}

	printf "%s %s%s%s\n", $prefix, $line_num,  $timestamp, $line;
}

if (! $no_expect_stats) {
	print  "\n";
	print  "** EXPECT statistics:\n";
	print  "**\n";
	printf "**   EXPECT found          : %4i\n", $expect_found;
	printf "**   EXPECT not found      : %4i\n", $expect_not_found;
	printf "**   missing EXPECT begin  : %4i\n", $expect_missing_begin;
	printf "**   missing EXPECT end    : %4i\n", $expect_missing_end;
	printf "**   unittest FAIL         : %4i\n", $unittest_fail;
	printf "**   internal error        : %4i\n", $internal_err;
}

if (@begin) {
	print "** ERROR: EXPECT begin without any EXPECT end:\n";
	print "          This list may be misleading.\n";
	foreach $begin (@begin) {
		print "       begin ---> $begin\n";
	}
}
Commit	Line	Data
8e4296c2 FR	1	#!/usr/bin/perl
	2	# SPDX-License-Identifier: GPL-2.0
	3	#
	4	# Copyright 2020, 2022 Sony Corporation
	5	#
	6	# Author: Frank Rowand
	7
	8	# This program is meant to be an aid to reading the verbose output of
	9	# on the console log that results from executing the Linux kernel
	10	# devicetree unittest (drivers/of/unitest.c).
	11
	12	$VUFX = "220201a";
	13
	14	use strict 'refs';
	15	use strict subs;
	16
	17	use Getopt::Long;
	18	use Text::Wrap;
	19
	20	# strip off everything before final "/"
	21	(undef, $script_name) = split(/^.*\//, $0);
	22
	23	# following /usr/include/sysexits.h
	24	$EX_OK=0;
	25	$EX_USAGE=64;
	26
	27
	28	#______________________________________________________________________________
	29	sub compare {
	30	my ($expect, $got) = @_;
	31	my $expect_next;
	32	my $expect_next_lit;
	33	my $got_next;
	34	my $type;
	35
	36	while ($expect) {
	37
	38	($expect_next, $type) = split(/<</, $expect);
	39	($type) = split(/>>/, $type);
	40	$expect =~ s/^.*?>>//; # '?' is non-greedy, minimal match
	41
	42	# literal, ignore all metacharacters when used in a regex
	43	$expect_next_lit = quotemeta($expect_next);
	44
	45	$got_next = $got;
	46	$got_next =~ s/^($expect_next_lit).*/\1/;
	47	$got =~ s/^$expect_next_lit//;
	48
	49	if ($expect_next ne $got_next) {
	50	return 0;
	51	}
	52
	53	if ($type eq "int") {
	54	if ($got =~ /^[+-]*[0-9]+/) {
	55	$got =~ s/^[+-]*[0-9]+//;
	56	} else {
	57	return 0;
	58	}
	59	} elsif ($type eq "hex") {
	60	if ($got =~ /^(0x)*[0-9a-f]+/) {
	61	$got =~ s/^(0x)*[0-9a-f]+//;
	62	} else {
	63	return 0;
	64	}
65	} elsif ($type eq "") {
66	if ($expect_next ne $got_next) {
67	return 0;
68	} else {
69	return 1;
70	}
71	} else {
72	$internal_err++;
73	print "** ERROR: special pattern not recognized: <<$type>>, CONSOLE_LOG line: $.\n";
74	return 0;
75	}
76
77	}
78
79	# should not get here
80	$internal_err++;
81	print "** ERROR: $script_name internal error, at end of compare(), CONSOLE_LOG line: $.\n";
82
83	return 0;
84	}
85
86
87	#______________________________________________________________________________
88	sub usage {
89
90	# ***** when editing, be careful to not put tabs in the string printed:
91
92	print STDERR
93	"
94	usage:
95
96	$script_name CONSOLE_LOG
97
98	-h print program usage
99	--help print program usage
100	--hide-expect suppress output of EXPECTed lines
101	--line-num report line number of CONSOLE_LOG
102	--no-expect-stats do not report EXPECT statistics
103	--no-strip-ts do not strip leading console timestamps
104	--verbose do not suppress EXPECT begin and end lines
105	--version print program version and exit
106
107
108	Process a console log for EXPECTed test related messages to either
109	highlight expected devicetree unittest related messages or suppress
110	the messages. Leading console timestamps will be stripped.
111
112	Various unittests may trigger kernel messages from outside the
113	unittest code. The unittest annotates that it expects the message
114	to occur with an 'EXPECT \\ : text' (begin) before triggering the
115	message, and an 'EXPECT / : text' (end) after triggering the message.
116
117	If an expected message does not occur, that will be reported.
118
119	For each expected message, the 'EXPECT \\ : text' (begin) and
120	'EXPECT / : text' (end), 'text' will contain the message text.
121
122	If 'EXPECT \\' (begin) and 'EXPECT /' (end) lines do not contain
123	matching 'text', that will be reported.
124
125	If EXPECT lines are nested, 'EXPECT /' (end) lines must be in the
126	reverse order of the corresponding 'EXPECT \\' (begin) lines.
127
128	'EXPECT \\ : text' (begin) and 'EXPECT / : text' (end) lines can
129	contain special patterns in 'text':
130
131	<<int>> matches: [+-]*[0-9]+
132	<<hex>> matches: (0x)*[0-9a-f]+
133
134	'EXPECT \\' (begin) and 'EXPECT /' (end) lines are suppressed.
135
136	A prefix is added to every line of output:
137
138	'ok ' Line matches an enclosing EXPECT begin/end pair
139
140	'** ' Line reports $script_name warning or error
141
142	'-> ' Line reports start or end of the unittests
143
144	'>> ' Line reports a unittest test FAIL
145
146	' ' Lines that are not otherwise prefixed
147
148	Issues detected in CONSOLE_LOG are reported to STDOUT, not to STDERR.
149
150	Known Issues:
151
152	--line-num causes the CONSOLE_LOG line number to be printed in 4 columns.
153	If CONSOLE_LOG contains more than 9999 lines then more columns will be
154	used to report the line number for lines greater than 9999 (eg for
155	lines 10000 - 99999, 5 columns will be used).
156	";
157
158	return {};
159	}
160
161	#______________________________________________________________________________
162	#______________________________________________________________________________
163
164	if (!GetOptions(
165	"h" => \$help,
166	"help" => \$help,
167	"hide-expect" => \$hide_expect,
168	"line-num" => \$print_line_num,
169	"no-expect-stats" => \$no_expect_stats,
170	"no-strip-ts" => \$no_strip_ts,
171	"verbose" => \$verbose,
172	"version" => \$version,
173	)) {
174	print STDERR "\n";
175	print STDERR "ERROR processing command line options\n";
176	print STDERR "\n";
177	print STDERR "For help, type '$script_name --help'\n";
178	print STDERR "\n";
179
180	exit $EX_OK;
181	}
182
183
184	if ($no_strip_ts) {
185	$strip_ts = 1;
186	$no_strip_ts = 0;
187	} else {
188	$strip_ts = 0;
189	$no_strip_ts = 1;
190	}
191
192
193	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
194	if ($help){
195
196	&usage;
197
198	exit $EX_OK;
199	}
200
201
202	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
203
204	if ($version) {
205	print STDERR "\n$script_name $VUFX\n\n";
206	print STDERR "\n";
207
208	exit $EX_OK;
209	}
210
211
212	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
213	if ($#ARGV != 0) {
214
215	# Limit input files to exactly one.
216	#
217	# 'while ($line = <ARGV>) {' in the code below supports multiple file
218	# names on the command line, but the EXPECT statistics are reported
219	# once for all input - it is not an expected use case to generate one
220	# set of statistics for multiple input files.
221
222	print STDERR "\n";
223	print STDERR "Required arguments: CONSOLE_LOG\n";
224	print STDERR "\n";
225
226	exit $EX_USAGE;
227	}
228
229
230	#______________________________________________________________________________
231
232	# Patterns to match 'EXPECT \ : ' (begin) and 'EXPECT / : ' (end)
233	#
234	# $exp_* are used as regex match patterns,
235	# so '\\\\' in $exp_begin matches a single '\'
236	# quotemeta() does not do the right thing in this case
237	#
238	# $pr_fmt is the prefix that unittest prints for every message
239
240	$pr_fmt = "### dt-test ### ";
241	$exp_begin = "${pr_fmt}EXPECT \\\\ : ";
242	$exp_end = "${pr_fmt}EXPECT / : ";
243
244
245	$line_num = "";
246	$timestamp = "";
247
248	LINE:
249	while ($line = <ARGV>) {
250
251	chomp $line;
252
253	$prefix = " "; ## 2 characters
254
255
256	if ($strip_ts) {
257
258	$timestamp = $line;
259
260	if ($timestamp =~ /^\[\s[0-9]+\.[0-9]\] /) {
261	($timestamp, $null) = split(/]/, $line);
262	$timestamp = $timestamp . "] ";
263
264	} else {
265	$timestamp = "";
266	}
267	}
268
269	$line =~ s/^\[\s[0-9]+\.[0-9]\] //;
270
271
272	# ----- find EXPECT begin
273
274	if ($line =~ /^\s*$exp_begin/) {
275	$data = $line;
276	$data =~ s/^\s*$exp_begin//;
277	push @begin, $data;
278
279	if ($verbose) {
280	if ($print_line_num) {
281	$line_num = sprintf("%4s ", $.);
282	}
283	printf "%s %s%s%s\n", $prefix, $line_num, $timestamp, $line;
284	}
285
286	next LINE;
287	}
288
289
290	# ----- find EXPECT end
291
292	if ($line =~ /^\s*$exp_end/) {
293	$data = $line;
294	$data =~ s/^\s*$exp_end//;
295
296	if ($verbose) {
297	if ($print_line_num) {
298	$line_num = sprintf("%4s ", $.);
299	}
300	printf "%s %s%s%s\n", $prefix, $line_num, $timestamp, $line;
301	}
302
303	$found = 0;
304	$no_begin = 0;
305	if (@found_or_begin > 0) {
306	$begin = pop @found_or_begin;
307	if (compare($data, $begin)) {
308	$found = 1;
309	}
310	} elsif (@begin > 0) {
311	$begin = pop @begin;
312	} else {
313	$no_begin = 1;
314	}
315
316	if ($no_begin) {
317
318	$expect_missing_begin++;
319	print "** ERROR: EXPECT end without any EXPECT begin:\n";
320	print " end ---> $line\n";
321
322	} elsif (! $found) {
323
324	if ($print_line_num) {
325	$line_num = sprintf("%4s ", $.);
326	}
327
328	$expect_not_found++;
329	printf "** %s%s$script_name WARNING - not found ---> %s\n",
330	$line_num, $timestamp, $data;
331
332	} elsif (! compare($data, $begin)) {
333
334	$expect_missing_end++;
335	print "** ERROR: EXPECT end does not match EXPECT begin:\n";
336	print " begin -> $begin\n";
337	print " end ---> $line\n";
338
339	} else {
340
341	$expect_found++;
342
343	}
344
345	next LINE;
346	}
347
348
349	# ----- not an EXPECT line
350
351	if (($line =~ /^${pr_fmt}start of unittest - you will see error messages$/) \|\|
352	($line =~ /^${pr_fmt}end of unittest - [0-9]+ passed, [0-9]+ failed$/ ) ) {
353	$prefix = "->"; # 2 characters
354	} elsif ($line =~ /^${pr_fmt}FAIL /) {
355	$unittest_fail++;
356	$prefix = ">>"; # 2 characters
357	}
358
359	$found = 0;
360	foreach $begin (@begin) {
361	if (compare($begin, $line)) {
362	$found = 1;
363	last;
364	}
365	}
366
367	if ($found) {
368	$begin = shift @begin;
369	while (! compare($begin, $line)) {
370	push @found_or_begin, $begin;
371	$begin = shift @begin;
372	}
373	push @found_or_begin, $line;
374
375	if ($hide_expect) {
376	$suppress_line = 1;
377	next LINE;
378	}
379	$prefix = "ok"; # 2 characters
380	}
381
382
383	if ($print_line_num) {
384	$line_num = sprintf("%4s ", $.);
385	}
386
387	printf "%s %s%s%s\n", $prefix, $line_num, $timestamp, $line;
388	}
389
390	if (! $no_expect_stats) {
391	print "\n";
392	print "** EXPECT statistics:\n";
393	print "**\n";
394	printf "** EXPECT found : %4i\n", $expect_found;
395	printf "** EXPECT not found : %4i\n", $expect_not_found;
396	printf "** missing EXPECT begin : %4i\n", $expect_missing_begin;
397	printf "** missing EXPECT end : %4i\n", $expect_missing_end;
398	printf "** unittest FAIL : %4i\n", $unittest_fail;
399	printf "** internal error : %4i\n", $internal_err;
400	}
401
402	if (@begin) {
403	print "** ERROR: EXPECT begin without any EXPECT end:\n";
404	print " This list may be misleading.\n";
405	foreach $begin (@begin) {
406	print " begin ---> $begin\n";
407	}
408	}