Commit | Line | Data |
---|---|---|
136fc5c4 TH |
1 | #!/usr/bin/env perl |
2 | # | |
3 | # (c) 2017 Tobin C. Harding <me@tobin.cc> | |
4 | # Licensed under the terms of the GNU GPL License version 2 | |
5 | # | |
1410fe4e | 6 | # leaking_addresses.pl: Scan the kernel for potential leaking addresses. |
136fc5c4 TH |
7 | # - Scans dmesg output. |
8 | # - Walks directory tree and parses each file (for each directory in @DIRS). | |
9 | # | |
136fc5c4 TH |
10 | # Use --debug to output path before parsing, this is useful to find files that |
11 | # cause the script to choke. | |
136fc5c4 | 12 | |
472c9e10 TH |
13 | # |
14 | # When the system is idle it is likely that most files under /proc/PID will be | |
15 | # identical for various processes. Scanning _all_ the PIDs under /proc is | |
16 | # unnecessary and implies that we are thoroughly scanning /proc. This is _not_ | |
17 | # the case because there may be ways userspace can trigger creation of /proc | |
18 | # files that leak addresses but were not present during a scan. For these two | |
19 | # reasons we exclude all PID directories under /proc except '1/' | |
20 | ||
136fc5c4 TH |
21 | use warnings; |
22 | use strict; | |
23 | use POSIX; | |
24 | use File::Basename; | |
25 | use File::Spec; | |
26 | use Cwd 'abs_path'; | |
27 | use Term::ANSIColor qw(:constants); | |
28 | use Getopt::Long qw(:config no_auto_abbrev); | |
62139c12 | 29 | use Config; |
87e37588 | 30 | use bigint qw/hex/; |
2f042c93 | 31 | use feature 'state'; |
136fc5c4 TH |
32 | |
33 | my $P = $0; | |
34 | my $V = '0.01'; | |
35 | ||
36 | # Directories to scan. | |
37 | my @DIRS = ('/proc', '/sys'); | |
38 | ||
dd98c252 TH |
39 | # Timer for parsing each file, in seconds. |
40 | my $TIMEOUT = 10; | |
41 | ||
1410fe4e TH |
42 | # Kernel addresses vary by architecture. We can only auto-detect the following |
43 | # architectures (using `uname -m`). (flag --32-bit overrides auto-detection.) | |
44 | my @SUPPORTED_ARCHITECTURES = ('x86_64', 'ppc64', 'x86'); | |
62139c12 | 45 | |
136fc5c4 TH |
46 | # Command line options. |
47 | my $help = 0; | |
48 | my $debug = 0; | |
d09bd8da TH |
49 | my $raw = 0; |
50 | my $output_raw = ""; # Write raw results to file. | |
51 | my $input_raw = ""; # Read raw results from file instead of scanning. | |
d09bd8da TH |
52 | my $suppress_dmesg = 0; # Don't show dmesg in output. |
53 | my $squash_by_path = 0; # Summary report grouped by absolute path. | |
54 | my $squash_by_filename = 0; # Summary report grouped by filename. | |
f9d2a42d | 55 | my $kernel_config_file = ""; # Kernel configuration file. |
1410fe4e TH |
56 | my $opt_32bit = 0; # Scan 32-bit kernel. |
57 | my $page_offset_32bit = 0; # Page offset for 32-bit kernel. | |
136fc5c4 | 58 | |
b401f56f TH |
59 | # Skip these absolute paths. |
60 | my @skip_abs = ( | |
61 | '/proc/kmsg', | |
62 | '/proc/device-tree', | |
63 | '/sys/firmware/devicetree', | |
64 | '/sys/kernel/debug/tracing/trace_pipe', | |
65 | '/sys/kernel/security/apparmor/revision'); | |
66 | ||
67 | # Skip these under any subdirectory. | |
68 | my @skip_any = ( | |
69 | 'pagemap', | |
70 | 'events', | |
71 | 'access', | |
72 | 'registers', | |
73 | 'snapshot_raw', | |
74 | 'trace_pipe_raw', | |
75 | 'ptmx', | |
76 | 'trace_pipe', | |
77 | 'fd', | |
78 | 'usbmon'); | |
136fc5c4 TH |
79 | |
80 | sub help | |
81 | { | |
82 | my ($exitcode) = @_; | |
83 | ||
84 | print << "EOM"; | |
d09bd8da | 85 | |
136fc5c4 TH |
86 | Usage: $P [OPTIONS] |
87 | Version: $V | |
88 | ||
89 | Options: | |
90 | ||
15d60a35 TH |
91 | -o, --output-raw=<file> Save results for future processing. |
92 | -i, --input-raw=<file> Read results from file instead of scanning. | |
93 | --raw Show raw results (default). | |
94 | --suppress-dmesg Do not show dmesg results. | |
95 | --squash-by-path Show one result per unique path. | |
96 | --squash-by-filename Show one result per unique filename. | |
f9d2a42d | 97 | --kernel-config-file=<file> Kernel configuration file (e.g /boot/config) |
1410fe4e TH |
98 | --32-bit Scan 32-bit kernel. |
99 | --page-offset-32-bit=o Page offset (for 32-bit kernel 0xABCD1234). | |
15d60a35 TH |
100 | -d, --debug Display debugging output. |
101 | -h, --help, --version Display this help and exit. | |
d09bd8da | 102 | |
1410fe4e | 103 | Scans the running kernel for potential leaking addresses. |
136fc5c4 TH |
104 | |
105 | EOM | |
106 | exit($exitcode); | |
107 | } | |
108 | ||
109 | GetOptions( | |
136fc5c4 TH |
110 | 'd|debug' => \$debug, |
111 | 'h|help' => \$help, | |
d09bd8da TH |
112 | 'version' => \$help, |
113 | 'o|output-raw=s' => \$output_raw, | |
114 | 'i|input-raw=s' => \$input_raw, | |
115 | 'suppress-dmesg' => \$suppress_dmesg, | |
116 | 'squash-by-path' => \$squash_by_path, | |
117 | 'squash-by-filename' => \$squash_by_filename, | |
118 | 'raw' => \$raw, | |
f9d2a42d | 119 | 'kernel-config-file=s' => \$kernel_config_file, |
1410fe4e TH |
120 | '32-bit' => \$opt_32bit, |
121 | 'page-offset-32-bit=o' => \$page_offset_32bit, | |
136fc5c4 TH |
122 | ) or help(1); |
123 | ||
124 | help(0) if ($help); | |
125 | ||
d09bd8da TH |
126 | if ($input_raw) { |
127 | format_output($input_raw); | |
128 | exit(0); | |
129 | } | |
130 | ||
131 | if (!$input_raw and ($squash_by_path or $squash_by_filename)) { | |
132 | printf "\nSummary reporting only available with --input-raw=<file>\n"; | |
133 | printf "(First run scan with --output-raw=<file>.)\n"; | |
134 | exit(128); | |
135 | } | |
136 | ||
1410fe4e | 137 | if (!(is_supported_architecture() or $opt_32bit or $page_offset_32bit)) { |
62139c12 TH |
138 | printf "\nScript does not support your architecture, sorry.\n"; |
139 | printf "\nCurrently we support: \n\n"; | |
140 | foreach(@SUPPORTED_ARCHITECTURES) { | |
141 | printf "\t%s\n", $_; | |
142 | } | |
6efb7458 | 143 | printf("\n"); |
62139c12 | 144 | |
1410fe4e TH |
145 | printf("If you are running a 32-bit architecture you may use:\n"); |
146 | printf("\n\t--32-bit or --page-offset-32-bit=<page offset>\n\n"); | |
147 | ||
6efb7458 TH |
148 | my $archname = `uname -m`; |
149 | printf("Machine hardware name (`uname -m`): %s\n", $archname); | |
62139c12 TH |
150 | |
151 | exit(129); | |
152 | } | |
153 | ||
d09bd8da TH |
154 | if ($output_raw) { |
155 | open my $fh, '>', $output_raw or die "$0: $output_raw: $!\n"; | |
156 | select $fh; | |
157 | } | |
158 | ||
136fc5c4 TH |
159 | parse_dmesg(); |
160 | walk(@DIRS); | |
161 | ||
162 | exit 0; | |
163 | ||
136fc5c4 TH |
164 | sub dprint |
165 | { | |
166 | printf(STDERR @_) if $debug; | |
167 | } | |
168 | ||
62139c12 TH |
169 | sub is_supported_architecture |
170 | { | |
1410fe4e TH |
171 | return (is_x86_64() or is_ppc64() or is_ix86_32()); |
172 | } | |
173 | ||
174 | sub is_32bit | |
175 | { | |
176 | # Allow --32-bit or --page-offset-32-bit to override | |
177 | if ($opt_32bit or $page_offset_32bit) { | |
178 | return 1; | |
179 | } | |
180 | ||
181 | return is_ix86_32(); | |
182 | } | |
183 | ||
184 | sub is_ix86_32 | |
185 | { | |
5e4bac34 | 186 | state $arch = `uname -m`; |
1410fe4e TH |
187 | |
188 | chomp $arch; | |
189 | if ($arch =~ m/i[3456]86/) { | |
190 | return 1; | |
191 | } | |
192 | return 0; | |
62139c12 TH |
193 | } |
194 | ||
5eb0da05 | 195 | sub is_arch |
62139c12 | 196 | { |
5eb0da05 TH |
197 | my ($desc) = @_; |
198 | my $arch = `uname -m`; | |
199 | ||
200 | chomp $arch; | |
201 | if ($arch eq $desc) { | |
202 | return 1; | |
203 | } | |
204 | return 0; | |
205 | } | |
62139c12 | 206 | |
5eb0da05 TH |
207 | sub is_x86_64 |
208 | { | |
5e4bac34 TH |
209 | state $is = is_arch('x86_64'); |
210 | return $is; | |
62139c12 TH |
211 | } |
212 | ||
213 | sub is_ppc64 | |
214 | { | |
5e4bac34 TH |
215 | state $is = is_arch('ppc64'); |
216 | return $is; | |
62139c12 TH |
217 | } |
218 | ||
f9d2a42d TH |
219 | # Gets config option value from kernel config file. |
220 | # Returns "" on error or if config option not found. | |
221 | sub get_kernel_config_option | |
222 | { | |
223 | my ($option) = @_; | |
224 | my $value = ""; | |
225 | my $tmp_file = ""; | |
226 | my @config_files; | |
227 | ||
228 | # Allow --kernel-config-file to override. | |
229 | if ($kernel_config_file ne "") { | |
230 | @config_files = ($kernel_config_file); | |
231 | } elsif (-R "/proc/config.gz") { | |
232 | my $tmp_file = "/tmp/tmpkconf"; | |
233 | ||
234 | if (system("gunzip < /proc/config.gz > $tmp_file")) { | |
235 | dprint "$0: system(gunzip < /proc/config.gz) failed\n"; | |
236 | return ""; | |
237 | } else { | |
238 | @config_files = ($tmp_file); | |
239 | } | |
240 | } else { | |
241 | my $file = '/boot/config-' . `uname -r`; | |
242 | chomp $file; | |
243 | @config_files = ($file, '/boot/config'); | |
244 | } | |
245 | ||
246 | foreach my $file (@config_files) { | |
247 | dprint("parsing config file: %s\n", $file); | |
248 | $value = option_from_file($option, $file); | |
249 | if ($value ne "") { | |
250 | last; | |
251 | } | |
252 | } | |
253 | ||
254 | if ($tmp_file ne "") { | |
255 | system("rm -f $tmp_file"); | |
256 | } | |
257 | ||
258 | return $value; | |
259 | } | |
260 | ||
261 | # Parses $file and returns kernel configuration option value. | |
262 | sub option_from_file | |
263 | { | |
264 | my ($option, $file) = @_; | |
265 | my $str = ""; | |
266 | my $val = ""; | |
267 | ||
268 | open(my $fh, "<", $file) or return ""; | |
269 | while (my $line = <$fh> ) { | |
270 | if ($line =~ /^$option/) { | |
271 | ($str, $val) = split /=/, $line; | |
272 | chomp $val; | |
273 | last; | |
274 | } | |
275 | } | |
276 | ||
277 | close $fh; | |
278 | return $val; | |
279 | } | |
280 | ||
136fc5c4 TH |
281 | sub is_false_positive |
282 | { | |
7e5758f7 TH |
283 | my ($match) = @_; |
284 | ||
1410fe4e TH |
285 | if (is_32bit()) { |
286 | return is_false_positive_32bit($match); | |
287 | } | |
288 | ||
289 | # 64 bit false positives. | |
290 | ||
7e5758f7 TH |
291 | if ($match =~ '\b(0x)?(f|F){16}\b' or |
292 | $match =~ '\b(0x)?0{16}\b') { | |
293 | return 1; | |
294 | } | |
136fc5c4 | 295 | |
87e37588 TH |
296 | if (is_x86_64() and is_in_vsyscall_memory_region($match)) { |
297 | return 1; | |
7e5758f7 | 298 | } |
136fc5c4 | 299 | |
7e5758f7 | 300 | return 0; |
136fc5c4 TH |
301 | } |
302 | ||
1410fe4e TH |
303 | sub is_false_positive_32bit |
304 | { | |
305 | my ($match) = @_; | |
306 | state $page_offset = get_page_offset(); | |
307 | ||
308 | if ($match =~ '\b(0x)?(f|F){8}\b') { | |
309 | return 1; | |
310 | } | |
311 | ||
312 | if (hex($match) < $page_offset) { | |
313 | return 1; | |
314 | } | |
315 | ||
316 | return 0; | |
317 | } | |
318 | ||
319 | # returns integer value | |
320 | sub get_page_offset | |
321 | { | |
322 | my $page_offset; | |
323 | my $default_offset = 0xc0000000; | |
324 | ||
325 | # Allow --page-offset-32bit to override. | |
326 | if ($page_offset_32bit != 0) { | |
327 | return $page_offset_32bit; | |
328 | } | |
329 | ||
330 | $page_offset = get_kernel_config_option('CONFIG_PAGE_OFFSET'); | |
331 | if (!$page_offset) { | |
332 | return $default_offset; | |
333 | } | |
334 | return $page_offset; | |
335 | } | |
336 | ||
87e37588 TH |
337 | sub is_in_vsyscall_memory_region |
338 | { | |
339 | my ($match) = @_; | |
340 | ||
341 | my $hex = hex($match); | |
342 | my $region_min = hex("0xffffffffff600000"); | |
343 | my $region_max = hex("0xffffffffff601000"); | |
344 | ||
345 | return ($hex >= $region_min and $hex <= $region_max); | |
346 | } | |
347 | ||
136fc5c4 TH |
348 | # True if argument potentially contains a kernel address. |
349 | sub may_leak_address | |
350 | { | |
7e5758f7 | 351 | my ($line) = @_; |
62139c12 | 352 | my $address_re; |
136fc5c4 | 353 | |
7e5758f7 TH |
354 | # Signal masks. |
355 | if ($line =~ '^SigBlk:' or | |
a11949ec | 356 | $line =~ '^SigIgn:' or |
7e5758f7 TH |
357 | $line =~ '^SigCgt:') { |
358 | return 0; | |
359 | } | |
136fc5c4 | 360 | |
7e5758f7 TH |
361 | if ($line =~ '\bKEY=[[:xdigit:]]{14} [[:xdigit:]]{16} [[:xdigit:]]{16}\b' or |
362 | $line =~ '\b[[:xdigit:]]{14} [[:xdigit:]]{16} [[:xdigit:]]{16}\b') { | |
136fc5c4 | 363 | return 0; |
7e5758f7 | 364 | } |
136fc5c4 | 365 | |
2f042c93 | 366 | $address_re = get_address_re(); |
62139c12 | 367 | while (/($address_re)/g) { |
7e5758f7 TH |
368 | if (!is_false_positive($1)) { |
369 | return 1; | |
370 | } | |
371 | } | |
136fc5c4 | 372 | |
7e5758f7 | 373 | return 0; |
136fc5c4 TH |
374 | } |
375 | ||
2f042c93 TH |
376 | sub get_address_re |
377 | { | |
1410fe4e | 378 | if (is_ppc64()) { |
2f042c93 | 379 | return '\b(0x)?[89abcdef]00[[:xdigit:]]{13}\b'; |
1410fe4e TH |
380 | } elsif (is_32bit()) { |
381 | return '\b(0x)?[[:xdigit:]]{8}\b'; | |
2f042c93 | 382 | } |
1410fe4e TH |
383 | |
384 | return get_x86_64_re(); | |
2f042c93 TH |
385 | } |
386 | ||
387 | sub get_x86_64_re | |
388 | { | |
389 | # We handle page table levels but only if explicitly configured using | |
390 | # CONFIG_PGTABLE_LEVELS. If config file parsing fails or config option | |
391 | # is not found we default to using address regular expression suitable | |
392 | # for 4 page table levels. | |
393 | state $ptl = get_kernel_config_option('CONFIG_PGTABLE_LEVELS'); | |
394 | ||
395 | if ($ptl == 5) { | |
396 | return '\b(0x)?ff[[:xdigit:]]{14}\b'; | |
397 | } | |
398 | return '\b(0x)?ffff[[:xdigit:]]{12}\b'; | |
399 | } | |
400 | ||
136fc5c4 TH |
401 | sub parse_dmesg |
402 | { | |
403 | open my $cmd, '-|', 'dmesg'; | |
404 | while (<$cmd>) { | |
405 | if (may_leak_address($_)) { | |
406 | print 'dmesg: ' . $_; | |
407 | } | |
408 | } | |
409 | close $cmd; | |
410 | } | |
411 | ||
412 | # True if we should skip this path. | |
413 | sub skip | |
414 | { | |
b401f56f | 415 | my ($path) = @_; |
136fc5c4 | 416 | |
b401f56f | 417 | foreach (@skip_abs) { |
136fc5c4 TH |
418 | return 1 if (/^$path$/); |
419 | } | |
420 | ||
421 | my($filename, $dirs, $suffix) = fileparse($path); | |
b401f56f | 422 | foreach (@skip_any) { |
136fc5c4 TH |
423 | return 1 if (/^$filename$/); |
424 | } | |
425 | ||
426 | return 0; | |
427 | } | |
428 | ||
dd98c252 TH |
429 | sub timed_parse_file |
430 | { | |
431 | my ($file) = @_; | |
432 | ||
433 | eval { | |
434 | local $SIG{ALRM} = sub { die "alarm\n" }; # NB: \n required. | |
435 | alarm $TIMEOUT; | |
436 | parse_file($file); | |
437 | alarm 0; | |
438 | }; | |
439 | ||
440 | if ($@) { | |
441 | die unless $@ eq "alarm\n"; # Propagate unexpected errors. | |
442 | printf STDERR "timed out parsing: %s\n", $file; | |
443 | } | |
444 | } | |
445 | ||
136fc5c4 TH |
446 | sub parse_file |
447 | { | |
448 | my ($file) = @_; | |
449 | ||
450 | if (! -R $file) { | |
451 | return; | |
452 | } | |
453 | ||
e2858cad TH |
454 | if (! -T $file) { |
455 | return; | |
456 | } | |
457 | ||
136fc5c4 TH |
458 | open my $fh, "<", $file or return; |
459 | while ( <$fh> ) { | |
460 | if (may_leak_address($_)) { | |
461 | print $file . ': ' . $_; | |
462 | } | |
463 | } | |
464 | close $fh; | |
465 | } | |
466 | ||
136fc5c4 TH |
467 | # Recursively walk directory tree. |
468 | sub walk | |
469 | { | |
470 | my @dirs = @_; | |
136fc5c4 TH |
471 | |
472 | while (my $pwd = shift @dirs) { | |
136fc5c4 TH |
473 | next if (!opendir(DIR, $pwd)); |
474 | my @files = readdir(DIR); | |
475 | closedir(DIR); | |
476 | ||
477 | foreach my $file (@files) { | |
478 | next if ($file eq '.' or $file eq '..'); | |
479 | ||
480 | my $path = "$pwd/$file"; | |
481 | next if (-l $path); | |
482 | ||
472c9e10 TH |
483 | # skip /proc/PID except /proc/1 |
484 | next if (($path =~ /^\/proc\/[0-9]+$/) && | |
485 | ($path !~ /^\/proc\/1$/)); | |
486 | ||
b401f56f TH |
487 | next if (skip($path)); |
488 | ||
136fc5c4 TH |
489 | if (-d $path) { |
490 | push @dirs, $path; | |
b401f56f | 491 | next; |
136fc5c4 | 492 | } |
b401f56f TH |
493 | |
494 | dprint "parsing: $path\n"; | |
495 | timed_parse_file($path); | |
136fc5c4 TH |
496 | } |
497 | } | |
498 | } | |
d09bd8da TH |
499 | |
500 | sub format_output | |
501 | { | |
502 | my ($file) = @_; | |
503 | ||
504 | # Default is to show raw results. | |
505 | if ($raw or (!$squash_by_path and !$squash_by_filename)) { | |
506 | dump_raw_output($file); | |
507 | return; | |
508 | } | |
509 | ||
510 | my ($total, $dmesg, $paths, $files) = parse_raw_file($file); | |
511 | ||
512 | printf "\nTotal number of results from scan (incl dmesg): %d\n", $total; | |
513 | ||
514 | if (!$suppress_dmesg) { | |
515 | print_dmesg($dmesg); | |
516 | } | |
517 | ||
518 | if ($squash_by_filename) { | |
519 | squash_by($files, 'filename'); | |
520 | } | |
521 | ||
522 | if ($squash_by_path) { | |
523 | squash_by($paths, 'path'); | |
524 | } | |
525 | } | |
526 | ||
527 | sub dump_raw_output | |
528 | { | |
529 | my ($file) = @_; | |
530 | ||
531 | open (my $fh, '<', $file) or die "$0: $file: $!\n"; | |
532 | while (<$fh>) { | |
533 | if ($suppress_dmesg) { | |
534 | if ("dmesg:" eq substr($_, 0, 6)) { | |
535 | next; | |
536 | } | |
537 | } | |
538 | print $_; | |
539 | } | |
540 | close $fh; | |
541 | } | |
542 | ||
543 | sub parse_raw_file | |
544 | { | |
545 | my ($file) = @_; | |
546 | ||
547 | my $total = 0; # Total number of lines parsed. | |
548 | my @dmesg; # dmesg output. | |
549 | my %files; # Unique filenames containing leaks. | |
550 | my %paths; # Unique paths containing leaks. | |
551 | ||
552 | open (my $fh, '<', $file) or die "$0: $file: $!\n"; | |
553 | while (my $line = <$fh>) { | |
554 | $total++; | |
555 | ||
556 | if ("dmesg:" eq substr($line, 0, 6)) { | |
557 | push @dmesg, $line; | |
558 | next; | |
559 | } | |
560 | ||
561 | cache_path(\%paths, $line); | |
562 | cache_filename(\%files, $line); | |
563 | } | |
564 | ||
565 | return $total, \@dmesg, \%paths, \%files; | |
566 | } | |
567 | ||
568 | sub print_dmesg | |
569 | { | |
570 | my ($dmesg) = @_; | |
571 | ||
572 | print "\ndmesg output:\n"; | |
573 | ||
574 | if (@$dmesg == 0) { | |
575 | print "<no results>\n"; | |
576 | return; | |
577 | } | |
578 | ||
579 | foreach(@$dmesg) { | |
580 | my $index = index($_, ': '); | |
581 | $index += 2; # skid ': ' | |
582 | print substr($_, $index); | |
583 | } | |
584 | } | |
585 | ||
586 | sub squash_by | |
587 | { | |
588 | my ($ref, $desc) = @_; | |
589 | ||
590 | print "\nResults squashed by $desc (excl dmesg). "; | |
591 | print "Displaying [<number of results> <$desc>], <example result>\n"; | |
592 | ||
593 | if (keys %$ref == 0) { | |
594 | print "<no results>\n"; | |
595 | return; | |
596 | } | |
597 | ||
598 | foreach(keys %$ref) { | |
599 | my $lines = $ref->{$_}; | |
600 | my $length = @$lines; | |
601 | printf "[%d %s] %s", $length, $_, @$lines[0]; | |
602 | } | |
603 | } | |
604 | ||
605 | sub cache_path | |
606 | { | |
607 | my ($paths, $line) = @_; | |
608 | ||
609 | my $index = index($line, ': '); | |
610 | my $path = substr($line, 0, $index); | |
611 | ||
612 | $index += 2; # skip ': ' | |
613 | add_to_cache($paths, $path, substr($line, $index)); | |
614 | } | |
615 | ||
616 | sub cache_filename | |
617 | { | |
618 | my ($files, $line) = @_; | |
619 | ||
620 | my $index = index($line, ': '); | |
621 | my $path = substr($line, 0, $index); | |
622 | my $filename = basename($path); | |
623 | ||
624 | $index += 2; # skip ': ' | |
625 | add_to_cache($files, $filename, substr($line, $index)); | |
626 | } | |
627 | ||
628 | sub add_to_cache | |
629 | { | |
630 | my ($cache, $key, $value) = @_; | |
631 | ||
632 | if (!$cache->{$key}) { | |
633 | $cache->{$key} = (); | |
634 | } | |
635 | push @{$cache->{$key}}, $value; | |
636 | } |