Debian Codegraph @

bin/mce_grep

raw
1 #!/usr/bin/env perl
2 ###############################################################################
3 ## ----------------------------------------------------------------------------
4 ## A MCE-driven 'wrapper' script for grep-like C binaries.
5 ##
6 ## Making a wrapper for other grep-like binaries is easy. Simply link this
7 ## script or make a copy. The prefix 'mce_' is stripped from the name for
8 ## determining the actual binary to use. A trailing '.pl' extension is
9 ## optional. Please ensure the binary is in your path.
10 ##
11 ## agrep.exe (z)grep.exe (z)egrep.exe (z)fgrep.exe tre-agrep.exe
12 ## agrep (z)grep (z)egrep (z)fgrep tre-agrep
13 ##
14 ## ln mce_grep mce_egrep ; ln mce_grep mce_zegrep ; (or) ln -s ...
15 ## ln mce_grep mce_fgrep ; ln mce_grep mce_zfgrep ; (or) cp ...
16 ## ln mce_grep mce_zgrep
17 ##
18 ## ln mce_grep mce_tre-agrep # binary is named tre-agrep
19 ## ln mce_grep mce_agrep # or agrep
20 ##
21 ## Caveat for (z)grep, (z)egrep, and (z)fgrep. When '--chunklevel=file' is
22 ## specified or passing a single file, MCE workers read the file in smaller
23 ## chunks. This is fine, typically. However, the following grep options may
24 ## report inaccurately due to crossing boundaries in regards to chunks.
25 ## To ensure accuracy, run with '--chunklevel=list'.
26 ##
27 ## -A NUM, --after-context=NUM
28 ## -B NUM, --before-context=NUM
29 ## -C NUM, --context=NUM
30 ##
31 ## ============================================================================
32 ## ----------------------------------------------------------------------------
33 ## 2014-01-21 v1.008
34 ## Initial release by Mario Roy.
35 ##
36 ## 2014-07-23 v1.009
37 ## ${^CHILD_ERROR_NATIVE} is not defined in Perl 5.8.x. Use $? instead.
38 ## Compute chunk_level => 'auto' to use 'file' when reading STDIN.
39 ## Set chunk_size to 8M when not specified (from 4M previously).
40 ##
41 ## 2014-12-22 v1.010
42 ## Small code refactoring.
43 ##
44 ## 2017-02-25 v1.011
45 ## When -r is specified and zero paths are given, start recursively in the
46 ## current directory. Set chunk-level accordingly to list mode.
47 ##
48 ## 2017-03-01 v1.012
49 ## Updated logic for determining chunk level mode. Ditto for chunk size.
50 ## Fixed an issue for not seeing STDERR output with '--chunk-level=file'.
51 ## Added support for zgrep, zegrep, and zfgrep. Thank you, Jeff Rouse.
52 ## https://www.activestate.com/blog/2016/12/grep-losing-its-grip
53 ##
54 ## 2017-03-27 v1.013
55 ## Check for $!{'EINTR'} during syswrite.
56 ##
57 ###############################################################################
58
59 ###############################################################################
60 ## ----------------------------------------------------------------------------
61 ## Which to choose? bin/mce_grep or mce-examples/other/egrep.pl
62 ##
63 ## (A) This wrapper script is good for expensive pattern matching, especially
64 ## with agrep and tre-agrep. It supports more options due to being passed
65 ## to the binary. It supports two levels of chunking specified with the
66 ## --chunk-level={auto|file|list} option. Choose file for large files.
67 ##
68 ## (B) The egrep.pl script is a pure-Perl implementation with fewer options.
69 ## It's strengh is searching a single file and/or with many expressions.
70 ## https://github.com/marioroy/mce-examples/blob/master/other/egrep.pl
71 ##
72 ###############################################################################
73
74 use strict; use warnings;
75
76 ## no critic (InputOutput::ProhibitBarewordFileHandles)
77 ## no critic (InputOutput::ProhibitTwoArgOpen)
78
79 use Cwd 'abs_path'; ## Insert lib-path at the head of @INC.
80 use lib abs_path($0 =~ m{^(.*)[\\/]} && $1 || abs_path) . '/../lib';
81
82 my ($prog_name, $prog_dir);
83
84 BEGIN {
85 $prog_name = $0; $prog_name =~ s{^.*[\\/]}{}g;
86 $prog_dir = abs_path($0); $prog_dir =~ s{[\\/][^\\/]*$}{};
87
88 $ENV{PATH} .= ($^O eq 'MSWin32' ? ';' : ':') . $prog_dir;
89 }
90 sub INIT {
91 ## Provide file globbing support under Windows similar to Unix.
92 @ARGV = <@ARGV> if ($^O eq 'MSWin32');
93 }
94
95 use Getopt::Long qw(
96 :config bundling pass_through no_ignore_case no_auto_abbrev
97 );
98 use Fcntl qw( O_RDONLY );
99 use Scalar::Util qw( looks_like_number );
100 use Errno ();
101
102 use MCE::Signal qw( -use_dev_shm );
103 use MCE 1.5; # or later release
104
105 ###############################################################################
106 ## ----------------------------------------------------------------------------
107 ## Usage and validation.
108 ##
109 ###############################################################################
110
111 sub usage {
112 print <<"::_USAGE_BLOCK_END_::";
113
114 Options for Script:
115 --max-workers=NUM override max workers (default auto, 8 maximum)
116 --maxworkers=NUM e.g. auto*2, 16
117
118 --chunk-level=LEVEL override chunk level (default auto)
119 --chunklevel=LEVEL chunk at [file] or [list] level
120
121 --chunk-size=NUM[KM] override chunk size (set at limit if under or over)
122 --chunksize=NUM[KM] [file] default: 8M minimum: 200K maximum: 20M
123 [list] default: 1 minimum: 1 maximum: 200
124
125 --lang=LOCALE override locale
126 e.g. C, en_US.UTF-8, en_US.ISO-8859-1
127
128 Options for Binary:
129 ::_USAGE_BLOCK_END_::
130
131 return;
132 }
133
134 my $is_mswin32 = $^O eq 'MSWin32';
135 my ($cmd_name, $cmd_path);
136
137 $cmd_name = $prog_name;
138 $cmd_name =~ s{^mce_}{};
139 $cmd_name =~ s{\.pl$}{};
140
141 if ($is_mswin32) {
142 $cmd_name .= '.exe';
143 for ( split ';', $ENV{'PATH'} ) {
144 $cmd_path = "$_\\$cmd_name", last if (-x "$_\\$cmd_name");
145 }
146 }
147 else {
148 $cmd_name .= '.exe' if $^O eq 'cygwin';
149 for ( split ':', $ENV{'PATH'} ) {
150 $cmd_path = "$_/$cmd_name", last if (-x "$_/$cmd_name");
151 }
152 }
153
154 unless (defined $cmd_path) {
155 print {*STDERR} "$prog_name: $cmd_name: command not found\n";
156 exit 2;
157 }
158 {
159 my %valid_names = map { $_ => 1 } qw(
160 grep.exe egrep.exe fgrep.exe zgrep.exe zegrep.exe zfgrep.exe
161 grep egrep fgrep zgrep zegrep zfgrep
162
163 agrep.exe tre-agrep.exe agrep tre-agrep
164 );
165 unless (exists $valid_names{$cmd_name}) {
166 print {*STDERR} "$prog_name: $cmd_name: command not supported\n";
167 exit 2;
168 }
169 }
170
171 ###############################################################################
172 ## ----------------------------------------------------------------------------
173 ## Process arguments.
174 ##
175 ###############################################################################
176
177 my ($h_patn, $b_flag, $c_flag, $H_flag, $h_flag, $n_flag, $q_flag) = ((0) x 7);
178 my (@r_patn, @args, $arg, @files, $file); my ($f_list, $r_flag) = (0, 0);
179 my ($exit_status, $found_match, $skip_args, $w_filename) = (0, 0, 0, 0);
180
181 my $max_workers = 'auto'; my $chunk_level = 'auto'; my $chunk_size;
182 my $max_count = 0; my $no_msg = 0; my @TMP_ARGV;
183
184 ## Option parsing step 1.
185
186 for my $i (0 .. @ARGV - 1) {
187 if ($ARGV[$i] eq '--') {
188 @TMP_ARGV = @ARGV[$i .. @ARGV - 1]; @ARGV = @ARGV[0 .. $i - 1];
189 last;
190 }
191 }
192
193 {
194 local $SIG{__WARN__} = sub { };
195
196 GetOptions(
197 'maxworkers|max-workers|max_workers=s' => \$max_workers,
198 'chunklevel|chunk-level|chunk_level=s' => \$chunk_level,
199 'chunksize|chunk-size|chunk_size=s' => \$chunk_size,
200
201 'lang=s' => sub {
202 my ($self, $lang) = @_;
203 delete @ENV{ qw( LC_MESSAGES LC_COLLATE LC_CTYPE LC_ALL ) };
204 $ENV{'LANG'} = $lang;
205 },
206 'help' => sub {
207 usage(); system $cmd_path, '--help'; print "\n";
208 exit 0;
209 },
210 'V|version' => sub {
211 system $cmd_path, '--version'; exit 0;
212 },
213
214 'q|quiet|silent' => \$q_flag,
215 'H|with-filename' => sub { $H_flag = 1; $h_flag = 0; },
216 'h|no-filename' => sub { $H_flag = 0; $h_flag = 1; },
217 'm|max-count=s' => \$max_count,
218 'R|r|recursive' => \$r_flag
219 );
220
221 if ($max_workers !~ /^auto/) {
222 unless (looks_like_number($max_workers) && $max_workers > 0) {
223 print {*STDERR} "$prog_name: invalid max workers\n";
224 exit 2;
225 }
226 }
227 if ($chunk_level !~ /^(?:auto|file|list)$/) {
228 print {*STDERR} "$prog_name: invalid chunk level\n";
229 exit 2;
230 }
231 if (defined $chunk_size) {
232 if ($chunk_size =~ /^(\d+)K/i) {
233 $chunk_size = $1 * 1024;
234 }
235 elsif ($chunk_size =~ /^(\d+)M/i) {
236 $chunk_size = $1 * 1024 * 1024;
237 }
238 if (!looks_like_number($chunk_size) || $chunk_size < 1) {
239 print {*STDERR} "$prog_name: invalid chunk size\n";
240 exit 2;
241 }
242 }
243 if ($max_count) {
244 unless (looks_like_number($max_count) && $max_count >= 0) {
245 print {*STDERR} "$prog_name: invalid max count\n";
246 exit 2;
247 }
248 }
249 }
250
251 ## Option parsing step 2.
252
253 if (@TMP_ARGV) {
254 @ARGV = (@ARGV, @TMP_ARGV); undef @TMP_ARGV;
255 if ($ARGV[0] eq '--') {
256 shift @ARGV; $skip_args = 1; push @args, '--';
257 }
258 }
259
260 while ( @ARGV ) {
261 $arg = shift @ARGV; $arg =~ s/ /\\ /g;
262
263 if ($skip_args) {
264 push @files, $arg;
265 }
266 elsif (substr($arg, 0, 2) eq '--') { ## --OPTION
267 if ($arg eq '--') {
268 $skip_args = 1; push @args, $arg;
269 next;
270 }
271 $h_patn = 1 if $arg =~ /^--regexp=/;
272 $h_patn = 1 if $arg =~ /^--file=/;
273 $b_flag = 1 if $arg eq '--byte-offset';
274 $c_flag = 1 if $arg eq '--count';
275 $f_list = 1 if $arg eq '--files-without-match';
276 $f_list = 1 if $arg eq '--files-with-matches';
277 $n_flag = 1 if $arg eq '--record-number';
278 $n_flag = 1 if $arg eq '--line-number';
279 $no_msg = 1 if $arg eq '--no-messages';
280
281 if ($arg =~ /^--directories=(.+)/) {
282 if ($1 ne 'recurse') {
283 push @args, $arg;
284 } else {
285 $r_flag = 1;
286 }
287 }
288 elsif ($arg =~ /^--include=.+/) {
289 push @r_patn, $arg;
290 }
291 elsif ($arg =~ /^--exclude=.+/) {
292 push @r_patn, $arg;
293 }
294 elsif ($arg =~ /^--exclude-from=.+/) {
295 push @r_patn, $arg;
296 }
297 elsif ($arg =~ /^--exclude-dir=.+/) {
298 push @r_patn, $arg;
299 }
300 else {
301 ## Pass arguments to the C binary
302 push @args, $arg;
303 }
304 }
305 elsif (substr($arg, 0, 1) eq '-') { ## -OPTION
306 if ($arg eq '-') {
307 push @files, $arg;
308 next;
309 }
310 my $len = length $arg;
311
312 for (my $x = 1; $x < $len; $x++) {
313 my $a = substr($arg, $x, 1);
314
315 $f_list = 1 if $a eq 'L' || $a eq 'l';
316 $h_patn = 1 if $a eq 'e' || $a eq 'f';
317 $b_flag = 1 if $a eq 'b';
318 $c_flag = 1 if $a eq 'c';
319 $n_flag = 1 if $a eq 'n';
320
321 $no_msg = 1
322 if ($a eq 's' && $cmd_name !~ /agrep/);
323 }
324
325 next if $arg eq '-';
326
327 ## Pass arguments to the C binary
328 if ($cmd_name =~ /agrep/) {
329 push @args, $arg;
330 if (substr($arg, -1) =~ /[efDISEd]/) {
331 $arg = shift @ARGV;
332 $arg =~ s/ /\\ /g;
333 push @args, $arg;
334 }
335 }
336 else {
337 my $a = substr($arg, -1);
338 push @args, $arg if ($arg ne '-d');
339 if ($a =~ /[efABCD]/) {
340 $arg = shift @ARGV;
341 $arg =~ s/ /\\ /g;
342 push @args, $arg;
343 }
344 elsif ($a eq 'd') {
345 $arg = shift @ARGV;
346 if ($arg ne 'recurse') {
347 push @args, '-d', $arg;
348 }
349 else {
350 $r_flag = 1;
351 }
352 }
353 }
354 }
355 else { ## FILE
356 push @files, $arg;
357 }
358 }
359
360 ## Option parsing step 3.
361
362 push @args, shift @files if ($h_patn == 0 && @files > 0);
363
364 if ((!$h_flag && @files > 1) || (!$h_flag && $r_flag) || $H_flag) {
365 $w_filename = 1;
366 }
367
368 if ($r_flag && !@files) {
369 push @files, '.';
370 }
371
372 if (@args == 0) {
373 system $cmd_path;
374 exit 2;
375 }
376
377 ###############################################################################
378 ## ----------------------------------------------------------------------------
379 ## MCE callback functions: Error, File, and Count.
380 ##
381 ###############################################################################
382
383 my ($_order_id, %_tmp, %_nrecs, %_nsize, $_start_nrecs, $_start_nsize);
384 my ($_abort_all, $_abort_job, $_total_found);
385
386 sub _error {
387
388 my ($msg) = @_;
389
390 print {*STDERR} $msg;
391 $exit_status = 2;
392
393 return;
394 }
395
396 sub _abort_job {
397
398 if (!$_abort_job) {
399 MCE->abort;
400 $_abort_job = $_total_found = $found_match = 1;
401 $_abort_all = 1 if $q_flag;
402 }
403
404 return;
405 }
406
407 sub _output_cnt {
408
409 my ($chunk_id, $out_file, @_rest) = @_;
410 my $cnt;
411
412 if (-s $out_file) {
413 $found_match = 1;
414
415 open my $fh, '<', $out_file;
416 chomp($cnt = <$fh>);
417 close $fh;
418
419 $_total_found += $cnt;
420
421 if ($q_flag && !$_abort_all) {
422 MCE->abort; $_abort_all = $_abort_job = 1;
423 }
424 }
425
426 unlink $out_file;
427
428 return;
429 }
430
431 sub _set_found_match {
432
433 $found_match = 1;
434
435 return;
436 }
437
438 ###############################################################################
439 ## ----------------------------------------------------------------------------
440 ## MCE callback function: Output without line-number or byte-offset
441 ##
442 ###############################################################################
443
444 sub _output_n0 {
445
446 my ($chunk_id, $out_file, @_rest) = @_;
447
448 $_tmp{ $chunk_id } = $out_file;
449
450 return unless exists $_tmp{ $_order_id };
451
452 do {
453 my $out_file = $_tmp{ $_order_id };
454
455 if (!$_abort_job && -s $out_file) {
456 my ($fh, $buffer); $found_match = 1;
457
458 if ($q_flag) {
459 unless ($_abort_all) {
460 MCE->abort; $_abort_all = $_abort_job = 1;
461 }
462 }
463 else {
464 if ($w_filename) {
465 open $fh, '<', $out_file;
466 while (<$fh>) {
467 print $file . ':' . $_;
468 if ($max_count && ++$_total_found == $max_count) {
469 MCE->abort; $_abort_job = 1;
470 last;
471 }
472 }
473 close $fh;
474 }
475 else {
476 if ($max_count) {
477 open $fh, '<', $out_file;
478 while (<$fh>) {
479 print $_;
480 if ($max_count && ++$_total_found == $max_count) {
481 MCE->abort; $_abort_job = 1;
482 last;
483 }
484 }
485 close $fh;
486 }
487 else {
488 sysopen $fh, $out_file, O_RDONLY;
489 sysread $fh, $buffer, -s $fh;
490 close $fh;
491
492 print $buffer;
493 }
494 }
495 }
496 }
497
498 delete $_tmp{ $_order_id };
499 unlink $out_file;
500
501 } while (exists $_tmp{ ++$_order_id });
502
503 return;
504 }
505
506 ###############################################################################
507 ## ----------------------------------------------------------------------------
508 ## MCE callback function: Output with line-number or byte-offset
509 ##
510 ###############################################################################
511
512 sub _output_n1 {
513
514 my ($chunk_id, $out_file, $n_records, $size) = @_;
515
516 $_tmp{ $chunk_id } = $out_file;
517 $_nsize{ $chunk_id } = $n_flag ? $n_records : $size;
518
519 return unless exists $_tmp{ $_order_id };
520
521 do {
522 my $out_file = $_tmp{ $_order_id };
523
524 if ($_order_id > 1) {
525 $_start_nsize += $_nsize{ $_order_id - 1 };
526 delete $_nsize{ $_order_id - 1 };
527 }
528
529 if (!$_abort_job && -s $out_file) {
530 my ($p1, $size); $found_match = 1;
531
532 if ($q_flag) {
533 unless ($_abort_all) {
534 MCE->abort; $_abort_all = $_abort_job = 1;
535 }
536 }
537 else {
538 open my $fh, '<', $out_file;
539
540 if ($w_filename) {
541 while (<$fh>) {
542 $p1 = index($_, ':');
543 $size = $_start_nsize + substr($_, 0, $p1);
544
545 print $file . ':' . $size . substr($_, $p1);
546 if ($max_count && ++$_total_found == $max_count) {
547 MCE->abort; $_abort_job = 1;
548 last;
549 }
550 }
551 }
552 else {
553 while (<$fh>) {
554 $p1 = index($_, ':');
555 $size = $_start_nsize + substr($_, 0, $p1);
556
557 print $size . substr($_, $p1);
558 if ($max_count && ++$_total_found == $max_count) {
559 MCE->abort; $_abort_job = 1;
560 last;
561 }
562 }
563 }
564
565 close $fh;
566 }
567 }
568
569 delete $_tmp{ $_order_id };
570 unlink $out_file;
571
572 } while (exists $_tmp{ ++$_order_id });
573
574 return;
575 }
576
577 ###############################################################################
578 ## ----------------------------------------------------------------------------
579 ## MCE callback function: Output with line-number and byte-offset
580 ##
581 ###############################################################################
582
583 sub _output_n2 {
584
585 my ($chunk_id, $out_file, $n_records, $size) = @_;
586
587 $_tmp{ $chunk_id } = $out_file;
588 $_nrecs{ $chunk_id } = $n_records;
589 $_nsize{ $chunk_id } = $size;
590
591 return unless exists $_tmp{ $_order_id };
592
593 do {
594 my $out_file = $_tmp{ $_order_id };
595
596 if ($_order_id > 1) {
597 $_start_nrecs += $_nrecs{ $_order_id - 1 };
598 delete $_nrecs{ $_order_id - 1 };
599
600 $_start_nsize += $_nsize{ $_order_id - 1 };
601 delete $_nsize{ $_order_id - 1 };
602 }
603
604 if (!$_abort_job && -s $out_file) {
605 my ($p1, $p2, $recs, $size); $found_match = 1;
606
607 if ($q_flag) {
608 unless ($_abort_all) {
609 MCE->abort; $_abort_all = $_abort_job = 1;
610 }
611 }
612 else {
613 open my $fh, '<', $out_file;
614
615 if ($w_filename) {
616 while (<$fh>) {
617 $p1 = index($_, ':');
618 $recs = $_start_nrecs + substr($_, 0, $p1++);
619
620 $p2 = index($_, ':', $p1);
621 $size = $_start_nsize + substr($_, $p1, $p2 - $p1);
622
623 print $file . ':' . $recs . ':' . $size . substr($_, $p2);
624 if ($max_count && ++$_total_found == $max_count) {
625 MCE->abort; $_abort_job = 1;
626 last;
627 }
628 }
629 }
630 else {
631 while (<$fh>) {
632 $p1 = index($_, ':');
633 $recs = $_start_nrecs + substr($_, 0, $p1++);
634
635 $p2 = index($_, ':', $p1);
636 $size = $_start_nsize + substr($_, $p1, $p2 - $p1);
637
638 print $recs . ':' . $size . substr($_, $p2);
639 if ($max_count && ++$_total_found == $max_count) {
640 MCE->abort; $_abort_job = 1;
641 last;
642 }
643 }
644 }
645
646 close $fh;
647 }
648 }
649
650 delete $_tmp{ $_order_id };
651 unlink $out_file;
652
653 } while (exists $_tmp{ ++$_order_id });
654
655 return;
656 }
657
658 ###############################################################################
659 ## ----------------------------------------------------------------------------
660 ## MCE user functions: run-mode = file.
661 ##
662 ###############################################################################
663
664 sub user_begin_file {
665
666 $0 = $^X;
667
668 return;
669 }
670
671 sub make_user_func_file {
672
673 my $first_time = 1;
674
675 return sub {
676 my ($self, $chunk_ref, $chunk_id) = @_;
677
678 my ($out_fh, $err_fh, $cmd_fh, $has_err); my $n_records = 0;
679 my $out_file = MCE->sess_dir .'/'. $chunk_id;
680
681 if ($n_flag) {
682 $n_records++ while ($$chunk_ref =~ m!\n!mg);
683 }
684
685 if ($is_mswin32) {
686 $out_file =~ s{/}{\\\\}g;
687
688 open my $in_fh, '+>', $out_file . '.in'; binmode $in_fh, ':raw';
689 print {$in_fh} $$chunk_ref;
690 close $in_fh;
691
692 my $err_file = $first_time ? "2> $out_file.err" : '';
693 system("$cmd_path < $out_file.in @args > $out_file $err_file");
694
695 unlink "$out_file.in";
696 }
697 else {
698 ## Borrowed bits from IPC::Run3 for STDOUT/ERR. However, I settled
699 ## on passing STDIN via open, for lesser overhead, versus calling
700 ## system from observations made during testing.
701
702 local (*STDOUT_SAVE, *STDERR_SAVE);
703
704 open STDOUT_SAVE, '>&STDOUT';
705 open $out_fh, '+>', $out_file; binmode $out_fh, ':raw';
706 open STDOUT, '>&' . fileno $out_fh;
707
708 if ($first_time) {
709 open STDERR_SAVE, '>&STDERR';
710 open $err_fh, '+>', "$out_file.err"; binmode $err_fh, ':raw';
711 open STDERR, '>&' . fileno $err_fh;
712 }
713
714 ## Got "maximal count of pending signals (NUM) exceeded" message.
715 ## Thus the reason for using syswrite versus print below.
716
717 local $SIG{PIPE} = sub { };
718
719 open $cmd_fh, '|-', $cmd_path, @args; # Run external command
720
721 my $wrote = 0; # Write to STDIN
722
723 WRITE: {
724 $wrote += ( syswrite (
725 $cmd_fh, $$chunk_ref, length($$chunk_ref) - $wrote, $wrote
726 )) or do {
727 unless ( defined $wrote ) {
728 redo WRITE if ( $! == Errno::EINTR() );
729 }
730 };
731 }
732
733 close $cmd_fh;
734
735 open STDOUT, '>&STDOUT_SAVE';
736 close $out_fh;
737
738 if ($first_time) {
739 open STDERR, '>&STDERR_SAVE';
740 close $err_fh;
741 }
742 }
743
744 MCE->abort if ($q_flag && -s $out_file);
745
746 ## Send error.
747
748 if ($first_time) {
749 my $err_file = "$out_file.err";
750
751 if (-s $err_file) {
752 $has_err = 1; MCE->abort;
753 if ($chunk_id == 1) {
754 open $err_fh, '<', $err_file;
755 local $/ = undef; MCE->do('_error', <$err_fh>);
756 close $err_fh;
757 }
758 }
759
760 unlink $err_file;
761 $first_time = 0;
762 }
763
764 ## Gather output.
765
766 if ($f_list) {
767 MCE->do('_abort_job') if (!$has_err && -s $out_file);
768 unlink $out_file;
769 }
770 else {
771 MCE->gather($chunk_id, $out_file, $n_records, length $$chunk_ref)
772 unless $has_err;
773 }
774
775 return;
776 };
777 }
778
779 ###############################################################################
780 ## ----------------------------------------------------------------------------
781 ## MCE user functions: run-mode = list.
782 ##
783 ###############################################################################
784
785 sub user_begin_list {
786
787 $0 = $^X;
788
789 use vars qw( $child_found_match );
790 our $child_found_match = 0;
791
792 return;
793 }
794
795 sub user_end_list {
796
797 MCE->do('_set_found_match') if $child_found_match;
798
799 return;
800 }
801
802 sub user_func_list {
803
804 my ($self, $chunk_ref, $chunk_id) = @_;
805
806 my ($output, $err_fh, $status);
807 my $err_file = MCE->sess_dir .'/'. $chunk_id . '.err';
808
809 $$chunk_ref =~ s/\n/ /mg;
810 local $?;
811
812 if ($is_mswin32) {
813 $err_file =~ s{/}{\\\\}g;
814 $output = `$cmd_path @args $$chunk_ref 2> $err_file`;
815 $status = $? >> 8;
816 }
817 else {
818 local *STDERR_SAVE;
819
820 open STDERR_SAVE, '>&STDERR';
821 open $err_fh, '+>', $err_file; binmode $err_fh, ':raw';
822 open STDERR, '>&' . fileno $err_fh;
823
824 $output = `$cmd_path @args $$chunk_ref`;
825 $status = $? >> 8;
826
827 open STDERR, '>&STDERR_SAVE';
828 close $err_fh;
829 }
830
831 MCE->abort if ($q_flag && length $output);
832
833 ## Send error.
834
835 if (-s $err_file) {
836 open $err_fh, '<', $err_file;
837 local $/ = undef; MCE->do('_error', <$err_fh>);
838 close $err_fh;
839 }
840
841 unlink $err_file;
842
843 ## Gather output.
844
845 if ($q_flag) {
846 MCE->do('_abort_job') if ($status == 0);
847 }
848 else {
849 if (length $output) {
850 MCE->print($output);
851 $child_found_match = 1;
852 }
853 }
854
855 return;
856 }
857
858 ###############################################################################
859 ## ----------------------------------------------------------------------------
860 ## Process routines: run-mode = file.
861 ##
862 ###############################################################################
863
864 sub process_file {
865
866 ($file) = @_;
867
868 if ($file eq '-') {
869 open(STDIN, '<', ($is_mswin32) ? 'CON' : '/dev/tty') or die $!;
870 process_stdin();
871 }
872 elsif (! -e $file) {
873 $exit_status = 2;
874
875 print {*STDERR} "$prog_name: $file: No such file or directory\n"
876 unless $no_msg;
877 }
878 elsif (-d $file) {
879 $exit_status = 1;
880 }
881 else {
882 $_abort_job = $_start_nrecs = $_start_nsize = $_total_found = 0;
883 $_order_id = 1;
884
885 MCE->process($file);
886 %_nrecs = (); %_nsize = ();
887
888 if (!$q_flag && $f_list) {
889 print "$file\n" if $_total_found;
890 }
891 elsif (!$q_flag && $c_flag) {
892 $_total_found = $max_count
893 if ($max_count && $_total_found > $max_count);
894
895 print "$file:" if $w_filename;
896 print "$_total_found\n";
897 }
898 }
899
900 return;
901 }
902
903 sub process_stdin {
904
905 $file = '(standard input)';
906
907 $_abort_job = $_start_nrecs = $_start_nsize = $_total_found = 0;
908 $_order_id = 1;
909
910 MCE->process(\*STDIN);
911 %_nrecs = (); %_nsize = ();
912
913 if (!$q_flag && $f_list) {
914 print "$file\n" if $_total_found;
915 }
916 elsif (!$q_flag && $c_flag) {
917 $_total_found = $max_count
918 if ($max_count && $_total_found > $max_count);
919
920 print "$file:" if $w_filename;
921 print "$_total_found\n";
922 }
923
924 return;
925 }
926
927 ###############################################################################
928 ## ----------------------------------------------------------------------------
929 ## Configure Many-Core Engine.
930 ##
931 ###############################################################################
932
933 my $gather_func;
934
935 if ($chunk_level eq 'auto') {
936 if ((@files <= 1 || $files[0] eq '-') && !$r_flag && $cmd_name !~ /^z/i) {
937 $chunk_level = 'file';
938 } else {
939 $chunk_level = 'list';
940 }
941 }
942
943 if ($chunk_level eq 'list') {
944 $chunk_size = 200 if (!defined $chunk_size && @files > 200);
945 $chunk_size = 1 if (!defined $chunk_size);
946 $chunk_size = 200 if ($chunk_size > 200);
947 $chunk_size = 1 if ($chunk_size < 1);
948
949 unshift @args, '-H' if (!$h_flag && ($H_flag || $r_flag || @files > 1));
950 unshift @args, '-h' if ($h_flag);
951 unshift @args, '-q' if ($q_flag);
952
953 MCE->new(
954 max_workers => $max_workers, chunk_size => $chunk_size, use_slurpio => 1,
955 user_begin => \&user_begin_list, user_func => \&user_func_list,
956 user_end => \&user_end_list
957 );
958 }
959 else {
960 $chunk_size = 8_388_608 unless defined $chunk_size; ## 8M
961 $chunk_size = 20_971_520 if $chunk_size > 20_971_520; ## 20M
962 $chunk_size = 204_800 if $chunk_size < 204_800; ## 200K
963
964 if ($f_list) {
965 $gather_func = undef;
966 }
967 elsif ($c_flag) {
968 $gather_func = \&_output_cnt;
969 }
970 elsif ($n_flag && $b_flag) {
971 $gather_func = \&_output_n2;
972 }
973 elsif ($n_flag || $b_flag) {
974 $gather_func = \&_output_n1;
975 }
976 else {
977 $gather_func = \&_output_n0;
978 }
979
980 MCE->new(
981 max_workers => $max_workers, chunk_size => $chunk_size, use_slurpio => 1,
982 user_begin => \&user_begin_file, user_func => make_user_func_file(),
983 gather => $gather_func
984 );
985 }
986
987 ###############################################################################
988 ## ----------------------------------------------------------------------------
989 ## Run.
990 ##
991 ###############################################################################
992
993 if ($r_flag && @files > 0) {
994 my ($list_fh, $list);
995
996 MCE->spawn;
997
998 if ($is_mswin32) {
999 $list = `egrep -lsr @r_patn ^ @files`;
1000 open $list_fh, '<', \$list;
1001 }
1002 else {
1003 open $list_fh, '-|', 'egrep', '-lsr', @r_patn, '^', @files;
1004 }
1005
1006 if ($chunk_level eq 'list') {
1007 MCE->process($list_fh);
1008 }
1009 else {
1010 while (<$list_fh>) {
1011 chomp;
1012 process_file($_);
1013 last if $_abort_all;
1014 }
1015 }
1016
1017 close $list_fh;
1018 }
1019 elsif (@files > 0) {
1020 if ($chunk_level eq 'list') {
1021 my $list = join("\n", @files) . "\n"; undef @files;
1022 open my $list_fh, '<', \$list;
1023 MCE->process($list_fh);
1024 close $list_fh;
1025 }
1026 else {
1027 foreach (@files) {
1028 process_file($_);
1029 last if $_abort_all;
1030 }
1031 }
1032 }
1033 else {
1034 if ($chunk_level eq 'list') {
1035 my $status = system($cmd_path, @args);
1036 exit($status >> 8);
1037 }
1038 else {
1039 process_stdin();
1040 }
1041 }
1042
1043 ###############################################################################
1044 ## ----------------------------------------------------------------------------
1045 ## Finish.
1046 ##
1047 ###############################################################################
1048
1049 MCE->shutdown;
1050
1051 if (!$q_flag && $exit_status) {
1052 exit($exit_status);
1053 }
1054 else {
1055 exit($found_match ? 0 : ($exit_status ? $exit_status : 1));
1056 }
1057