From ff229d0bd434ce87034f1929354082f862cb03f2 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 9 Jul 2010 22:37:45 +0200 Subject: [PATCH] Easier calc of max_command_length. Convert Global:: to Private::. Bugfix: ETA for args given on command line. --- src/parallel | 201 ++++++++++++++++---------------- unittest/actual-results/test25 | 10 ++ unittest/tests-to-run/test25.sh | 5 + unittest/wanted-results/test16 | 14 +-- unittest/wanted-results/test25 | 10 ++ 5 files changed, 130 insertions(+), 110 deletions(-) diff --git a/src/parallel b/src/parallel index be172674..39b0dc7f 100755 --- a/src/parallel +++ b/src/parallel @@ -671,6 +671,20 @@ To compress all html files using B run: B +=head1 EXAMPLE: Reading arguments from command line + +GNU B can take the arguments from command line instead of +stdin (standard input). To compress all html files in the current dir +using B run: + +B + +To convert *.wav to *.mp3 using LAME running one process per CPU core +run: + +B + + =head1 EXAMPLE: Inserting multiple arguments When moving a lot of files like this: B you will @@ -1677,7 +1691,7 @@ if($::opt_halt_on_error) { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20100707; + $Global::version = 20100708; $Global::progname = 'parallel'; $Global::debug = 0; $Global::verbose = 0; @@ -1695,7 +1709,6 @@ sub parse_options { $Global::exitstatus = 0; $Global::halt_on_error_exitstatus = 0; $Global::total_jobs = 0; - $Global::eta = 0; $Global::arg_sep = ":::"; Getopt::Long::Configure ("bundling","require_order"); @@ -1821,8 +1834,9 @@ sub parse_options { my @new_argv = (); while(@ARGV) { my $arg = shift @ARGV; - if($arg =~ /^$Global::arg_sep$/o) { + if($arg eq $Global::arg_sep) { unget_arg(@ARGV); + $Global::total_jobs += @ARGV; @ARGV=(); } else { push @new_argv, $arg; @@ -1880,7 +1894,6 @@ sub parse_options { $Global::default_simultaneous_sshlogins; } } - $Global::job_end_sequence=1; } sub cleanup { @@ -2126,75 +2139,51 @@ sub max_length_of_command_line { # Find the max_length of a command line # Returns: # number of chars on the longest command line allowed - # First find an upper bound - if(not $Global::command_line_max_len) { - $Global::command_line_max_len = limited_max_length(); + if(not $Private::command_line_max_len) { + $Private::command_line_max_len = limited_max_length(); if($::opt_s) { - if($::opt_s <= $Global::command_line_max_len) { - $Global::command_line_max_len = $::opt_s; + if($::opt_s <= $Private::command_line_max_len) { + $Private::command_line_max_len = $::opt_s; } else { print STDERR "$Global::progname: ", - "value for -s option should be < $Global::command_line_max_len\n"; + "value for -s option should be < $Private::command_line_max_len\n"; } } } - return $Global::command_line_max_len; + return $Private::command_line_max_len; +} + +sub max_length_limited_by_opt_s { + # Returns: + # min(opt_s, number of chars on the longest command line allowed) + if(is_acceptable_command_line_length($::opt_s)) { + debug("-s is OK: $::opt_s\n"); + return $::opt_s; + } + # -s is too long: Find the correct + return binary_find_max_length(0,$::opt_s); } sub limited_max_length { # Returns: # min(opt_s, number of chars on the longest command line allowed) - if($::opt_s) { - if(is_acceptable_command_line_length($::opt_s)) { - debug("-s is OK: $::opt_s\n"); - return $::opt_s; - } - # -s is too long: Find the correct - return binary_find_max_length(0,$::opt_s); - } + if($::opt_s) { return max_length_limited_by_opt_s() } -#TODO - # Running is_acceptable_command_line_length is expensive - # Try to run as few times as possible. - # If all arguments fit on the line now, don't try a longer length - my ($quoted_args,$quoted_args_no_ext); - my $more = more_arguments(); - my $len = 8; - my $is_acceptable; - do { - $len *= 16; - $is_acceptable = is_acceptable_command_line_length($len); - ($quoted_args,$quoted_args_no_ext) = - get_multiple_args($Global::command,$len,1); - $more = more_arguments(); - debug("Test len: $len\n"); - # Reset the getting of args - my $next = get_next_arg(); - if($next) { - push @$quoted_args, $next; - } - if (@$quoted_args) { - unget_arg(@$quoted_args); - } - } while ($more and $is_acceptable); - - if(not $is_acceptable) { - # There are more arguments than will fit on one line - # Then search for the actual max length between 0 and upper bound - return binary_find_max_length(int(($len)/16),$len); - } else { - # The arguments will fit on one line of length $len - return $len; - } + return real_max_length(); } sub real_max_length { + # Returns: + # The maximal command line length + # Use an upper bound of 8 MB if the shell allows for for infinite long lengths + my $upper = 8_000_000; my $len = 8; do { + if($len > $upper) { return $len }; $len *= 16; } while (is_acceptable_command_line_length($len)); # Then search for the actual max length between 0 and upper bound - return binary_find_max_length(int(($len)/16),$len); + return binary_find_max_length(int($len/16),$len); } sub binary_find_max_length { @@ -2218,8 +2207,9 @@ sub is_acceptable_command_line_length { # 0 if the command line length is too long # 1 otherwise my $len = shift; - $Global::is_acceptable_command_line_length++; - debug("$Global::is_acceptable_command_line_length $len\n"); + + $Private::is_acceptable_command_line_length++; + debug("$Private::is_acceptable_command_line_length $len\n"); local *STDERR; open (STDERR,">/dev/null"); system "true "."x"x$len; @@ -2484,7 +2474,7 @@ sub no_of_processing_units_sshlogin { sub no_of_cpus { # Returns: # Number of physical CPUs - if(not $Global::no_of_cpus) { + if(not $Private::no_of_cpus) { local $/="\n"; # If delimiter is set, then $/ will be wrong my $no_of_cpus = (no_of_cpus_freebsd() || no_of_cpus_darwin() @@ -2492,19 +2482,19 @@ sub no_of_cpus { || no_of_cpus_gnu_linux() ); if($no_of_cpus) { - $Global::no_of_cpus = $no_of_cpus; + $Private::no_of_cpus = $no_of_cpus; } else { warn("Cannot figure out number of cpus. Using 1"); - $Global::no_of_cpus = 1; + $Private::no_of_cpus = 1; } } - return $Global::no_of_cpus; + return $Private::no_of_cpus; } sub no_of_cores { # Returns: # Number of CPU cores - if(not $Global::no_of_cores) { + if(not $Private::no_of_cores) { local $/="\n"; # If delimiter is set, then $/ will be wrong my $no_of_cores = (no_of_cores_freebsd() || no_of_cores_darwin() @@ -2512,13 +2502,13 @@ sub no_of_cores { || no_of_cores_gnu_linux() ); if($no_of_cores) { - $Global::no_of_cores = $no_of_cores; + $Private::no_of_cores = $no_of_cores; } else { warn("Cannot figure out number of CPU cores. Using 1"); - $Global::no_of_cores = 1; + $Private::no_of_cores = 1; } } - return $Global::no_of_cores; + return $Private::no_of_cores; } sub no_of_cpus_gnu_linux { @@ -2633,6 +2623,17 @@ sub min { return $min; } +sub max { + # Returns: + # Maximum value of array + my $max = shift; + my @args = @_; + for my $a (@args) { + $max = ($max > $a) ? $max : $a; + } + return $max; +} + # # Running and printing the jobs @@ -2851,13 +2852,13 @@ sub progress { my $completed = 0; for(@workers) { $completed += ($Global::host{$_}{'completed'}||0) } if($completed) { - $Global::first_completed ||= time; - my $avgtime = (time-$Global::first_completed)/$completed; + $Private::first_completed ||= time; + my $avgtime = (time-$Private::first_completed)/$completed; my $this_eta = ($Global::total_jobs - $completed) * $avgtime; - $Global::eta ||= $this_eta; + $Private::eta ||= $this_eta; # Smooth the eta so it does not jump wildly - $Global::eta = 0.9 * $Global::eta + 0.1 * $this_eta; - $eta = sprintf("ETA: %ds ", $Global::eta); + $Private::eta = 0.9 * $Private::eta + 0.1 * $this_eta; + $eta = sprintf("ETA: %ds ", $Private::eta); } } @@ -2974,15 +2975,15 @@ sub columns { # Get the number of columns of the display # Returns: # number of columns of the screen - if(not $Global::columns) { - $Global::columns = $ENV{'COLUMNS'}; - if(not $Global::columns) { + if(not $Private::columns) { + $Private::columns = $ENV{'COLUMNS'}; + if(not $Private::columns) { my $resize = qx{ resize 2>/dev/null }; - $resize =~ /COLUMNS=(\d+);/ and do { $Global::columns = $1; }; + $resize =~ /COLUMNS=(\d+);/ and do { $Private::columns = $1; }; } - $Global::columns ||= 80; + $Private::columns ||= 80; } - return $Global::columns; + return $Private::columns; } sub start_more_jobs { @@ -3051,12 +3052,11 @@ sub start_job { my ($pid,$out,$err,%out,%err,$outname,$errname,$name); if($Global::grouped) { # To group we create temporary files for STDOUT and STDERR - # Filehandles are global, so to not overwrite the filehandles use a hash with new keys # To avoid the cleanup unlink the files immediately (but keep them open) - $outname = ++$Global::TmpFilename; + $outname = ++$Private::TmpFilename; ($out{$outname},$name) = tempfile(SUFFIX => ".par"); unlink $name; - $errname = ++$Global::TmpFilename; + $errname = ++$Private::TmpFilename; ($err{$errname},$name) = tempfile(SUFFIX => ".par"); unlink $name; @@ -3089,9 +3089,9 @@ sub start_job { $Global::total_started++; debug("$Global::total_running processes. Starting: $command\n"); #print STDERR "LEN".length($command)."\n"; - $Global::job_start_sequence++; + $Private::job_start_sequence++; - if($::opt_a and $Global::job_start_sequence == 1) { + if($::opt_a and $Private::job_start_sequence == 1) { # Give STDIN to the first job if using -a $pid = open3("<&STDIN", ">&STDOUT", ">&STDERR", $command) || die("open3 failed. Report a bug to \n"); @@ -3109,14 +3109,14 @@ sub start_job { or die "Can't dup \$Global::original_stderr: $!"; if($Global::grouped) { - return ("seq" => $Global::job_start_sequence, + return ("seq" => $Private::job_start_sequence, "pid" => $pid, "out" => $out{$outname}, "err" => $err{$errname}, "sshlogin" => $sshlogin, "command" => $command); } else { - return ("seq" => $Global::job_start_sequence, + return ("seq" => $Private::job_start_sequence, "pid" => $pid, "sshlogin" => $sshlogin, "command" => $command); @@ -3244,7 +3244,9 @@ sub sshcommand_of_sshlogin { $serverlogin = $sshlogin; #my $master = "ssh -MTS ".control_path_dir()."/ssh-%r@%h:%p ".$serverlogin; my $master = "ssh -MTS ".control_path_dir()."/ssh-%r@%h:%p ".$serverlogin." sleep 1"; - if(not $Global::control_path{$control_path}++) { + if(not $Private::control_path{$control_path}++) { + # Master is not running for this control_path + # Start it my $pid = fork(); if($pid) { $Global::sshmaster{$pid}++; @@ -3264,10 +3266,10 @@ sub sshcommand_of_sshlogin { sub control_path_dir { # Returns: # path to directory - if(not $Global::control_path_dir) { - $Global::control_path_dir = tempdir("/tmp/parallel-ssh-XXXX", CLEANUP => 1 ); + if(not $Private::control_path_dir) { + $Private::control_path_dir = tempdir("/tmp/parallel-ssh-XXXX", CLEANUP => 1 ); } - return $Global::control_path_dir; + return $Private::control_path_dir; } sub sshtransfer { @@ -3426,9 +3428,9 @@ sub reaper { # Start another job # Returns: N/A do_not_reap(); - $Global::reaperlevel++; + $Private::reaperlevel++; my $stiff; - debug("Reaper called $Global::reaperlevel\n"); + debug("Reaper called $Private::reaperlevel\n"); while (($stiff = waitpid(-1, &WNOHANG)) > 0) { if($Global::sshmaster{$stiff}) { # This is one of the ssh -M: ignore @@ -3444,11 +3446,12 @@ sub reaper { if($Global::keeporder and not $print_now) { $Global::print_later{$Global::running{$stiff}{"seq"}} = $Global::running{$stiff}; - while($Global::print_later{$Global::job_end_sequence}) { - debug("Found job end $Global::job_end_sequence"); - print_job($Global::print_later{$Global::job_end_sequence}); - delete $Global::print_later{$Global::job_end_sequence}; - $Global::job_end_sequence++; + $Private::job_end_sequence ||= 1; + while($Global::print_later{$Private::job_end_sequence}) { + debug("Found job end $Private::job_end_sequence"); + print_job($Global::print_later{$Private::job_end_sequence}); + delete $Global::print_later{$Private::job_end_sequence}; + $Private::job_end_sequence++; } } else { print_job ($Global::running{$stiff}); @@ -3482,8 +3485,8 @@ sub reaper { start_more_jobs(); } reap_if_needed(); - debug("Reaper exit $Global::reaperlevel\n"); - $Global::reaperlevel--; + debug("Reaper exit $Private::reaperlevel\n"); + $Private::reaperlevel--; } # @@ -3604,12 +3607,4 @@ sub my_dump { } # Keep perl -w happy -$main::opt_u = $main::opt_e = $main::opt_c = $main::opt_f = -$main::opt_q = $main::opt_0 = $main::opt_s = $main::opt_v = -$main::opt_g = $main::opt_P = $main::opt_D = $main::opt_m = -$main::opt_X = $main::opt_x = $main::opt_k = $main::opt_d = -$main::opt_P = $main::opt_i = $main::opt_p = $main::opt_a = -$main::opt_version = $main::opt_L = $main::opt_l = -$main::opt_show_limits = $main::opt_n = $main::opt_e = $main::opt_verbose = -$main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder = -$Global::control_path = 0; +$Private::control_path = 0; diff --git a/unittest/actual-results/test25 b/unittest/actual-results/test25 index a45a9668..8b5613cd 100644 --- a/unittest/actual-results/test25 +++ b/unittest/actual-results/test25 @@ -30,3 +30,13 @@ b echo a a cat +### Bug made 4 5 go before 1 2 3 +1 +2 +3 +4 +5 +### Bug made 3 go before 1 2 +1 +2 +3 diff --git a/unittest/tests-to-run/test25.sh b/unittest/tests-to-run/test25.sh index 0cc731a6..bb6e54dd 100644 --- a/unittest/tests-to-run/test25.sh +++ b/unittest/tests-to-run/test25.sh @@ -12,3 +12,8 @@ echo via first cat |parallel -kv cat ::: - - echo via cat |parallel --arg-sep .--- -kv .--- 'cat' 'echo b' echo via cat |parallel -kv ::: 'cat' 'echo b' echo no output |parallel -kv ::: 'echo a' 'cat' + +echo '### Bug made 4 5 go before 1 2 3' +parallel -k ::: "sleep 1; echo 1" "echo 2" "echo 3" "echo 4" "echo 5" +echo '### Bug made 3 go before 1 2' +parallel -kj 1 ::: "sleep 1; echo 1" "echo 2" "echo 3" diff --git a/unittest/wanted-results/test16 b/unittest/wanted-results/test16 index 700b822e..235c8355 100644 --- a/unittest/wanted-results/test16 +++ b/unittest/wanted-results/test16 @@ -63,8 +63,8 @@ rm -- 中国\ \(Zhōngguó\)/abc-中国\ \(Zhōngguó\)-中国\ \(Zhōngguó\) a1.gif 2.gif 3.gif 4.gif 5.gif 6.gifb1 2 3 4 5 6c1 2 3 4 5 6 a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6 ### Test -m with 60000 args -98c94dcab1efedab3f820935d230bc77 - - 12 180011 1286837 +cded9cd15e00550b08e57afc0172caa8 - + 12 180000 1286718 ### Test -X with 60000 args 12de4813eda45d364a51bef697eee299 - 13 120000 1586682 @@ -79,15 +79,15 @@ a1.gifb1c1 abc a2.gifb2c2 abc a3.gifb3c3 abc a4.gifb4c4 abc a5.gifb5c5 abc a6.gifb6c6 abc a7.gifb7c7 abc a8.gifb8c8 abc a9.gifb9c9 abc a10.gifb10c10 abc a11.gifb11c11 abc a12.gifb12c12 abc a13.gifb13c13 abc a14.gifb14c14 abc a15.gifb15c15 abc -a1.gif 2.gif 3.gif 4.gif 5.gif b1 2 3 4 5 6c1 2 3 4 5 6 -a6.gif 7.gif 8.gif 9.gif 10.gif b6 7 8 9 10 11c6 7 8 9 10 11 -a11.gif 12.gif 13.gif 14.gif b11 12 13 14 15c11 12 13 14 15 +a1.gif 2.gif 3.gif 4.gif 5.gif b1 2 3 4 5 c1 2 3 4 5 +a6.gif 7.gif 8.gif 9.gif 10.gif b6 7 8 9 10 c6 7 8 9 10 +a11.gif 12.gif 13.gif 14.gif b11 12 13 14 c11 12 13 14 a15.gif b15 c15 a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6 a7.gifb7c7 a8.gifb8c8 a9.gifb9c9 a10.gifb10c10 a11.gifb11c11 a12.gifb12c12 a13.gifb13c13 a14.gifb14c14 a15.gifb15c15 -a1.gif 2.gif 3.gif 4.gif 5.gif 6.gif 7.gifb1 2 3 4 5 6 7 8c1 2 3 4 5 6 7 8 -a8.gif 9.gif 10.gif 11.gif 12.gif 13.gifb8 9 10 11 12 13 14c8 9 10 11 12 13 14 +a1.gif 2.gif 3.gif 4.gif 5.gif 6.gif 7.gifb1 2 3 4 5 6 7c1 2 3 4 5 6 7 +a8.gif 9.gif 10.gif 11.gif 12.gif 13.gifb8 9 10 11 12 13c8 9 10 11 12 13 a14.gif 15.gifb14 15c14 15 ### Test -I with shell meta chars 9 diff --git a/unittest/wanted-results/test25 b/unittest/wanted-results/test25 index a45a9668..8b5613cd 100644 --- a/unittest/wanted-results/test25 +++ b/unittest/wanted-results/test25 @@ -30,3 +30,13 @@ b echo a a cat +### Bug made 4 5 go before 1 2 3 +1 +2 +3 +4 +5 +### Bug made 3 go before 1 2 +1 +2 +3