From 1229288418b0f333bd3a0894800239371fd6c9ad Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 5 Feb 2017 00:27:10 +0100 Subject: [PATCH 1/5] Fixed bug #50081: --keep-order --round-robin should give predictable results. --- src/parallel | 34 +++++++++++++++------ src/parallel.pod | 3 ++ testsuite/tests-to-run/parallel-local-1s.sh | 1 + testsuite/tests-to-run/parallel-local-3s.sh | 25 +++++++++++++++ testsuite/wanted-results/parallel-local-3s | 2 ++ 5 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/parallel b/src/parallel index d052174d..b7639e28 100755 --- a/src/parallel +++ b/src/parallel @@ -597,24 +597,38 @@ sub nindex { push @robin_queue, (sort { $a->seq() <=> $b->seq() } values %Global::running); } - do { - $written = 0; + if($opt::keeporder) { for my $job (@robin_queue) { if($job->block_length() > 0) { $written += $job->non_blocking_write(); } else { - $job->set_block($header_ref,$buffer_ref, - $endpos,$recstart,$recend); + $job->set_block($header_ref,$buffer_ref,$endpos,$recstart,$recend); $block_passed = 1; $job->set_virgin(0); $written += $job->non_blocking_write(); last; } } - if($written) { - $sleep = $sleep/1.5+0.001; - } - } while($written and not $block_passed); + } else { + do { + $written = 0; + for my $job (@robin_queue) { + if($job->block_length() > 0) { + $written += $job->non_blocking_write(); + } else { + $job->set_block($header_ref,$buffer_ref, + $endpos,$recstart,$recend); + $block_passed = 1; + $job->set_virgin(0); + $written += $job->non_blocking_write(); + last; + } + } + if($written) { + $sleep = $sleep/1.5+0.001; + } + } while($written and not $block_passed); + } $sleep = ::reap_usleep($sleep); } return $written; @@ -961,6 +975,7 @@ sub options_hash { "cat" => \$opt::cat, "fifo" => \$opt::fifo, "pipepart|pipe-part" => \$opt::pipepart, + "tee" => \$opt::tee, "hgrp|hostgrp|hostgroup|hostgroups" => \$opt::hostgroups, ); } @@ -7876,7 +7891,8 @@ sub start { if($opt::pipe) { my ($stdin_fh) = ::gensym(); $pid = open3_setpgrp($stdin_fh,$stdout_fh,$stderr_fh,$command); - if($opt::roundrobin) { + if($opt::roundrobin and not $opt::keeporder) { + # --keep-order will make sure the order will be reproducible ::set_fh_non_blocking($stdin_fh); } $job->set_fh(0,"w",$stdin_fh); diff --git a/src/parallel.pod b/src/parallel.pod index 8b652660..f7035d32 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -988,6 +988,9 @@ to see the difference: If used with B<--onall> or B<--nonall> the output will grouped by sshlogin in sorted order. +If used with B<--pipe --roundrobin> and the same input, the jobslots +will get the same blocks in the same order in every run. + =item B<-L> I diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 2063802a..c1274e9d 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -196,6 +196,7 @@ par_result_replace() { rm -rf /tmp/par_*_49983-* } + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index dc1d0ae8..cd08097b 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -79,6 +79,31 @@ par_pipepart_block() { rm /run/shm/parallel$$ } +par_keeporder_roundrobin() { + echo 'bug #50081: --keep-order --round-robin should give predictable results' + + export PARALLEL="-j13 --block 1m --pipe --roundrobin" + random500m() { + < /dev/zero openssl enc -aes-128-ctr -K 1234 -iv 1234 2>/dev/null | + head -c 500m; + } + a=$(random500m | parallel -k 'echo {#} $(md5sum)' | sort) + b=$(random500m | parallel -k 'echo {#} $(md5sum)' | sort) + c=$(random500m | parallel 'echo {#} $(md5sum)' | sort) + if [ "$a" == "$b" ] ; then + # Good: -k should be == -k + if [ "$a" == "$c" ] ; then + # Bad: without -k the command should give different output + echo 'Broken: a == c' + printf "$a\n$b\n$c\n" + else + echo OK + fi + else + echo 'Broken: a <> b' + printf "$a\n$b\n$c\n" + fi +} export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index c1b41843..6483d5e6 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -4,6 +4,8 @@ par_children_receive_sig Got TERM par_children_receive_sig Got TERM par_children_receive_sig Got INT par_children_receive_sig Got TERM +par_keeporder_roundrobin bug #50081: --keep-order --round-robin should give predictable results +par_keeporder_roundrobin OK par_kill_int_twice ### Are children killed if GNU Parallel receives INT twice? There should be no sleep at the end par_kill_int_twice bash-+-perl---bash---sleep par_kill_int_twice `-pstree From c028fa0ad749d2b9f1871844d39976802b6d8dde Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 5 Feb 2017 02:46:29 +0100 Subject: [PATCH 2/5] Fixed bug #45479: --pipe/--pipepart --tee. --- src/parallel | 81 ++++++++++++++++--- src/parallel.pod | 21 +++-- testsuite/tests-to-run/parallel-local-0.3s.sh | 5 ++ testsuite/wanted-results/parallel-local-0.3s | 20 +++++ 4 files changed, 108 insertions(+), 19 deletions(-) diff --git a/src/parallel b/src/parallel index b7639e28..0ecb240a 100755 --- a/src/parallel +++ b/src/parallel @@ -198,14 +198,6 @@ drain_job_queue(); ::debug("init", "Done draining\n"); reaper(); ::debug("init", "Done reaping\n"); -if($opt::pipe and @opt::a) { - for my $job (@Global::tee_jobs) { - ::rm($job->fh(2,"name")); - $job->set_fh(2,"name",""); - $job->print(); - ::rm($job->fh(1,"name")); - } -} ::debug("init", "Cleaning\n"); if($Global::semaphore) { $sem->release(); @@ -376,6 +368,15 @@ sub spreadstdin { # %Global::running # Returns: N/A + if($opt::tee) { + # Spawn all jobs + # read a record + # Write record to all jobs + if(not $Global::JobQueue->empty()) { + ::error("--tee requres --jobs to be higher. Try --jobs 0."); + } + } + my $buf = ""; my ($recstart,$recend) = recstartrecend(); my $recendrecstart = $recend.$recstart; @@ -635,6 +636,50 @@ sub nindex { } } +{ + my $sleep = 1; + + sub tee_write { + # Write the block to all jobs + # + # Input: + # $header_ref = ref to $header string + # $block_ref = ref to $block to be written + # $recstart = record start string + # $recend = record end string + # $endpos = end position of $block + # Uses: + # %Global::running + # Returns: + # $written = amount of bytes written + my ($header_ref,$buffer_ref,$recstart,$recend,$endpos) = @_; + my $written = 0; + my $done = 0; + my %done; + while(not $done) { + $done = 1; + for my $job (values %Global::running) { + if(not $done{$job}) { + $done = 0; + if($job->block_length() > 0) { + # Flush old block + $written += $job->non_blocking_write(); + } else { + # Give a copy of the new block + $job->set_block($header_ref,$buffer_ref,$endpos,$recstart,$recend); + $job->set_virgin(0); + $written += $job->non_blocking_write(); + # Mark this job as done + $done{$job} = 1; + } + } + } + $sleep = ::reap_usleep($sleep); + } + return $written; + } +} + sub index64 { # Do index on strings > 2GB. # index in Perl < v5.22 does not work for > 2GB @@ -744,8 +789,13 @@ sub write_record_to_pipe { if($endpos == 0) { return 0; } if(vec($Global::job_already_run,$chunk_number,1)) { return 1; } if($opt::roundrobin) { + # Write the block to one of the already running jobs return round_robin_write($header_ref,$buffer_ref,$recstart,$recend,$endpos); } + if($opt::tee) { + # Write the block to all jobs + return tee_write($header_ref,$buffer_ref,$recstart,$recend,$endpos); + } # If no virgin found, backoff my $sleep = 0.0001; # 0.01 ms - better performance on highend while(not @Global::virgin_jobs) { @@ -1152,6 +1202,11 @@ sub parse_options { map { (0..9,"a".."z","A".."Z")[rand(62)] } (1..50); push @ARGV, $Global::arg_sep, "\0"; } + if(defined $opt::tee) { + if(not defined $opt::jobs) { + $opt::jobs = 0; + } + } if(defined $opt::tty) { # Defaults for --tty: -j1 -u # Can be overridden with -jXXX -g @@ -5377,7 +5432,7 @@ sub compute_number_of_processes { sub get_args_or_jobs { # Get an arg or a job (depending on mode) - if($Global::semaphore or $opt::pipe) { + if($Global::semaphore or ($opt::pipe and not $opt::tee)) { # Skip: No need to get args return 1; } elsif(defined $opt::retries and $count_jobs_already_read) { @@ -6617,9 +6672,7 @@ sub openoutputfiles { } elsif(not $opt::ungroup) { # To group we create temporary files for STDOUT and STDERR # To avoid the cleanup unlink the files immediately (but keep them open) - if(@Global::tee_jobs) { - # files must be removed when the tee is done - } elsif($opt::files) { + if($opt::files) { ($outfhw, $outname) = ::tmpfile(SUFFIX => ".par"); ($errfhw, $errname) = ::tmpfile(SUFFIX => ".par"); # --files => only remove stderr @@ -10223,6 +10276,10 @@ sub total_jobs { # shorthand for $job->slot(); $job->slot(); } + sub seq { + # shorthand for $job->seq(); + $job->seq(); + } sub replace { # Calculates the corresponding value for a given perl expression diff --git a/src/parallel.pod b/src/parallel.pod index f7035d32..baf3cef7 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -273,11 +273,11 @@ perl quote a string number of jobs in total -=item Z<> B<$job->>B +=item Z<> B slot number of job -=item Z<> B<$job->>B +=item Z<> B sequence number of job @@ -1770,13 +1770,13 @@ of GNU B's internal functions and data structures. Here are a few examples: Is the job sequence even or odd? - --rpl '{odd} $_ = $job->seq() % 2 ? "odd" : "even"' + --rpl '{odd} $_ = seq() % 2 ? "odd" : "even"' Pad job sequence with leading zeros to get equal width - --rpl '{0#} $f = "%0".int(1+log(total_jobs())/log(10))."d"; $_=sprintf($f,$job->seq())' + --rpl '{0#} $f = "%0".int(1+log(total_jobs())/log(10))."d"; $_=sprintf($f,seq())' Job sequence counting from 0 - --rpl '{#0} $_ = $job->seq() - 1' + --rpl '{#0} $_ = seq() - 1' Job slot counting from 2 - --rpl '{%1} $_ = $job->slot() + 1' + --rpl '{%1} $_ = slot() + 1' See also: B<{= perl expression =}> B<--parens> @@ -2168,6 +2168,13 @@ B<{}>. B<--tagstring> is ignored when using B<-u>, B<--onall>, and B<--nonall>. +=item B<--tee> (alpha testing) + +Pipe all data to all jobs. Used with B<--pipe> and B<:::>. + + seq 1000 | parallel --pipe --tee -v wc {} ::: -w -l -c + + =item B<--termseq> I Termination sequence. When a job is killed due to B<--timeout>, @@ -4141,7 +4148,7 @@ Killed by Ctrl-C, timeout, not enough memory or similar. =item Z<>-2 (In joblog and SQL table) -$job->skip() was called in B<{= =}>. +skip() was called in B<{= =}>. =item Z<>-1000 (In SQL table) diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 6b29dc09..1f9b02f7 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -672,6 +672,11 @@ par_retries_replacement_string() { rm $tmp } +par_tee() { + export PARALLEL='-k --tee --pipe --tag' + seq 1000000 | parallel 'echo {%};LANG=C wc' ::: {1..5} ::: {a..b} +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index b6115bbe..0385f4f9 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1630,3 +1630,23 @@ par_retries_replacement_string 22 par_retries_replacement_string 33 par_retries_replacement_string 33 par_retries_replacement_string 33 +par_tee 1 a 1 +par_tee 1 a 1000000 1000000 6888896 +par_tee 1 b 2 +par_tee 1 b 1000000 1000000 6888896 +par_tee 2 a 3 +par_tee 2 a 1000000 1000000 6888896 +par_tee 2 b 4 +par_tee 2 b 1000000 1000000 6888896 +par_tee 3 a 5 +par_tee 3 a 1000000 1000000 6888896 +par_tee 3 b 6 +par_tee 3 b 1000000 1000000 6888896 +par_tee 4 a 7 +par_tee 4 a 1000000 1000000 6888896 +par_tee 4 b 8 +par_tee 4 b 1000000 1000000 6888896 +par_tee 5 a 9 +par_tee 5 a 1000000 1000000 6888896 +par_tee 5 b 10 +par_tee 5 b 1000000 1000000 6888896 From 70006774e875640ae5bdbb3a522da3ec7ccdc6b4 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 5 Feb 2017 03:24:23 +0100 Subject: [PATCH 3/5] Fixed bug #50228: --pipe --tagstring broken. --- src/parallel | 7 ++++++- testsuite/tests-to-run/parallel-local-0.3s.sh | 5 +++++ testsuite/wanted-results/parallel-local-0.3s | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/parallel b/src/parallel index 0ecb240a..ca9f5812 100755 --- a/src/parallel +++ b/src/parallel @@ -96,7 +96,10 @@ sub set_input_source_header { for my $s (split /$delimiter/o, $line) { ::debug("init", "Colname: '$s'"); # Replace {colname} with {2} - for(@command,@Global::ret_files,@Global::transfer_files) { + for(@command,@Global::ret_files,@Global::transfer_files, + $opt::tagstring, $opt::workdir, $opt::results, $opt::retries) { + # Skip if undefined + $_ or next; s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g; # {=header1 ... =} => {=1 ... =} s:$left $s (.*?) $right:$l$id$1$r:gx; @@ -7930,6 +7933,8 @@ sub start { $command = "true"; } $job->openoutputfiles(); + # Call slot to store the slot value + $job->slot(); my($stdout_fh,$stderr_fh) = ($job->fh(1,"w"),$job->fh(2,"w")); if($opt::ungroup or $opt::sqlworker) { print_dryrun_and_verbose($stdout_fh,$job,$command); diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 1f9b02f7..58a82571 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -677,6 +677,11 @@ par_tee() { seq 1000000 | parallel 'echo {%};LANG=C wc' ::: {1..5} ::: {a..b} } +par_tagstring_pipe() { + echo 'bug #50228: --pipe --tagstring broken' + seq 3000 | parallel -j4 --pipe -N1000 -k --tagstring {%} LANG=C wc +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 0385f4f9..34ef6b5b 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1630,6 +1630,10 @@ par_retries_replacement_string 22 par_retries_replacement_string 33 par_retries_replacement_string 33 par_retries_replacement_string 33 +par_tagstring_pipe bug #50228: --pipe --tagstring broken +par_tagstring_pipe 1 1000 1000 3893 +par_tagstring_pipe 2 1000 1000 5000 +par_tagstring_pipe 3 1000 1000 5000 par_tee 1 a 1 par_tee 1 a 1000000 1000000 6888896 par_tee 1 b 2 From a2358aebe43f009ab8b89fc79092d0ef3cf4a2f3 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 5 Feb 2017 23:53:28 +0100 Subject: [PATCH 4/5] Faster version bug #45479: --pipe/--pipepart --tee. --- src/parallel | 371 +++++++++--------- src/parallel.pod | 10 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 1 + testsuite/tests-to-run/parallel-local-10s.sh | 25 ++ testsuite/tests-to-run/parallel-tutorial.sh | 2 + testsuite/wanted-results/parallel-local-0.3s | 10 + testsuite/wanted-results/parallel-local-10s | 6 + testsuite/wanted-results/parallel-tutorial | 2 - 8 files changed, 247 insertions(+), 180 deletions(-) diff --git a/src/parallel b/src/parallel index ca9f5812..a493e8c0 100755 --- a/src/parallel +++ b/src/parallel @@ -51,8 +51,17 @@ my @command = @ARGV; my @input_source_fh; if($opt::pipepart) { - # -a is used for data - not for command line args - @input_source_fh = map { open_or_exit($_) } "/dev/null"; + if($opt::tee) { + @input_source_fh = map { open_or_exit($_) } @opt::a; + # Remove the first: It will be the file piped. + shift @input_source_fh; + if(not @input_source_fh and not $opt::pipe) { + @input_source_fh = (*STDIN); + } + } else { + # -a is used for data - not for command line args + @input_source_fh = map { open_or_exit($_) } "/dev/null"; + } } else { @input_source_fh = map { open_or_exit($_) } @opt::a; if(not @input_source_fh and not $opt::pipe) { @@ -77,6 +86,65 @@ if($opt::skip_first_line) { set_input_source_header(); +if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) { + # Parallel check all hosts are up. Remove hosts that are down + filter_hosts(); +} + +if($opt::nonall or $opt::onall) { + onall(\@input_source_fh,@command); + wait_and_exit(min(undef_as_zero($Global::exitstatus),254)); +} + +# TODO --transfer foo/./bar --cleanup +# multiple --transfer and --basefile with different /./ + +$Global::JobQueue = JobQueue->new( + \@command,\@input_source_fh,$Global::ContextReplace, + $number_of_args,\@Global::transfer_files,\@Global::ret_files); + +if($opt::pipepart) { + pipepart_setup(); +} elsif($opt::pipe and $opt::tee) { + pipe_tee_setup(); +} + +if($opt::eta or $opt::bar or $opt::shuf or $Global::halt_pct) { + # Count the number of jobs or shuffle all jobs + # before starting any. + # Must be done after ungetting any --pipepart jobs. + $Global::JobQueue->total_jobs(); +} +# Compute $Global::max_jobs_running +# Must be done after ungetting any --pipepart jobs. +max_jobs_running(); + +init_run_jobs(); +my $sem; +if($Global::semaphore) { + $sem = acquire_semaphore(); +} +$SIG{TERM} = \&start_no_new_jobs; +start_more_jobs(); +if($opt::tee) { + # All jobs must be running in parallel for --tee + $Global::start_no_new_jobs = 1; +} elsif($opt::pipe and not $opt::pipepart) { + spreadstdin(); +} +::debug("init", "Start draining\n"); +drain_job_queue(); +::debug("init", "Done draining\n"); +reaper(); +::debug("init", "Done reaping\n"); +::debug("init", "Cleaning\n"); +if($Global::semaphore) { + $sem->release(); +} +cleanup(); +::debug("init", "Halt\n"); +halt(); + sub set_input_source_header { if($opt::header and not $opt::pipe) { # split with colsep or \t @@ -117,114 +185,102 @@ sub set_input_source_header { } } -if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) { - # Parallel check all hosts are up. Remove hosts that are down - filter_hosts(); -} - -if($opt::nonall or $opt::onall) { - onall(\@input_source_fh,@command); - wait_and_exit(min(undef_as_zero($Global::exitstatus),254)); -} - -# TODO --transfer foo/./bar --cleanup -# multiple --transfer and --basefile with different /./ - -$Global::JobQueue = JobQueue->new( - \@command,\@input_source_fh,$Global::ContextReplace, - $number_of_args,\@Global::transfer_files,\@Global::ret_files); - -if($opt::pipepart) { - if(not $opt::blocksize) { - # --blocksize with 10 jobs per jobslot - $opt::blocksize = -10; - } - if($opt::roundrobin) { - # --blocksize with 1 job per jobslot - $opt::blocksize = -1; - } - if($opt::blocksize < 0) { - my $size = 0; - # Compute size of -a - for(@opt::a) { - if(-f $_) { - $size += -s $_; - } elsif(-b $_) { - $size += size_of_block_dev($_); - } else { - ::error("$_ is neither a file nor a block device"); - wait_and_exit(255); - } - } +sub max_jobs_running { + if(not $Global::max_jobs_running) { # Compute $Global::max_jobs_running - $Global::dummy_jobs = 1; for my $sshlogin (values %Global::host) { $sshlogin->max_jobs_running(); } - $Global::max_jobs_running or - ::die_bug("Global::max_jobs_running not set"); - # Run in total $job_slots*(- $blocksize) jobs - # Set --blocksize = size / no of proc / (- $blocksize) - $Global::blocksize = 1 + - int($size / $Global::max_jobs_running / -$opt::blocksize); } - @Global::cat_partials = map { pipe_part_files($_) } @opt::a; - # Unget the empty arg as many times as there are parts - $Global::JobQueue->{'commandlinequeue'}{'arg_queue'}->unget( - map { [Arg->new("\0")] } @Global::cat_partials - ); -} -if($opt::eta or $opt::bar or $opt::shuf or $Global::halt_pct) { - # Count the number of jobs or shuffle all jobs - # before starting any. - # Must be done after ungetting any --pipepart jobs. - $Global::JobQueue->total_jobs(); -} -# Compute $Global::max_jobs_running -# Must be done after ungetting any --pipepart jobs. -for my $sshlogin (values %Global::host) { - $sshlogin->max_jobs_running(); + return $Global::max_jobs_running; } -init_run_jobs(); -my $sem; -if($Global::semaphore) { - $sem = acquire_semaphore(); -} -$SIG{TERM} = \&start_no_new_jobs; -start_more_jobs(); -if($opt::pipe and not $opt::pipepart) { - spreadstdin(); -} -::debug("init", "Start draining\n"); -drain_job_queue(); -::debug("init", "Done draining\n"); -reaper(); -::debug("init", "Done reaping\n"); -::debug("init", "Cleaning\n"); -if($Global::semaphore) { - $sem->release(); -} -cleanup(); -::debug("init", "Halt\n"); -if($opt::halt and $Global::halt_when ne "never") { - if(not defined $Global::halt_exitstatus) { - if($Global::halt_pct) { - $Global::halt_exitstatus = - ::ceil($Global::total_failed / $Global::total_started * 100); - } elsif($Global::halt_count) { - $Global::halt_exitstatus = - ::min(undef_as_zero($Global::total_failed),101); +sub halt { + if($opt::halt and $Global::halt_when ne "never") { + if(not defined $Global::halt_exitstatus) { + if($Global::halt_pct) { + $Global::halt_exitstatus = + ::ceil($Global::total_failed / $Global::total_started * 100); + } elsif($Global::halt_count) { + $Global::halt_exitstatus = + ::min(undef_as_zero($Global::total_failed),101); + } } + wait_and_exit($Global::halt_exitstatus); + } else { + wait_and_exit(min(undef_as_zero($Global::exitstatus),101)); } - wait_and_exit($Global::halt_exitstatus); -} else { - wait_and_exit(min(undef_as_zero($Global::exitstatus),101)); } - sub __PIPE_MODE__ {} +sub pipepart_setup { + if($opt::tee) { + # Prepend each command with + # cat file + my $cat_string = "cat ".::shell_quote_scalar($opt::a[0]); + for(1..max_jobs_running()) { + push @Global::cat_prepends, $cat_string; + } + } else { + if(not $opt::blocksize) { + # --blocksize with 10 jobs per jobslot + $opt::blocksize = -10; + } + if($opt::roundrobin) { + # --blocksize with 1 job per jobslot + $opt::blocksize = -1; + } + if($opt::blocksize < 0) { + my $size = 0; + # Compute size of -a + for(@opt::a) { + if(-f $_) { + $size += -s $_; + } elsif(-b $_) { + $size += size_of_block_dev($_); + } else { + ::error("$_ is neither a file nor a block device"); + wait_and_exit(255); + } + } + # Run in total $job_slots*(- $blocksize) jobs + # Set --blocksize = size / no of proc / (- $blocksize) + $Global::dummy_jobs = 1; + $Global::blocksize = 1 + + int($size / max_jobs_running() / -$opt::blocksize); + } + @Global::cat_prepends = map { pipe_part_files($_) } @opt::a; + # Unget the empty arg as many times as there are parts + $Global::JobQueue->{'commandlinequeue'}{'arg_queue'}->unget( + map { [Arg->new("\0")] } @Global::cat_prepends + ); + } +} + +sub pipe_tee_setup { + # mkfifo t1..5 + my @fifos; + for(1..max_jobs_running()) { + push @fifos, tmpfifo(); + } + + # cat foo | tee t1 t2 t3 t4 t5 > /dev/null + if(not fork()){ + # Let tee inheirit our stdin + # and redirect stdout to null + open STDOUT, ">","/dev/null"; + exec "tee",@fifos; + } + + # cat t1 | grep 1 + # cat t2 | grep 2 + # cat t3 | grep 3 + # cat t4 | grep 4 + # cat t5 | grep 5 + # Remove the tmpfifo as soon as it is open + @Global::cat_prepends = map { "(rm $_;cat) < $_" } @fifos; +} sub pipe_part_files { # Input: @@ -240,12 +296,12 @@ sub pipe_part_files { my $header = find_header(\$buf,open_or_exit($file)); # find positions my @pos = find_split_positions($file,$Global::blocksize,length $header); - # Make @cat_partials - my @cat_partials = (); + # Make @cat_prepends + my @cat_prepends = (); for(my $i=0; $i<$#pos; $i++) { - push @cat_partials, cat_partial($file, 0, length($header), $pos[$i], $pos[$i+1]); + push @cat_prepends, cat_partial($file, 0, length($header), $pos[$i], $pos[$i+1]); } - return @cat_partials; + return @cat_prepends; } sub find_header { @@ -336,22 +392,26 @@ sub cat_partial { # $file = the file to read # ($start, $end, [$start2, $end2, ...]) = start byte, end byte # Returns: - # Efficient perl command to copy $start..$end, $start2..$end2, ... to stdout + # Efficient command to copy $start..$end, $start2..$end2, ... to stdout my($file, @start_end) = @_; my($start, $i); # Convert start_end to start_len - my @start_len = map { if(++$i % 2) { $start = $_; } else { $_-$start } } @start_end; - my $script = spacefree(0, - q{ - while(@ARGV) { - sysseek(STDIN,shift,0) || die; - $left = shift; - while($read = sysread(STDIN,$buf, $left > 131072 ? 131072 : $left)){ - $left -= $read; - syswrite(STDOUT,$buf); - } - } - }); + my @start_len = map { + if(++$i % 2) { $start = $_; } else { $_-$start } + } @start_end; + my $script = spacefree + (0, + q{ + while(@ARGV) { + sysseek(STDIN,shift,0) || die; + $left = shift; + while($read = + sysread(STDIN,$buf, $left > 131072 ? 131072 : $left)){ + $left -= $read; + syswrite(STDOUT,$buf); + } + } + }); return "<". shell_quote_scalar($file) . " perl -e '$script' @start_len"; } @@ -639,50 +699,6 @@ sub nindex { } } -{ - my $sleep = 1; - - sub tee_write { - # Write the block to all jobs - # - # Input: - # $header_ref = ref to $header string - # $block_ref = ref to $block to be written - # $recstart = record start string - # $recend = record end string - # $endpos = end position of $block - # Uses: - # %Global::running - # Returns: - # $written = amount of bytes written - my ($header_ref,$buffer_ref,$recstart,$recend,$endpos) = @_; - my $written = 0; - my $done = 0; - my %done; - while(not $done) { - $done = 1; - for my $job (values %Global::running) { - if(not $done{$job}) { - $done = 0; - if($job->block_length() > 0) { - # Flush old block - $written += $job->non_blocking_write(); - } else { - # Give a copy of the new block - $job->set_block($header_ref,$buffer_ref,$endpos,$recstart,$recend); - $job->set_virgin(0); - $written += $job->non_blocking_write(); - # Mark this job as done - $done{$job} = 1; - } - } - } - $sleep = ::reap_usleep($sleep); - } - return $written; - } -} - sub index64 { # Do index on strings > 2GB. # index in Perl < v5.22 does not work for > 2GB @@ -795,10 +811,6 @@ sub write_record_to_pipe { # Write the block to one of the already running jobs return round_robin_write($header_ref,$buffer_ref,$recstart,$recend,$endpos); } - if($opt::tee) { - # Write the block to all jobs - return tee_write($header_ref,$buffer_ref,$recstart,$recend,$endpos); - } # If no virgin found, backoff my $sleep = 0.0001; # 0.01 ms - better performance on highend while(not @Global::virgin_jobs) { @@ -1349,7 +1361,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20170123; + $Global::version = 20170202; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -4119,17 +4131,17 @@ sub tmpname { do { $tmpname = $ENV{'TMPDIR'}."/".$name. join"", map { (0..9,"a".."z","A".."Z")[rand(62)] } (1..5); - } while($Global::unlink{$tmpname}++ or -e $tmpname); + } while(-e $tmpname or $Global::unlink{$tmpname}++); return $tmpname; } -#sub tmpfifo { -# # Find an unused name and mkfifo on it -# use POSIX qw(mkfifo); -# my $tmpfifo = tmpname("fif",@_); -# mkfifo($tmpfifo,0600); -# return $tmpfifo; -#} +sub tmpfifo { + # Find an unused name and mkfifo on it + use POSIX qw(mkfifo); + my $tmpfifo = tmpname("fif",@_); + mkfifo($tmpfifo,0600); + return $tmpfifo; +} sub rm { # Remove file and remove it from %Global::unlink @@ -7129,7 +7141,8 @@ sub wrapped { # * --cat # * --fifo # * --sshlogin - # * --pipepart (@Global::cat_partials) + # * --pipepart (@Global::cat_prepends) + # * --tee (@Global::cat_prepends) # * --pipe # * --tmux # The ordering of the wrapping is important: @@ -7141,7 +7154,7 @@ sub wrapped { # $Global::shell # $opt::cat # $opt::fifo - # @Global::cat_partials + # @Global::cat_prepends # $opt::pipe # $opt::tmux # Returns: @@ -7201,15 +7214,21 @@ sub wrapped { } # Wrap with ssh + tranferring of files $command = $self->sshlogin_wrap($command); - if(@Global::cat_partials) { - # Prepend: + if(@Global::cat_prepends) { + # --pipepart: prepend: # < /tmp/foo perl -e 'while(@ARGV) { # sysseek(STDIN,shift,0) || die; $left = shift; # while($read = sysread(STDIN,$buf, ($left > 131072 ? 131072 : $left))){ # $left -= $read; syswrite(STDOUT,$buf); # } # }' 0 0 0 11 | - $command = (shift @Global::cat_partials). " | ($command)"; + # + # --pipepart --tee: prepend: + # cat dash-a-file | + # + # --pipe --tee: prepend: + # cat fifo | + $command = (shift @Global::cat_prepends). " | ($command)"; } elsif($opt::pipe) { # Wrap with EOF-detector to avoid starting $command if EOF. $command = empty_input_wrapper($command); diff --git a/src/parallel.pod b/src/parallel.pod index baf3cef7..8753b2a4 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1772,7 +1772,7 @@ Here are a few examples: Is the job sequence even or odd? --rpl '{odd} $_ = seq() % 2 ? "odd" : "even"' Pad job sequence with leading zeros to get equal width - --rpl '{0#} $f = "%0".int(1+log(total_jobs())/log(10))."d"; $_=sprintf($f,seq())' + --rpl '{0#} $f=1+int("".(log(total_jobs())/log(10))); $_=sprintf("%0${f}d",seq())' Job sequence counting from 0 --rpl '{#0} $_ = seq() - 1' Job slot counting from 2 @@ -2170,10 +2170,16 @@ B<--tagstring> is ignored when using B<-u>, B<--onall>, and B<--nonall>. =item B<--tee> (alpha testing) -Pipe all data to all jobs. Used with B<--pipe> and B<:::>. +Pipe all data to all jobs. Used with B<--pipe>/B<--pipepart> and +B<:::>. seq 1000 | parallel --pipe --tee -v wc {} ::: -w -l -c +How many numbers in 1..1000 contain 0..9, and how many bytes do they +fill: + + seq 1000 | parallel --pipe --tee --tag 'grep {1} | wc {2}' ::: {0..9} ::: -l -c + =item B<--termseq> I diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 58a82571..f9f7b8c8 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -675,6 +675,7 @@ par_retries_replacement_string() { par_tee() { export PARALLEL='-k --tee --pipe --tag' seq 1000000 | parallel 'echo {%};LANG=C wc' ::: {1..5} ::: {a..b} + seq 300000 | parallel 'grep {1} | LANG=C wc {2}' ::: {1..5} ::: -l -c } par_tagstring_pipe() { diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index e06a4120..c3950834 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -239,6 +239,31 @@ par_tmux_fg() { stdout parallel --tmux --fg sleep ::: 3 | perl -pe 's/.tmp\S+/tmp/' } +par_pipe_tee() { + echo 'bug #45479: --pipe/--pipepart --tee' + echo '--pipe --tee' + + random1G() { + < /dev/zero openssl enc -aes-128-ctr -K 1234 -iv 1234 2>/dev/null | + head -c 1G; + } + random1G | parallel --pipe --tee cat ::: {1..3} | LANG=C wc -c +} + +par_pipepart_tee() { + echo 'bug #45479: --pipe/--pipepart --tee' + echo '--pipepart --tee' + + random1G() { + < /dev/zero openssl enc -aes-128-ctr -K 1234 -iv 1234 2>/dev/null | + head -c 1G; + } + tmp=$(mktemp) + random1G >$tmp + parallel --pipepart --tee -a $tmp cat ::: {1..3} | LANG=C wc -c + rm $tmp +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel --joblog /tmp/jl-`basename $0` -j10 --tag -k '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index 4e64998f..db2879be 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -74,6 +74,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ # Timings are often off s/^(\d)$/9/; s/^(\d\d)$/99/; + # Sometime these vars are not present + s/^PAM_KWALLET5*_LOGIN$//; # Fails often due to race s/cat: input_file: No such file or directory\n//; s{rsync: link_stat ".*/home/parallel/input_file.out" .*\n}{}; diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 34ef6b5b..92334e7d 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1654,3 +1654,13 @@ par_tee 5 a 9 par_tee 5 a 1000000 1000000 6888896 par_tee 5 b 10 par_tee 5 b 1000000 1000000 6888896 +par_tee 1 -l 181902 +par_tee 1 -c 1228633 +par_tee 2 -l 181902 +par_tee 2 -c 1228633 +par_tee 3 -l 122854 +par_tee 3 -c 815297 +par_tee 4 -l 122853 +par_tee 4 -c 815290 +par_tee 5 -l 122853 +par_tee 5 -c 815290 diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 19cdafdc..c36050a3 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -492,8 +492,14 @@ par_memleak should give 1 == true par_memleak Memory use should not depend very much on the total number of jobs run\n par_memleak Test if memory consumption(300 jobs) < memory consumption(30 jobs) * 110% par_memleak 1 +par_pipe_tee bug #45479: --pipe/--pipepart --tee +par_pipe_tee --pipe --tee +par_pipe_tee 3221225472 par_pipepart_spawn ### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922 par_pipepart_spawn 1:local / 8 / 999 +par_pipepart_tee bug #45479: --pipe/--pipepart --tee +par_pipepart_tee --pipepart --tee +par_pipepart_tee 3221225472 par_print_before_halt_on_error ### What is printed before the jobs are killed par_print_before_halt_on_error -2 exit code 0 par_print_before_halt_on_error -2 0 0 diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 4153579f..91cc586d 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -852,8 +852,6 @@ MFLAGS MOZ_NO_REMOTE ORACLE_HOME ORACLE_SID -PAM_KWALLET5_LOGIN -PAM_KWALLET_LOGIN PARALLEL PARALLEL_PID PARALLEL_SEQ From bcdea235697e9c6fc14932f7f24860a37c02ffbd Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 8 Feb 2017 00:54:45 +0100 Subject: [PATCH 5/5] env_parallel.{ash,dash}: added. Only aliases and variables are supported. --- Makefile.am | 56 ++++++++++++++++ Makefile.in | 56 ++++++++++++++++ NEWS | 5 ++ README | 12 ++-- configure | 20 +++--- configure.ac | 2 +- src/Makefile.am | 20 +++--- src/Makefile.in | 20 +++--- src/env_parallel | 49 ++++++++++---- src/env_parallel.ash | 145 +++++++++++++++++++++++++++++++++++++++++ src/env_parallel.dash | 145 +++++++++++++++++++++++++++++++++++++++++ src/env_parallel.pod | 89 +++++++++++++++++++++++++ src/niceload | 2 +- src/parallel | 2 +- src/parallel.pod | 5 ++ src/sql | 2 +- testsuite/REQUIREMENTS | 4 +- 17 files changed, 580 insertions(+), 54 deletions(-) create mode 100644 src/env_parallel.ash create mode 100755 src/env_parallel.dash diff --git a/Makefile.am b/Makefile.am index 8bef00a4..0dd1fc43 100644 --- a/Makefile.am +++ b/Makefile.am @@ -59,4 +59,60 @@ upload: pushd; \ sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD) +alphaupload: +# Copy of upload + eval `gpg-agent --daemon` +# make sure $YYYYMMDD is set + echo $(YYYYMMDD) | grep 20 + cp parallel-$(YYYYMMDD).tar.bz2 parallel-latest.tar.bz2 + cp doc/parallel.directive parallel-$(YYYYMMDD).tar.bz2.directive + perl -i -pe "s/20\d\d\d\d\d\d/$(YYYYMMDD)/" parallel-*.tar.*directive + gpg --clearsign --yes parallel-$(YYYYMMDD).tar.bz2.directive + + cp doc/parallel.latest.directive parallel-latest.tar.bz2.directive + perl -i -pe "s/20\d\d\d\d\d\d/$(YYYYMMDD)/" parallel-latest.tar.*directive + gpg --clearsign --yes parallel-latest.tar.bz2.directive + + (echo '#!/bin/bash'; \ + echo; \ + echo "# To check the signature run:"; \ + echo "# echo | gpg"; \ + echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig"; \ + echo; \ + echo "echo | gpg 2>/dev/null"; \ + echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \ + echo 'exit $$?' ; \ + echo; \ + gpg -ab -o - parallel-$(YYYYMMDD).tar.bz2; \ + ) > parallel-$(YYYYMMDD).tar.bz2.sig + + (echo '#!/bin/bash'; \ + echo; \ + echo "# To check the signature run:"; \ + echo "# echo | gpg"; \ + echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig"; \ + echo; \ + echo "echo | gpg 2>/dev/null"; \ + echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \ + echo 'exit $$?' ; \ + echo; \ + gpg -ab -o - parallel-$(YYYYMMDD).tar.bz2; \ + ) > parallel-latest.tar.bz2.sig + + gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig + gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig + + ../ftpsync/src/ftpsync parallel-$(YYYYMMDD).tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/alpha/ + +# This can take 7 minutes + pushd /tmp; \ + rm -rf /tmp/parallel-$(YYYYMMDD)*; \ + while ! wget http://alpha.gnu.org/gnu/parallel/parallel-$(YYYYMMDD).tar.bz2 ; do sleep 2; done; \ + tar xjvf parallel-$(YYYYMMDD).tar.bz2; \ + cd parallel-$(YYYYMMDD); \ + ./configure; \ + make -j && sudo make -j install; \ + pushd; \ + sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD) + EXTRA_DIST = CITATION CREDITS diff --git a/Makefile.in b/Makefile.in index fb1a5d29..12809a15 100644 --- a/Makefile.in +++ b/Makefile.in @@ -807,6 +807,62 @@ upload: pushd; \ sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD) +alphaupload: +# Copy of upload + eval `gpg-agent --daemon` +# make sure $YYYYMMDD is set + echo $(YYYYMMDD) | grep 20 + cp parallel-$(YYYYMMDD).tar.bz2 parallel-latest.tar.bz2 + cp doc/parallel.directive parallel-$(YYYYMMDD).tar.bz2.directive + perl -i -pe "s/20\d\d\d\d\d\d/$(YYYYMMDD)/" parallel-*.tar.*directive + gpg --clearsign --yes parallel-$(YYYYMMDD).tar.bz2.directive + + cp doc/parallel.latest.directive parallel-latest.tar.bz2.directive + perl -i -pe "s/20\d\d\d\d\d\d/$(YYYYMMDD)/" parallel-latest.tar.*directive + gpg --clearsign --yes parallel-latest.tar.bz2.directive + + (echo '#!/bin/bash'; \ + echo; \ + echo "# To check the signature run:"; \ + echo "# echo | gpg"; \ + echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig"; \ + echo; \ + echo "echo | gpg 2>/dev/null"; \ + echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \ + echo 'exit $$?' ; \ + echo; \ + gpg -ab -o - parallel-$(YYYYMMDD).tar.bz2; \ + ) > parallel-$(YYYYMMDD).tar.bz2.sig + + (echo '#!/bin/bash'; \ + echo; \ + echo "# To check the signature run:"; \ + echo "# echo | gpg"; \ + echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig"; \ + echo; \ + echo "echo | gpg 2>/dev/null"; \ + echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \ + echo 'exit $$?' ; \ + echo; \ + gpg -ab -o - parallel-$(YYYYMMDD).tar.bz2; \ + ) > parallel-latest.tar.bz2.sig + + gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig + gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig + + ../ftpsync/src/ftpsync parallel-$(YYYYMMDD).tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/alpha/ + +# This can take 7 minutes + pushd /tmp; \ + rm -rf /tmp/parallel-$(YYYYMMDD)*; \ + while ! wget http://alpha.gnu.org/gnu/parallel/parallel-$(YYYYMMDD).tar.bz2 ; do sleep 2; done; \ + tar xjvf parallel-$(YYYYMMDD).tar.bz2; \ + cd parallel-$(YYYYMMDD); \ + ./configure; \ + make -j && sudo make -j install; \ + pushd; \ + sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD) + # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. .NOEXPORT: diff --git a/NEWS b/NEWS index 6a8eba2c..4c19fb5a 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +20170206alpha + +* --tee introduced. + + 20170122 * sql now uses a temporary credentials file for MySQL to avoid diff --git a/README b/README index 9ffd0de1..dab1a946 100644 --- a/README +++ b/README @@ -44,9 +44,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20170122.tar.bz2 - bzip2 -dc parallel-20170122.tar.bz2 | tar xvf - - cd parallel-20170122 + wget http://ftpmirror.gnu.org/parallel/parallel-20170206.tar.bz2 + bzip2 -dc parallel-20170206.tar.bz2 | tar xvf - + cd parallel-20170206 ./configure && make && sudo make install @@ -55,9 +55,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20170122.tar.bz2 - bzip2 -dc parallel-20170122.tar.bz2 | tar xvf - - cd parallel-20170122 + wget http://ftpmirror.gnu.org/parallel/parallel-20170206.tar.bz2 + bzip2 -dc parallel-20170206.tar.bz2 | tar xvf - + cd parallel-20170206 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index 9f9095d1..435ab796 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20170122. +# Generated by GNU Autoconf 2.69 for parallel 20170206. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20170122' -PACKAGE_STRING='parallel 20170122' +PACKAGE_VERSION='20170206' +PACKAGE_STRING='parallel 20170206' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20170122 to adapt to many kinds of systems. +\`configure' configures parallel 20170206 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1281,7 +1281,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20170122:";; + short | recursive ) echo "Configuration of parallel 20170206:";; esac cat <<\_ACEOF @@ -1357,7 +1357,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20170122 +parallel configure 20170206 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20170122, which was +It was created by parallel $as_me 20170206, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2237,7 +2237,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20170122' + VERSION='20170206' cat >>confdefs.h <<_ACEOF @@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20170122, which was +This file was extended by parallel $as_me 20170206, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2942,7 +2942,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20170122 +parallel config.status 20170206 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 01ad29c0..b588ca17 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20170122], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20170206], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/src/Makefile.am b/src/Makefile.am index 6f47b5da..0742d208 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,6 +1,7 @@ -bin_SCRIPTS = parallel sql niceload parcat \ - env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \ - env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh +bin_SCRIPTS = parallel sql niceload parcat env_parallel \ + env_parallel.ash env_parallel.bash env_parallel.csh \ + env_parallel.dash env_parallel.fish env_parallel.ksh \ + env_parallel.pdksh env_parallel.tcsh env_parallel.zsh install-exec-hook: rm $(DESTDIR)$(bindir)/sem || true @@ -228,9 +229,10 @@ DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \ parallel_tutorial.pdf parallel_design.pdf parallel_alternatives.pdf \ parcat.pdf -EXTRA_DIST = parallel sem sql niceload parcat env_parallel \ - env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \ - env_parallel.pdksh env_parallel.csh env_parallel.tcsh \ - sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \ - parallel_design.pod parallel_alternatives.pod \ - $(DISTCLEANFILES) +EXTRA_DIST = parallel sem sql niceload parcat env_parallel \ + env_parallel.ash env_parallel.bash env_parallel.csh \ + env_parallel.dash env_parallel.fish env_parallel.ksh \ + env_parallel.pdksh env_parallel.tcsh env_parallel.zsh \ + sem.pod parallel.pod env_parallel.pod niceload.pod \ + parallel_tutorial.pod parallel_design.pod \ + parallel_alternatives.pod $(DISTCLEANFILES) diff --git a/src/Makefile.in b/src/Makefile.in index 2fa5e1c7..5b26de2a 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -229,9 +229,10 @@ target_alias = @target_alias@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ -bin_SCRIPTS = parallel sql niceload parcat \ - env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \ - env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh +bin_SCRIPTS = parallel sql niceload parcat env_parallel \ + env_parallel.ash env_parallel.bash env_parallel.csh \ + env_parallel.dash env_parallel.fish env_parallel.ksh \ + env_parallel.pdksh env_parallel.tcsh env_parallel.zsh @DOCUMENTATION_TRUE@man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \ @DOCUMENTATION_TRUE@ parallel_tutorial.7 parallel_design.7 parallel_alternatives.7 \ @@ -260,12 +261,13 @@ DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \ parallel_tutorial.pdf parallel_design.pdf parallel_alternatives.pdf \ parcat.pdf -EXTRA_DIST = parallel sem sql niceload parcat env_parallel \ - env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \ - env_parallel.pdksh env_parallel.csh env_parallel.tcsh \ - sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \ - parallel_design.pod parallel_alternatives.pod \ - $(DISTCLEANFILES) +EXTRA_DIST = parallel sem sql niceload parcat env_parallel \ + env_parallel.ash env_parallel.bash env_parallel.csh \ + env_parallel.dash env_parallel.fish env_parallel.ksh \ + env_parallel.pdksh env_parallel.tcsh env_parallel.zsh \ + sem.pod parallel.pod env_parallel.pod niceload.pod \ + parallel_tutorial.pod parallel_design.pod \ + parallel_alternatives.pod $(DISTCLEANFILES) all: all-am diff --git a/src/env_parallel b/src/env_parallel index 041486d4..632a919c 100755 --- a/src/env_parallel +++ b/src/env_parallel @@ -18,28 +18,41 @@ # or write to the Free Software Foundation, Inc., 51 Franklin St, # Fifth Floor, Boston, MA 02110-1301 USA -GREPQ="grep >/dev/null 2>/dev/null" +grepq() { + # grep -q for systems without -q + grep >/dev/null 2>/dev/null "$@" +} + +installer() { + source="$1" + script="$2" + into="$3" + if grepq $script $into; then + true already installed + else + echo $source \`which $script\` >> $into + fi +} while test $# -gt 0; do key="$1" case $key in -i|--install) - eval $GREPQ env_parallel.bash $HOME/.bashrc || - echo '. `which env_parallel.bash`' >> $HOME/.bashrc - eval $GREPQ env_parallel.zsh $HOME/.zshenv || - echo '. `which env_parallel.zsh`' >> $HOME/.zshenv + installer . env_parallel.bash $HOME/.bashrc + installer . env_parallel.zsh $HOME/.zshenv + installer source env_parallel.ksh $HOME/.kshrc + echo $SHELL | grepq /pdksh && + installer . env_parallel.pdksh $HOME/.profile + echo $SHELL | grepq /ash && + installer . env_parallel.ash $HOME/.profile + echo $SHELL | grepq /dash && + installer . env_parallel.dash $HOME/.profile + installer source env_parallel.csh $HOME/.cshrc + installer source env_parallel.tcsh $HOME/.tcshrc mkdir -p $HOME/.config/fish - eval $GREPQ env_parallel.fish $HOME/.config/fish/config.fish || + grepq env_parallel.fish $HOME/.config/fish/config.fish || echo '. (which env_parallel.fish)' >> $HOME/.config/fish/config.fish - eval $GREPQ env_parallel.ksh $HOME/.kshrc || - echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc - eval $GREPQ env_parallel.pdksh $HOME/.profile || - echo '. `which env_parallel.pdksh`' >> $HOME/.profile - eval $GREPQ env_parallel.csh $HOME/.cshrc || - echo 'source `which env_parallel.csh`' >> $HOME/.cshrc - eval $GREPQ env_parallel.tcsh $HOME/.tcshrc || - echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc echo 'Installed env_parallel in: ' echo " " $HOME/.bashrc echo " " $HOME/.zshenv @@ -83,6 +96,14 @@ pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh` E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile Supports: aliases, functions, variables, arrays +ash: Put this in $HOME/.profile: . `which env_parallel.ash` + E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile + Supports: aliases, variables + +dash: Put this in $HOME/.profile: . `which env_parallel.dash` + E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile + Supports: aliases, variables + csh: Put this in $HOME/.cshrc: source `which env_parallel.csh` E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc Supports: aliases, variables, arrays with no special chars diff --git a/src/env_parallel.ash b/src/env_parallel.ash new file mode 100644 index 00000000..f733b0b3 --- /dev/null +++ b/src/env_parallel.ash @@ -0,0 +1,145 @@ +#!/usr/bin/env ash + +# This file must be sourced in ash: +# +# . `which env_parallel.ash` +# +# after which 'env_parallel' works +# +# +# Copyright (C) 2017 +# Ole Tange and Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# or write to the Free Software Foundation, Inc., 51 Franklin St, +# Fifth Floor, Boston, MA 02110-1301 USA + +env_parallel() { + # env_parallel.ash + + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + local _grep_REGEXP="$( + perl -e ' + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + )" + # Deal with --env _ + local _ignore_UNDERSCORE="$( + perl -e ' + for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } + ' -- "$@" + )" + + _list_aliases() { + alias | perl -pe 's/=.*//' + } + _body_aliases() { + alias "$@" | perl -pe 's/^/alias /' + } + _list_functions() { + # TODO see http://unix.stackexchange.com/questions/343297/dash-list-declared-functions + true; + } + _list_variables() { + # This may screw up if variables contain \n and = + set | perl -ne 's/=.*// and print;' + } + _body_variables() { + # Crappy typeset -p + for _i in "$@" + do + echo -n "$_i"= + eval echo -n \"\$$_i\" | + perl -pe 's/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]/\\$&/go;'"s/'/\\\'/g; s/[\n]/'\\n'/go;"; + echo + done + } + + # --record-env + if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then + (_list_aliases; + _list_functions; + _list_variables) | + cat > $HOME/.parallel/ignored_vars + return 0 + fi + + # Grep alias names + local _alias_NAMES="$(_list_aliases | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" + local _list_alias_BODIES="_body_aliases $_alias_NAMES" + if [ "$_alias_NAMES" = "" ] ; then + # no aliases selected + _list_alias_BODIES="true" + fi + unset _alias_NAMES + + # Grep function names + local _function_NAMES="$(_list_functions | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" + local _list_function_BODIES="typeset -f $_function_NAMES" + if [ "$_function_NAMES" = "" ] ; then + # no functions selected + _list_function_BODIES="true" + fi + unset _function_NAMES + + # Grep variable names + local _variable_NAMES="$(_list_variables | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | +# grep -vFf <(readonly -p) | + grep -Ev '^(BASHOPTS|BASHPID|EUID|GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|PPID|SHELLOPTS|UID|USERNAME|BASH_[A-Z_]+)$')" + local _list_variable_VALUES="_body_variables $_variable_NAMES" + if [ "$_variable_NAMES" = "" ] ; then + # no variables selected + _list_variable_VALUES="true" + fi + unset _variable_NAMES + + # Copy shopt (so e.g. extended globbing works) + # But force expand_aliases as aliases otherwise do not work + export PARALLEL_ENV="$( + $_list_alias_BODIES; + $_list_variable_VALUES; + $_list_function_BODIES)"; + unset _list_alias_BODIES + unset _list_variable_VALUES + unset _list_function_BODIES + `which parallel` "$@"; + _parallel_exit_CODE=$? + unset PARALLEL_ENV; + return $_parallel_exit_CODE +} diff --git a/src/env_parallel.dash b/src/env_parallel.dash new file mode 100755 index 00000000..45d944e4 --- /dev/null +++ b/src/env_parallel.dash @@ -0,0 +1,145 @@ +#!/usr/bin/env dash + +# This file must be sourced in dash: +# +# . `which env_parallel.dash` +# +# after which 'env_parallel' works +# +# +# Copyright (C) 2017 +# Ole Tange and Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# or write to the Free Software Foundation, Inc., 51 Franklin St, +# Fifth Floor, Boston, MA 02110-1301 USA + +env_parallel() { + # env_parallel.dash + + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + local _grep_REGEXP="$( + perl -e ' + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + )" + # Deal with --env _ + local _ignore_UNDERSCORE="$( + perl -e ' + for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } + ' -- "$@" + )" + + _list_aliases() { + alias | perl -pe 's/=.*//' + } + _body_aliases() { + alias "$@" | perl -pe 's/^/alias /' + } + _list_functions() { + # TODO see http://unix.stackexchange.com/questions/343297/dash-list-declared-functions + true; + } + _list_variables() { + # This may screw up if variables contain \n and = + set | perl -ne 's/=.*// and print;' + } + _body_variables() { + # Crappy typeset -p + for _i in "$@" + do + echo -n "$_i"= + eval echo -n \"\$$_i\" | + perl -pe 's/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]/\\$&/go;'"s/'/\\\'/g; s/[\n]/'\\n'/go;"; + echo + done + } + + # --record-env + if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then + (_list_aliases; + _list_functions; + _list_variables) | + cat > $HOME/.parallel/ignored_vars + return 0 + fi + + # Grep alias names + local _alias_NAMES="$(_list_aliases | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" + local _list_alias_BODIES="_body_aliases $_alias_NAMES" + if [ "$_alias_NAMES" = "" ] ; then + # no aliases selected + _list_alias_BODIES="true" + fi + unset _alias_NAMES + + # Grep function names + local _function_NAMES="$(_list_functions | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" + local _list_function_BODIES="typeset -f $_function_NAMES" + if [ "$_function_NAMES" = "" ] ; then + # no functions selected + _list_function_BODIES="true" + fi + unset _function_NAMES + + # Grep variable names + local _variable_NAMES="$(_list_variables | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | +# grep -vFf <(readonly -p) | + grep -Ev '^(BASHOPTS|BASHPID|EUID|GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|PPID|SHELLOPTS|UID|USERNAME|BASH_[A-Z_]+)$')" + local _list_variable_VALUES="_body_variables $_variable_NAMES" + if [ "$_variable_NAMES" = "" ] ; then + # no variables selected + _list_variable_VALUES="true" + fi + unset _variable_NAMES + + # Copy shopt (so e.g. extended globbing works) + # But force expand_aliases as aliases otherwise do not work + export PARALLEL_ENV="$( + $_list_alias_BODIES; + $_list_variable_VALUES; + $_list_function_BODIES)"; + unset _list_alias_BODIES + unset _list_variable_VALUES + unset _list_function_BODIES + `which parallel` "$@"; + _parallel_exit_CODE=$? + unset PARALLEL_ENV; + return $_parallel_exit_CODE +} diff --git a/src/env_parallel.pod b/src/env_parallel.pod index 21a52694..32cb67e6 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -81,6 +81,50 @@ Same as GNU B. =head1 SUPPORTED SHELLS +=head2 Ash + +B<--env> is supported to export only the variable, or alias with the +given name. Multiple B<--env>s can be given. + +Installation + +Put this in $HOME/.profile: + + . `which env_parallel.ash` + +E.g. by doing: + + echo '. `which env_parallel.ash`' >> $HOME/.profile + +=over 8 + +=item aliases + + alias myecho='echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work + +=item functions + +Functions cannot be used. If you find a way to list function names +and definitions please contact B. + +=item variables + + myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + +=item arrays + +Arrays are not supported by Ash. + +=back + =head2 Bash B<--env> is supported to export only the variable, alias, function, or @@ -184,6 +228,51 @@ Not supported by B. =back +=head2 Dash + +B<--env> is supported to export only the variable, or alias with the +given name. Multiple B<--env>s can be given. + +Installation + +Put this in $HOME/.profile: + + . `which env_parallel.dash` + +E.g. by doing: + + echo '. `which env_parallel.dash`' >> $HOME/.profile + +=over 8 + +=item aliases + + alias myecho='echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work + +=item functions + +Functions cannot be used. If you find a way to list function names +and definitions please contact B. + +=item variables + + myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + +=item arrays + +Arrays are not supported by Dash. + +=back + + =head2 fish B<--env> is supported to export only the variable, alias, function, or diff --git a/src/niceload b/src/niceload index 30e2735e..294612a6 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20170123; +$Global::version = 20170206; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index a493e8c0..d146a259 100755 --- a/src/parallel +++ b/src/parallel @@ -1361,7 +1361,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20170202; + $Global::version = 20170208; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; diff --git a/src/parallel.pod b/src/parallel.pod index 8753b2a4..6c9dd6d5 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2180,6 +2180,11 @@ fill: seq 1000 | parallel --pipe --tee --tag 'grep {1} | wc {2}' ::: {0..9} ::: -l -c +How many words contain a..z and how many bytes do they fill? + + parallel -a /usr/share/dict/words --pipepart --tee --tag \ + 'grep {1} | wc {2}' ::: {a..z} ::: -l -c + =item B<--termseq> I diff --git a/src/sql b/src/sql index 26e18534..473809f2 100755 --- a/src/sql +++ b/src/sql @@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20170123; + $Global::version = 20170206; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index 9b7c99c8..fe7d87bb 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -35,12 +35,12 @@ sql mysql://root:"$mysqlrootpass"@/mysql "DROP DATABASE `whoami`;DROP USER '`who sql mysql://root:"$mysqlrootpass"@/mysql "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';" # SHELLS -$INSTALL ash csh fdclone fish fizsh ksh mksh pdksh posh rc rush sash tcsh yash zsh +$INSTALL ash csh dash fdclone fish fizsh ksh mksh pdksh posh rc rush sash tcsh yash zsh SSHPASS=`goodpasswd` export SSHPASS #shells="bash sh csh ash tcsh zsh ksh fish fizsh mksh pdksh posh rc sash yash nopathbash nopathcsh" -shells="bash sh csh ash tcsh zsh ksh fish fizsh mksh posh rc sash yash nopathbash nopathcsh" +shells="bash sh csh ash dash tcsh zsh ksh fish fizsh mksh posh rc sash yash nopathbash nopathcsh" create_shell_user() { shell="$1" sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM