From 42a5f4e0cc0bc62d80180d2b3148e8424a3bde0f Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 17 Jan 2011 13:38:58 +0100 Subject: [PATCH] testsuite passes. max_jobs_running could be resat if -j is a changed file. dummy children was not killed if cmd line was too long. default number of jobs for --semaphore set to 1. test25 had race condition. test22 -j10 added. test13 -j9 added. test47 remote cleanup added before test. test17 remote cleanup added before test. --- doc/FUTURE_IDEAS | 8 +++++++ doc/release_new_version | 3 +++ src/parallel | 22 ++++++++++++++---- testsuite/tests-to-run/test10.sh | 10 ++++----- testsuite/tests-to-run/test13.sh | 2 +- testsuite/tests-to-run/test14.sh | 8 +++---- testsuite/tests-to-run/test17.sh | 34 +++++++++++++++++++++++++--- testsuite/tests-to-run/test22.sh | 16 +++++++------- testsuite/tests-to-run/test25.sh | 2 +- testsuite/tests-to-run/test47.sh | 38 +++++++++++++++++++++++++++----- testsuite/tests-to-run/test48.sh | 2 +- testsuite/wanted-results/test13 | 2 +- testsuite/wanted-results/test17 | 8 +++++-- testsuite/wanted-results/test27 | 37 ++++++++++++++++++++----------- testsuite/wanted-results/test32 | 2 ++ testsuite/wanted-results/test47 | 10 +++++++++ 16 files changed, 156 insertions(+), 48 deletions(-) diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index d8ddc47b..2d8f47d5 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,3 +1,11 @@ +max_jobs_running could be resat if -j is a changed file. +dummy children was not killed if cmd line was too long. +default number of jobs for --semaphore set to 1. +test25 had race condition. +test22 -j10 added. +test13 -j9 added. +test47 remote cleanup added before test. +test17 remote cleanup added before test. parallel: spread arguments between all jobslots when reaching EOF of input codecoverage diff --git a/doc/release_new_version b/doc/release_new_version index a800f71b..99e4a4f1 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -152,6 +152,9 @@ New in this release: * Review in French. Thanks to Denis Dordoigne. http://linuxfr.org/2010/12/29/27715.html +* Review in Spanish. + http://gufete.net/index.php?entry=entry110116-200022 + * Use case for memcache. http://www.dctrwatson.com/2010/12/how-to-dump-memcache-keyvalue-pairs-fast/ diff --git a/src/parallel b/src/parallel index 32198302..91e46d83 100755 --- a/src/parallel +++ b/src/parallel @@ -197,7 +197,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20110101; + $Global::version = 20110111; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -320,6 +320,9 @@ sub parse_options { $Semaphore::fg = $::opt_fg; $Semaphore::wait = $::opt_wait; $Global::default_simultaneous_sshlogins = 1; + if(not defined $::opt_P) { + $::opt_P = 1; + } } if(defined $::opt_eta) { $::opt_progress = $::opt_eta; @@ -1244,6 +1247,12 @@ sub __USAGE__ {} sub wait_and_exit { # If we do not wait, we sometimes get segfault # Returns: N/A + for (keys %Global::unkilled_children) { + kill 9, $_; + waitpid($_,0); + delete $Global::unkilled_children{$_}; + } + wait(); exit(shift); } @@ -1445,7 +1454,10 @@ sub set_max_jobs_running { $Global::max_jobs_running -= $self->{'max_jobs_running'}; } $self->{'max_jobs_running'} = shift; - $Global::max_jobs_running += $self->{'max_jobs_running'}; + if(defined $self->{'max_jobs_running'}) { + # max_jobs_running could be resat if -j is a changed file + $Global::max_jobs_running += $self->{'max_jobs_running'}; + } } sub loadavg_too_high { @@ -1657,12 +1669,13 @@ sub processes_available_by_system_limit { my $child; if($child = fork()) { push (@children,$child); + $Global::unkilled_children{$child} = 1; } elsif(defined $child) { # The child takes one process slot # It will be killed later $SIG{TERM} = $Global::original_sigterm; - sleep 100000; - wait_and_exit(0); + sleep 10000000; + exit(0); } else { $max_system_proc_reached = 1; } @@ -1694,6 +1707,7 @@ sub processes_available_by_system_limit { for my $pid (@children) { kill 9, $pid; waitpid($pid,0); + delete $Global::unkilled_children{$pid}; } #wait(); # Cleanup: Unget the command_lines or the @args diff --git a/testsuite/tests-to-run/test10.sh b/testsuite/tests-to-run/test10.sh index d3321e72..f1c6354d 100755 --- a/testsuite/tests-to-run/test10.sh +++ b/testsuite/tests-to-run/test10.sh @@ -5,11 +5,11 @@ seq 1 40 | parallel -j 0 seq 1 10 | sort |md5sum seq 1 40 | parallel -j 0 seq 1 10 '| parallel -j 3 echo' | sort |md5sum echo '### Test of xargs -m and -X' -seq 1 60000 | parallel -m echo | mop -d 4 "|sort |md5sum" "| wc" -(echo foo;echo bar) | parallel -m echo 1{}2{}3 A{}B{}C -(echo foo;echo bar) | parallel -X echo 1{}2{}3 A{}B{}C -seq 1 60000 | parallel -m echo a{}b{}c | mop -d 4 "|sort |md5sum" "| wc" -seq 1 60000 | parallel -m echo a{}b{}c | \ +seq 1 60000 | parallel -j1 -m echo | mop -d 4 "|sort |md5sum" "| wc" +(echo foo;echo bar) | parallel -j1 -m echo 1{}2{}3 A{}B{}C +(echo foo;echo bar) | parallel -j1 -X echo 1{}2{}3 A{}B{}C +seq 1 60000 | parallel -m -j1 echo a{}b{}c | mop -d 4 "|sort |md5sum" "| wc" +seq 1 60000 | parallel -m -j1 echo a{}b{}c | \ mop -q "|sort |md5sum" :par echo -n "Chars per line: " CHAR=$(cat ~/.mop/:par | wc -c) diff --git a/testsuite/tests-to-run/test13.sh b/testsuite/tests-to-run/test13.sh index abb49d1f..8a19552a 100755 --- a/testsuite/tests-to-run/test13.sh +++ b/testsuite/tests-to-run/test13.sh @@ -12,4 +12,4 @@ echo '### Test --keeporder' (seq 0 2) | parallel --keeporder -j100% -S 1/:,2/parallel@parallel-server2 -q perl -e 'sleep 1;print "job{}\n";exit({})' echo '### Test SIGTERM' -(sleep 5; killall parallel -TERM) & seq 1 100 | stdout parallel -k sleep 3';' echo | sort +(sleep 5; killall parallel -TERM) & seq 1 100 | stdout parallel -k -j9 sleep 3';' echo | sort diff --git a/testsuite/tests-to-run/test14.sh b/testsuite/tests-to-run/test14.sh index 64928baf..727808ef 100755 --- a/testsuite/tests-to-run/test14.sh +++ b/testsuite/tests-to-run/test14.sh @@ -3,18 +3,18 @@ # Test -I seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::' -seq 1 10 | parallel -k 'seq 1 {} | parallel -X -k -I :: echo a{} b::' +seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -X -k -I :: echo a{} b::' -seq 1 10 | parallel -k 'seq 1 {} | parallel -m -k -I :: echo a{} b::' +seq 1 10 | parallel -k 'seq 1 {} | parallel -j1 -m -k -I :: echo a{} b::' -seq 1 60000 | parallel -I :: -m echo a::b::c | \ +seq 1 60000 | parallel -I :: -m -j1 echo a::b::c | \ mop -q "|sort |md5sum" :par CHAR=$(cat ~/.mop/:par | wc -c) LINES=$(cat ~/.mop/:par | wc -l) echo -n "Chars per line ($CHAR/$LINES): " echo "$CHAR/$LINES" | bc -seq 1 60000 | parallel -I :: -X echo a::b::c | \ +seq 1 60000 | parallel -I :: -X -j1 echo a::b::c | \ mop -q "|sort |md5sum" :par CHAR=$(cat ~/.mop/:par | wc -c) LINES=$(cat ~/.mop/:par | wc -l) diff --git a/testsuite/tests-to-run/test17.sh b/testsuite/tests-to-run/test17.sh index 65aafee4..8c65c7a5 100755 --- a/testsuite/tests-to-run/test17.sh +++ b/testsuite/tests-to-run/test17.sh @@ -40,6 +40,8 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' echo '### --transfer --cleanup - abspath' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' cat /tmp/test17abs | parallel -k --transfer --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {} echo good if no file # Should give: No such file or directory @@ -48,6 +50,8 @@ stdout ssh $SERVER1 ls '/tmp/parallel.file*' stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' echo '### --transfer --cleanup - relpath' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' cat /tmp/test17rel | parallel -k --transfer --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {} # Should give: No such file or directory echo good if no file @@ -56,25 +60,33 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --return - abspath' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out -cat /tmp/test17abs | parallel -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 echo {} ">"{.}.out +cat /tmp/test17abs | parallel -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p /tmp/parallel.file";"echo {} ">"{.}.out ls /tmp/parallel.file*out /tmp/parallel.file/*out echo '### --return - relpath' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf /tmp/parallel.file*out cat /tmp/test17rel | parallel -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out ls tmp/parallel.file*out tmp/parallel.file/*out echo '### --return - multiple files' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done \ - --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' ls tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done echo '### --return --cleanup - abspath' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done cat /tmp/test17abs | parallel -k --return {.}.out --return {}.done --cleanup \ - --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p /tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' ls /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done echo good if no file stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK @@ -83,6 +95,8 @@ stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK echo '### --return --cleanup - relpath' rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' ls tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done @@ -92,6 +106,8 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --return --cleanup - multiple returns' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' @@ -102,6 +118,8 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --transfer --return --cleanup - abspath' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done cat /tmp/test17abs | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ --sshlogin $SERVER1,parallel@$SERVER2 cat {} ">"{.}.out';'cat {} ">"{}.done';' @@ -113,6 +131,8 @@ stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK echo '### --transfer --return --cleanup - relpath' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' @@ -123,6 +143,8 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --transfer --return --cleanup - multiple files' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' @@ -132,6 +154,8 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --trc - abspath' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' @@ -142,6 +166,8 @@ stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK echo '### --trc - relpath' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done cat /tmp/test17rel | parallel -k --trc {.}.out --trc {}.done \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' @@ -152,6 +178,8 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --trc - multiple files' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' diff --git a/testsuite/tests-to-run/test22.sh b/testsuite/tests-to-run/test22.sh index e6e9d205..21de1356 100755 --- a/testsuite/tests-to-run/test22.sh +++ b/testsuite/tests-to-run/test22.sh @@ -12,23 +12,23 @@ echo false | parallel echo $? echo '### Test --halt-on-error' -(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel --halt-on-error 0 +(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 0 echo $? -(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel --halt-on-error 1 +(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 1 echo $? -(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel --halt-on-error 2 +(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2 echo $? -(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -H0 +(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 -H0 echo $? -(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -H1 +(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 -H1 echo $? -(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -H2 +(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 -H2 echo $? echo '### Test last dying print --halt-on-error' -(seq 0 8;echo 0; echo 9) | parallel -kqH1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift' +(seq 0 8;echo 0; echo 9) | parallel -j10 -kqH1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift' echo $? -(seq 0 8;echo 0; echo 9) | parallel -kqH2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift' +(seq 0 8;echo 0; echo 9) | parallel -j10 -kqH2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift' echo $? ) 2>&1 diff --git a/testsuite/tests-to-run/test25.sh b/testsuite/tests-to-run/test25.sh index 74ce139f..9ba046ab 100644 --- a/testsuite/tests-to-run/test25.sh +++ b/testsuite/tests-to-run/test25.sh @@ -17,7 +17,7 @@ echo via first cat |parallel -T -kv cat ::: - - EOF chmod 755 /tmp/parallel-script-for-script echo via pseudotty | script -q -f -c /tmp/parallel-script-for-script /dev/null -sleep 1 +sleep 2 echo '### Test stdin goes to first command only ("cat" as argument)' cat >/tmp/parallel-script-for-script2 <"{.}.out ls /tmp/parallel.file*out echo '### --return - relpath' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf /tmp/parallel.file*out cat /tmp/test17rel | parallel -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out ls tmp/parallel.file*out echo '### --return - multiple files' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file*done cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' ls tmp/parallel.file*out tmp/parallel.file*done echo '### --return --cleanup - abspath' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out /tmp/parallel.file*done cat /tmp/test17abs | parallel -k --return {.}.out --return {}.done --cleanup \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' @@ -84,9 +96,11 @@ stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK echo '### --return --cleanup - relpath' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file*done cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \ - --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' + --sshlogin $SERVER1,parallel@$SERVER2 echo {} ">"{.}.out';'echo {} ">"{}.done';' ls tmp/parallel.file*out tmp/parallel.file*done echo good if no file stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK @@ -94,9 +108,11 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --return --cleanup - multiple returns' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file*done cat /tmp/test17rel | parallel -k --return {.}.out --return {}.done --cleanup \ - --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp/parallel.file ';'echo {} ">"{.}.out';'echo {} ">"{}.done';' + --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp";"echo {} ">"{.}.out';'echo {} ">"{}.done';' ls /tmp/parallel.file*out /tmp/parallel.file*done echo good if no file stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK @@ -104,6 +120,8 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --transfer --return --cleanup - abspath' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out /tmp/parallel.file*done cat /tmp/test17abs | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ --sshlogin $SERVER1,parallel@$SERVER2 cat {} ">"{.}.out';'cat {} ">"{}.done';' @@ -115,9 +133,11 @@ stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK echo '### --transfer --return --cleanup - relpath' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file*done cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ - --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' + --sshlogin $SERVER1,parallel@$SERVER2 cat {} ">"{.}.out';'cat {} ">"{}.done';' ls /tmp/parallel.file*out /tmp/parallel.file*done echo good if no file stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK @@ -125,15 +145,19 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --transfer --return --cleanup - multiple files' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file*done cat /tmp/test17rel | parallel -k --transfer --return {.}.out --return {}.done --cleanup \ - --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' + --sshlogin $SERVER1,parallel@$SERVER2 cat {} ">"{.}.out';'cat {} ">"{}.done';' ls /tmp/parallel.file*out /tmp/parallel.file*done stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK # Should give: No such file or directory stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --trc - abspath' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out /tmp/parallel.file*done cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' @@ -144,9 +168,11 @@ stdout ssh $SERVER1 ls '/tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' || echo OK echo '### --trc - relpath' +stdout ssh $SERVER1 'rm -rf tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' rm -rf tmp/parallel.file*out tmp/parallel.file*done cat /tmp/test17rel | parallel -k --trc {.}.out --trc {}.done \ - --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' + --sshlogin $SERVER1,parallel@$SERVER2 cat {} ">"{.}.out';'cat {} ">"{}.done';' ls tmp/parallel.file*out tmp/parallel.file*done echo good if no file stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK @@ -154,6 +180,8 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK echo '### --trc - multiple files' +stdout ssh $SERVER1 'rm -rf /tmp/parallel.file*' +stdout ssh parallel@$SERVER2 'rm -rf /tmp/parallel.file*' rm -rf /tmp/parallel.file*out /tmp/parallel.file*done cat /tmp/test17abs | parallel -k --trc {.}.out --trc {}.done \ --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'cat {} ">"{.}.out';'cat {} ">"{}.done';' diff --git a/testsuite/tests-to-run/test48.sh b/testsuite/tests-to-run/test48.sh index b8d0aac0..4adba56f 100644 --- a/testsuite/tests-to-run/test48.sh +++ b/testsuite/tests-to-run/test48.sh @@ -13,7 +13,7 @@ echo '### Test -N is not broken by distribution - single line' seq 9 | parallel -N 10 echo echo '### Test -N is not broken by distribution - two lines' -seq 19 | parallel -N 10 echo +seq 19 | parallel -k -N 10 echo diff --git a/testsuite/wanted-results/test13 b/testsuite/wanted-results/test13 index 977417c4..05cef882 100644 --- a/testsuite/wanted-results/test13 +++ b/testsuite/wanted-results/test13 @@ -1,5 +1,5 @@ ### Test -k -Warning: Only enough filehandles to run 19 jobs in parallel. Raising ulimit -n may help +Warning: Only enough filehandles to run 20 jobs in parallel. Raising ulimit -n may help begin 1 2 diff --git a/testsuite/wanted-results/test17 b/testsuite/wanted-results/test17 index d7afc6ca..4b9ccc7d 100644 --- a/testsuite/wanted-results/test17 +++ b/testsuite/wanted-results/test17 @@ -220,8 +220,6 @@ tmp/parallel.file>fire.out /tmp/parallel.file>fire.file.done /tmp/parallel.file>fire.out good if no file -ls: cannot access /tmp/parallel.file*: No such file or directory -OK ### --return --cleanup - relpath tmp/parallel.file : & ) \n*.jpg.file.done tmp/parallel.file : & ) \n*.jpg.out @@ -394,6 +392,8 @@ OK good if no file ls: cannot access tmp/parallel.file*: No such file or directory OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK ### --transfer --return --cleanup - multiple files /tmp/parallel.file : & ) \n*.jpg.file.done /tmp/parallel.file : & ) \n*.jpg.out @@ -437,6 +437,8 @@ OK /tmp/parallel.file>fire.out ls: cannot access tmp/parallel.file*: No such file or directory OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK ### --trc - abspath /tmp/parallel.file : & ) \n*.jpg.file.done /tmp/parallel.file : & ) \n*.jpg.out @@ -525,6 +527,8 @@ tmp/parallel.file>fire.out good if no file ls: cannot access tmp/parallel.file*: No such file or directory OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK ### --trc - multiple files /tmp/parallel.file : & ) \n*.jpg.file.done /tmp/parallel.file : & ) \n*.jpg.out diff --git a/testsuite/wanted-results/test27 b/testsuite/wanted-results/test27 index 1f0bfa82..033265ae 100644 --- a/testsuite/wanted-results/test27 +++ b/testsuite/wanted-results/test27 @@ -25,7 +25,8 @@ hell rld ### -E_ -0 echo < eof_-0.xi one two _ three four -one two +one +two ### -i -0 echo from \{\} to x{}y < items-0.xi from one to xoney from @@ -270,7 +271,8 @@ three ### -r echo this plus that < blank.xi ### -0 -s118 echo < stairs-0.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 -1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +1 22 333 4444 55555 666666 7777777 88888888 999999999 +1 22 333 4444 55555 666666 7777777 88888888 999999999 ### -0 -s19 echo < stairs-0.xi 1 22 333 4444 55555 666666 @@ -312,7 +314,8 @@ three 88888888 7777777 666666 55555 -4444 333 22 1 +4444 333 +22 1 ### -0 -s20 echo < stairs-0.xi 1 22 333 4444 55555 666666 @@ -342,7 +345,8 @@ three 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 -88888888 999999999 +88888888 +999999999 ### -0 echo this plus that < space.xi this plus that @@ -401,7 +405,8 @@ from two to xtwoy hi there /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h /src/gnu/autoconf-1.11/autoconf.sh /src/gnu/autoconf-1.11/acgeneral.m4 /src/gnu/autoconf-1.11/acspecific.m4 /src/gnu/autoconf-1.11/configure hi there /src/gnu/autoconf-1.11/configure.in /src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex hi there /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h /src/gnu/autoconf-1.11/autoconf.sh /src/gnu/autoconf-1.11/acgeneral.m4 /src/gnu/autoconf-1.11/acspecific.m4 /src/gnu/autoconf-1.11/configure -hi there /src/gnu/autoconf-1.11/configure.in /src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +hi there /src/gnu/autoconf-1.11/configure.in /src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info +hi there /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex ### -IARG echo from ARG to xARGy -E_ < eof_.xi from one to xoney -E_ from two to xtwoy -E_ @@ -629,7 +634,8 @@ an embedded newline with 'single quotes' as well. ### -s118 echo < stairs.xi 1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 -1 22 333 4444 55555 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 88888888 999999999 +1 22 333 4444 55555 666666 7777777 88888888 999999999 +1 22 333 4444 55555 666666 7777777 88888888 999999999 ### -s19 echo < stairs2.xi 999999999 88888888 @@ -650,7 +656,8 @@ with 'single quotes' as well. 88888888 7777777 666666 55555 -4444 333 22 1 +4444 333 +22 1 ### -s19 echo < stairs.xi 1 22 333 4444 55555 666666 @@ -701,12 +708,14 @@ with 'single quotes' as well. 666666 7777777 88888888 999999999 1 22 333 4444 55555 666666 7777777 -88888888 999999999 +88888888 +999999999 ### -s470 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h /src/gnu/autoconf-1.11/autoconf.sh /src/gnu/autoconf-1.11/acgeneral.m4 /src/gnu/autoconf-1.11/acspecific.m4 /src/gnu/autoconf-1.11/configure /src/gnu/autoconf-1.11/configure.in /src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README /src/gnu/autoconf-1.11/Makefile.in /src/gnu/autoconf-1.11/INSTALL /src/gnu/autoconf-1.11/NEWS /src/gnu/autoconf-1.11/COPYING /src/gnu/autoconf-1.11/ChangeLog /src/gnu/autoconf-1.11/autoconf.texi /src/gnu/autoconf-1.11/acconfig.h /src/gnu/autoconf-1.11/autoconf.sh /src/gnu/autoconf-1.11/acgeneral.m4 /src/gnu/autoconf-1.11/acspecific.m4 /src/gnu/autoconf-1.11/configure /src/gnu/autoconf-1.11/configure.in -/src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info /src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex +/src/gnu/autoconf-1.11/autoheader.sh /src/gnu/autoconf-1.11/mkinstalldirs /src/gnu/autoconf-1.11/install.sh /src/gnu/autoconf-1.11/autoconf.info +/src/gnu/autoconf-1.11/standards.texi /src/gnu/autoconf-1.11/make-stds.texi /src/gnu/autoconf-1.11/standards.info /src/gnu/autoconf-1.11/texinfo.tex ### -s47 echo < files.xi /src/gnu/autoconf-1.11 /src/gnu/autoconf-1.11/README @@ -1059,14 +1068,16 @@ echo 88888888 999999999 88888888 999999999 echo 1 22 333 4444 55555 echo 666666 7777777 88888888 -echo 999999999 1 22 333 4444 -echo 55555 666666 7777777 -echo 88888888 999999999 1 22 333 4444 55555 +echo 999999999 1 22 333 4444 666666 7777777 88888888 +echo 55555 666666 7777777 999999999 1 22 333 4444 +echo 88888888 55555 666666 7777777 -88888888 999999999 +echo 999999999 +88888888 +999999999 ### -t echo this plus that < space.xi echo this plus that this plus that diff --git a/testsuite/wanted-results/test32 b/testsuite/wanted-results/test32 index cd65e0df..b66a149b 100644 --- a/testsuite/wanted-results/test32 +++ b/testsuite/wanted-results/test32 @@ -21,6 +21,8 @@ 4 2 ### Test of --retries on unreachable host +ssh: connect to host 4.3.2.1 port 22: Connection timed out +Warning: Could not figure out number of cpus on 4.3.2.1. Using 1 echo 1 1 echo 2 diff --git a/testsuite/wanted-results/test47 b/testsuite/wanted-results/test47 index 8989c537..0eafd24c 100644 --- a/testsuite/wanted-results/test47 +++ b/testsuite/wanted-results/test47 @@ -55,6 +55,8 @@ tmp/parallel.file2.out good if no file ls: cannot access tmp/parallel.file*: No such file or directory OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK ### --return --cleanup - multiple returns /tmp/parallel.file1.file.done /tmp/parallel.file1.out @@ -63,6 +65,8 @@ OK good if no file ls: cannot access tmp/parallel.file*: No such file or directory OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK ### --transfer --return --cleanup - abspath /tmp/parallel.file1.file.done /tmp/parallel.file1.out @@ -81,6 +85,8 @@ OK good if no file ls: cannot access tmp/parallel.file*: No such file or directory OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK ### --transfer --return --cleanup - multiple files /tmp/parallel.file1.file.done /tmp/parallel.file1.out @@ -88,6 +94,8 @@ OK /tmp/parallel.file2.out ls: cannot access tmp/parallel.file*: No such file or directory OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK ### --trc - abspath /tmp/parallel.file1.file.done /tmp/parallel.file1.out @@ -106,6 +114,8 @@ tmp/parallel.file2.out good if no file ls: cannot access tmp/parallel.file*: No such file or directory OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK ### --trc - multiple files /tmp/parallel.file1.file.done /tmp/parallel.file1.out