diff --git a/doc/release_new_version b/doc/release_new_version index 309bd82d..cb016b0a 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -231,7 +231,7 @@ New in this release: * GNU Parallel was cited in: Experimental study on the Wind Farm Substation Cable Installation Problem http://i11www.iti.uni-karlsruhe.de/_media/teaching/theses/ma-schmitz-14.pdf -* <> Comparing the CarbonTracker and TM5-4DVar data assimilation systems for CO2 surface flux inversions http://www.atmos-chem-phys-discuss.net/15/8883/2015/acpd-15-8883-2015-discussion.html +* <> Comparing the CarbonTracker and TM5-4DVar data assimilation systems for CO2 surface flux inversions http://www.atmos-chem-phys-discuss.net/15/8883/2015/acpd-15-8883-2015-discussion.html * <> CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf @@ -247,10 +247,16 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it) * GNU Parallel was used in: http://search.cpan.org/~ajpage/Bio-Roary-2.0.7/lib/Bio/Roary/JobRunner/Parallel.pm +* GNU Parallel was used in: Rapid haploid SNP calling https://github.com/tseemann/snippy + +* << afventer svar fra Rachel >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/ + * Pictures and Metadata http://www.ozzy.no/2015/02/05/pictures-and-metadata/ * Task automation with bash and parallel https://biowize.wordpress.com/2015/03/23/task-automation-with-bash-and-parallel/ +* How To: Speed Up File Transfers in Linux using RSync with GNU Parallel http://www.yourownlinux.com/2015/04/speed-up-file-transfers-using-rsync-with-gnu-parallel.html + * Mirroring a Git Repository https://avacariu.me/articles/mirroring-a-git-repository.html * Add an ENVI header to JAXA Global Mangrove Watch PALSAR tiles https://spectraldifferences.wordpress.com/2015/03/24/add-an-envi-header-to-jaxa-global-mangrove-watch-palsar-tiles/ diff --git a/src/parallel b/src/parallel index 02eae684..e19b2890 100755 --- a/src/parallel +++ b/src/parallel @@ -1051,7 +1051,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150409; + $Global::version = 20150416; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -2986,7 +2986,7 @@ sub reaper { $job or next; $job->set_exitstatus($? >> 8); $job->set_exitsignal($? & 127); - debug("run", "died (", $job->exitstatus(), "): ", $job->seq()); + debug("run", "seq ",$job->seq()," died (", $job->exitstatus(), ")"); $job->set_endtime(::now()); if($stiff == $Global::tty_taken) { # The process that died had the tty => release it @@ -3031,6 +3031,72 @@ sub reaper { sub __USAGE__ {} +sub killall { + # Kill all jobs + # Send all jobs TERM + # Wait + # Send all jobs TERM + # Wait + # Send all jobs KILL + # Send all (grand*)children KILL + + $Global::start_no_new_jobs ||= 1; + # pids of the all children and (grand*)children + # before we start the blood bath + my @family_pids = family_pids(keys %Global::running); + # Send jobs TERM + kill "TERM", keys %Global::running; + # Wait up to 200 ms + # Send jobs TERM (again) + my $sleepsum = 0; + my $sleep = 0; + for (; kill(0, keys %Global::running) and $sleepsum < 200; + $sleepsum += $sleep) { + # This can change %Global::running + $sleep = ::reap_usleep($sleep); + } + kill "TERM", keys %Global::running; + # Wait up to 200 ms + # Send jobs KILL + $sleepsum = 0; + $sleep = 0; + for (; kill(0, keys %Global::running) and $sleepsum < 200; + $sleepsum += $sleep) { + # This can change %Global::running + $sleep = ::reap_usleep($sleep); + } + kill "KILL", keys %Global::running; + # Send all (grand*)children KILL (if there are any left) + kill "KILL", @family_pids; +} + +sub family_pids { + # Find the pids with this->pid as (grand)*parent + # Input: + # @parents = pids of parents + # Returns: + # @pids = pids of (grand)*children + my @parents = @_; + my @pids; + + my ($children_of_ref, $parent_of_ref, $name_of_ref) = ::pid_table(); + + my @more = @parents; + # While more (grand)*children + while(@more) { + my @m; + push @pids, @more; + for my $parent (@more) { + if($children_of_ref->{$parent}) { + # add the children of this parent + push @m, @{$children_of_ref->{$parent}}; + } + } + @more = @m; + } + return (@pids); +} + sub wait_and_exit { # If we do not wait, we sometimes get segfault # Returns: N/A @@ -3038,9 +3104,7 @@ sub wait_and_exit { unlink keys %Global::unlink; if($error) { # Kill all without printing - for my $job (values %Global::running) { - $job->kill(); - } + killall(); } for (keys %Global::unkilled_children) { kill 9, $_; @@ -3359,13 +3423,14 @@ sub multiply_binary_prefix { } { - my ($disk_full_fh, $b8193, $name); + my ($disk_full_fh, $b8193, $error_printed); sub exit_if_disk_full { # Checks if $TMPDIR is full by writing 8kb to a tmpfile # If the disk is full: Exit immediately. # Returns: # N/A if(not $disk_full_fh) { + my $name; ($disk_full_fh, $name) = ::tmpfile(SUFFIX => ".df"); # Separate unlink due to NFS dealing badly with File::Temp unlink $name; @@ -3392,8 +3457,11 @@ sub multiply_binary_prefix { or tell $disk_full_fh != 8193) { # On raspbian the disk can be full except for 10 chars. - ::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n"); - ::error("Change \$TMPDIR with --tmpdir or use --compress.\n"); + if(not $error_printed) { + ::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n"); + ::error("Change \$TMPDIR with --tmpdir or use --compress.\n"); + $error_printed = 1; + } ::wait_and_exit(255); } truncate $disk_full_fh, 0; @@ -3507,6 +3575,9 @@ sub which { # Filter for SysV-style `ps` my $sysv = q( ps -ef | perl -ane '1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;). q(s/^.{$s}//; print "@F[1,2] $_"' ); + # Crazy msys: ' is not accepted on the cmd line, but " are treated as ' + my $msys = q( ps -ef | perl -ane "1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;). + q(s/^.{$s}//; print qq{@F[1,2] $_}" ); # BSD-style `ps` my $bsd = q(ps -o pid,ppid,command -ax); %pid_parentpid_cmd = @@ -3521,7 +3592,7 @@ sub which { 'hpux' => $sysv, 'linux' => $sysv, 'mirbsd' => $bsd, - 'msys' => $sysv, + 'msys' => $msys, 'MSWin32' => $sysv, 'netbsd' => $bsd, 'nto' => $sysv, @@ -4441,8 +4512,10 @@ sub compute_number_of_processes { # The child takes one process slot # It will be killed later $SIG{TERM} = $Global::original_sig{TERM}; - sleep 10000000; - exit(0); + # Exec 'sleep' to save RAM + # 32000 seconds should be plenty, and should be supported + # on all platforms. + exec("sleep","32000"); } else { # Failed to spawn $max_system_proc_reached = 1; @@ -5702,8 +5775,8 @@ sub filter_through_compress { # The tmpfile is used to tell the reader that the writer has started, # so unlink it to start with. unlink $self->fh($fdno,'name'); - my $wpid = open(my $fdw,"|-", empty_input_detector(). - "| ($opt::compress_program) >>". + my $wpid = open(my $fdw,"|-", "(".empty_input_detector(). + "| ($opt::compress_program)) >>". $self->fh($fdno,'name')) || die $?; $self->set_fh($fdno,'w',$fdw); $self->set_fh($fdno,'wpid',$wpid); @@ -5927,6 +6000,7 @@ sub kill { my $self = shift; my @signals = @_; my @family_pids = $self->family_pids(); + # Record this jobs as failed $self->set_exitstatus(-1); # Send two TERMs to give time to clean up @@ -6999,7 +7073,7 @@ sub print { } next; } - ::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):"); + ::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):\n"); if($opt::files) { $self->files_print($fdno,$in_fh,$out_fd); } elsif($opt::linebuffer) { @@ -7024,14 +7098,18 @@ sub files_print { my $self = shift; my ($fdno,$in_fh,$out_fd) = @_; - # If --compress: $in_fh must be closed first. + # If the job is dead: close printing fh. Needed for --compress close $self->fh($fdno,"w"); if($? and $opt::compress) { - ::error($opt::decompress_program." failed.\n"); + ::error($opt::compress_program." failed.\n"); $self->set_exitstatus(255); } + if($opt::compress) { + # Kill the decompressor which will not be needed + CORE::kill "TERM", $self->fh($fdno,"rpid"); + } close $in_fh; - # TODO if $?: die(?) + if($opt::pipe and $self->virgin()) { # Nothing was printed to this job: # cleanup unused tmp files if --files was set @@ -7054,7 +7132,7 @@ sub linebuffer_print { # If the job is dead: close printing fh. Needed for --compress close $self->fh($fdno,"w"); if($? and $opt::compress) { - ::error($opt::decompress_program." failed.\n"); + ::error($opt::compress_program." failed.\n"); $self->set_exitstatus(255); } if($opt::compress) { @@ -7123,7 +7201,10 @@ sub linebuffer_print { } else { # decompress done: close fh close $in_fh; - # TODO if $?: die(?) + if($? and $opt::compress) { + ::error($opt::decompress_program." failed.\n"); + $self->set_exitstatus(255); + } } } } @@ -7134,7 +7215,7 @@ sub tag_print { my $buf; close $self->fh($fdno,"w"); if($? and $opt::compress) { - ::error($opt::decompress_program." failed.\n"); + ::error($opt::compress_program." failed.\n"); $self->set_exitstatus(255); } seek $in_fh, 0, 0; @@ -7163,6 +7244,10 @@ sub tag_print { $self->add_returnsize($outputlength); } close $in_fh; + if($? and $opt::compress) { + ::error($opt::decompress_program." failed.\n"); + $self->set_exitstatus(255); + } } sub normal_print { @@ -7171,7 +7256,7 @@ sub normal_print { my $buf; close $self->fh($fdno,"w"); if($? and $opt::compress) { - ::error($opt::decompress_program." failed.\n"); + ::error($opt::compress_program." failed.\n"); $self->set_exitstatus(255); } seek $in_fh, 0, 0; @@ -7193,6 +7278,10 @@ sub normal_print { $self->add_returnsize($outputlength); } close $in_fh; + if($? and $opt::compress) { + ::error($opt::decompress_program." failed.\n"); + $self->set_exitstatus(255); + } } sub print_joblog { diff --git a/testsuite/Makefile b/testsuite/Makefile index 8308af20..cbc25f0a 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -7,7 +7,7 @@ testsuite: 3 make stopvm 1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote - time bash Start.sh - mem || true + TRIES=1 time bash Start.sh - mem || true touch ~/.parallel/will-cite make stopvm diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 60f0b83a..5d5b4b1c 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -84,6 +84,56 @@ echo '### bug #44546: If --compress-program fails: fail' parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $? parallel --compress-program false echo \;ls ::: /no-existing; echo $? +echo 'bug #41613: --compress --line-buffer - no newline'; + echo 'pipe compress tagstring' + perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat; echo "K" + echo 'pipe compress notagstring' + perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat; echo "K" + echo 'pipe nocompress tagstring' + perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat; echo "K" + echo 'pipe nocompress notagstring' + perl -e 'print "O"'| parallel --pipe --line-buffer cat; echo "K" + echo 'nopipe compress tagstring' + parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" + echo 'nopipe compress notagstring' + parallel --compress --line-buffer echo {} O ::: -n; echo "K" + echo 'nopipe nocompress tagstring' + parallel --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" + echo 'nopipe nocompress notagstring' + parallel --line-buffer echo {} O ::: -n; echo "K" + +echo 'Compress with failing (de)compressor' + parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag true true + parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag false true + parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false + parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag true false + parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: true true + parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: false true + parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false + parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: true false + parallel -k --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: line-buffer true true + parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: line-buffer false true + parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false + parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag line-buffer true false + parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag line-buffer true true + parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag line-buffer false true + parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag line-buffer false false + parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag line-buffer true false + parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: files true true | parallel rm + parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: files false true | parallel rm + parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm + parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: files true false | parallel rm + +echo 'bug #44250: pxz complains File format not recognized but decompresses anyway' + # The first line dumps core if run from make file. Why?! + stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file + stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file + stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output + stdout parallel --compress --compress-program pxz true ::: OK-if-no-output + +echo 'bug #41613: --compress --line-buffer no newline'; + perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo + echo '### bug #44614: --pipepart --header off by one' seq 10 >/tmp/parallel_44616; parallel --pipepart -a /tmp/parallel_44616 -k --block 5 'echo foo; cat'; diff --git a/testsuite/tests-to-run/parallel-local9.sh b/testsuite/tests-to-run/parallel-local9.sh index 40f3be72..70ad6924 100644 --- a/testsuite/tests-to-run/parallel-local9.sh +++ b/testsuite/tests-to-run/parallel-local9.sh @@ -8,16 +8,6 @@ NICEPAR="nice nice parallel" export NICEPAR cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1 -echo 'bug #44250: pxz complains File format not recognized but decompresses anyway' - # The first line dumps core if run from make file. Why?! - stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file - stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file - stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output - stdout parallel --compress --compress-program pxz true ::: OK-if-no-output - -echo 'bug #41613: --compress --line-buffer no newline'; - perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo - echo 'bug #41613: --compress --line-buffer no --tagstring'; diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| @@ -40,24 +30,6 @@ echo 'bug #41613: --compress --line-buffer with --tagstring'; >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' -echo 'bug #41613: --compress --line-buffer - no newline'; - echo 'pipe compress tagstring' - perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat; echo "K" - echo 'pipe compress notagstring' - perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat; echo "K" - echo 'pipe nocompress tagstring' - perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat; echo "K" - echo 'pipe nocompress notagstring' - perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat; echo "K" - echo 'nopipe compress tagstring' - $NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" - echo 'nopipe compress notagstring' - $NICEPAR --compress --line-buffer echo {} O ::: -n; echo "K" - echo 'nopipe nocompress tagstring' - $NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" - echo 'nopipe nocompress notagstring' - $NICEPAR --line-buffer echo {} O ::: -n; echo "K" - echo 'bug #41412: --timeout + --delay causes deadlock'; seq 10 | parallel -j10 --timeout 1 --delay .3 echo; parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3; diff --git a/testsuite/tests-to-run/parallel-polarhome.sh b/testsuite/tests-to-run/parallel-polarhome.sh index 85c60ffe..349f2cf7 100644 --- a/testsuite/tests-to-run/parallel-polarhome.sh +++ b/testsuite/tests-to-run/parallel-polarhome.sh @@ -16,7 +16,6 @@ echo '### Tests on polarhome machines' echo 'Setup on polarhome machines' stdout parallel -kj0 ssh -oLogLevel=quiet {} mkdir -p bin ::: $POLAR & - test_empty_cmd() { echo '### Test if empty command in process list causes problems' perl -e '$0=" ";sleep 1' & @@ -36,7 +35,8 @@ copy_and_test() { perl -pe 's:/[a-z0-9_]+.arg:/XXXXXXXX.arg:gi; s/\d\d\d\d/0000/gi;' } export -f copy_and_test -stdout parallel -j0 -k --retries 5 --timeout 80 --delay 0.1 --tag -v copy_and_test {} ::: $POLAR +# 20150414 --timeout 80 -> 40 +stdout parallel -j0 -k --retries 5 --timeout 40 --delay 0.1 --tag -v copy_and_test {} ::: $POLAR cat /tmp/test_empty_cmd rm /tmp/test_empty_cmd diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index 48d991e5..0ae97091 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -15,6 +15,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ s/zenity/zenity --timeout=12/; s:/usr/bin/time:/usr/bin/time -f %e:; s:ignored_vars:ignored_vars|sort:; + # Remove \n to join all joblogs into the previous block + s:cat /tmp/log\n:cat /tmp/log;:; # When parallelized: Sleep to make sure the abc-files are made /%head1/ and $_.="sleep .3\n\n"x10; ' | @@ -44,6 +46,10 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ s/\+ wc.*\n//; # + command_X | (Bash outputs these in random order) s/.*command_[ABC].*\n//; + # Due to multiple jobs "Second started" often ends up wrong + s/Second started\n//; + # Due to multiple jobs "tried 2" often ends up wrong + s/tried 2\n//; ' # 3+3 .par files (from --files), 1 .tms-file from tmux attach ls /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx 2>/dev/null | wc -l diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index a318ee1b..a001c4dd 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -104,30 +104,177 @@ echo '### bug #44546: If --compress-program fails: fail' ### bug #44546: If --compress-program fails: fail parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $? 1 -parallel: Error: false -dc failed. -parallel: Error: false -dc failed. +parallel: Error: false failed. +parallel: Error: false failed. parallel --tag --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $? 1 -parallel: Error: false -dc failed. -parallel: Error: false -dc failed. +parallel: Error: false failed. +parallel: Error: false failed. (parallel --files --tag --line-buffer --compress-program false echo \;sleep 1\;ls ::: /no-existing; echo $?) | tail -n1 1 -parallel: Error: false -dc failed. -parallel: Error: false -dc failed. -parallel: Error: false -dc failed. -parallel: Error: false -dc failed. +parallel: Error: false failed. +parallel: Error: false failed. +parallel: Error: false failed. +parallel: Error: false failed. parallel --tag --compress-program false echo \;ls ::: /no-existing; echo $? 1 -parallel: Error: false -dc failed. -parallel: Error: false -dc failed. +parallel: Error: false failed. +parallel: Error: false failed. parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $? 1 -parallel: Error: false -dc failed. -parallel: Error: false -dc failed. +parallel: Error: false failed. +parallel: Error: false failed. parallel --compress-program false echo \;ls ::: /no-existing; echo $? 1 -parallel: Error: false -dc failed. -parallel: Error: false -dc failed. +parallel: Error: false failed. +parallel: Error: false failed. +echo 'bug #41613: --compress --line-buffer - no newline'; echo 'pipe compress tagstring' +bug #41613: --compress --line-buffer - no newline +pipe compress tagstring + perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat; echo "K" +1 OK + echo 'pipe compress notagstring' +pipe compress notagstring + perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat; echo "K" +OK + echo 'pipe nocompress tagstring' +pipe nocompress tagstring + perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat; echo "K" +1 OK + echo 'pipe nocompress notagstring' +pipe nocompress notagstring + perl -e 'print "O"'| parallel --pipe --line-buffer cat; echo "K" +OK + echo 'nopipe compress tagstring' +nopipe compress tagstring + parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" +1 OK + echo 'nopipe compress notagstring' +nopipe compress notagstring + parallel --compress --line-buffer echo {} O ::: -n; echo "K" +OK + echo 'nopipe nocompress tagstring' +nopipe nocompress tagstring + parallel --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" +1 OK + echo 'nopipe nocompress notagstring' +nopipe nocompress notagstring + parallel --line-buffer echo {} O ::: -n; echo "K" +OK +echo 'Compress with failing (de)compressor' +Compress with failing (de)compressor + parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag true true +tag tag +true true +true true + parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag false true +tag tag +false false +true true +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. + parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false +tag tag +false false +false false +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. + parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag true false +tag tag +true true +false false + parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: true true +true +true + parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: false true +false +true +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. + parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false +false +false +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. + parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: true false +true +false + parallel -k --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: line-buffer true true +line-buffer +true +true + parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: line-buffer false true +line-buffer +false +true +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. + parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false +line-buffer +false +false +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. + parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag line-buffer true false +tag tag +line-buffer line-buffer +true true +false false + parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag line-buffer true true +tag tag +line-buffer line-buffer +true true +true true + parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag line-buffer false true +tag tag +line-buffer line-buffer +false false +true true +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. + parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag line-buffer false false +tag tag +line-buffer line-buffer +false false +false false +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. + parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag line-buffer true false +tag tag +line-buffer line-buffer +true true +false false + parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: files true true | parallel rm + parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: files false true | parallel rm +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. + parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. +parallel: Error: cat;false failed. + parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: files true false | parallel rm +echo 'bug #44250: pxz complains File format not recognized but decompresses anyway' +bug #44250: pxz complains File format not recognized but decompresses anyway + # The first line dumps core if run from make file. Why?! + stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file +Segmentation fault (core dumped) +parallel: Error: pxz failed. + stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file +ls: cannot access /OK-if-missing-file: No such file or directory + stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output + stdout parallel --compress --compress-program pxz true ::: OK-if-no-output +echo 'bug #41613: --compress --line-buffer no newline'; perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo +bug #41613: --compress --line-buffer no newline +It worked echo '### bug #44614: --pipepart --header off by one' ### bug #44614: --pipepart --header off by one seq 10 >/tmp/parallel_44616; parallel --pipepart -a /tmp/parallel_44616 -k --block 5 'echo foo; cat'; parallel --pipepart -a /tmp/parallel_44616 -k --block 2 --regexp --recend 3'\n' 'echo foo; cat'; rm /tmp/parallel_44616 diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 5b25441f..781d798f 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -1,17 +1,3 @@ -echo 'bug #44250: pxz complains File format not recognized but decompresses anyway' -bug #44250: pxz complains File format not recognized but decompresses anyway - # The first line dumps core if run from make file. Why?! - stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file -Segmentation fault (core dumped) -parallel: Error: pxz -dc failed. - stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file -ls: cannot access /OK-if-missing-file: No such file or directory - stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output - stdout parallel --compress --compress-program pxz true ::: OK-if-no-output -echo 'bug #41613: --compress --line-buffer no newline'; -bug #41613: --compress --line-buffer no newline - perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo -It worked echo 'bug #41613: --compress --line-buffer no --tagstring'; bug #41613: --compress --line-buffer no --tagstring diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' @@ -20,40 +6,6 @@ echo 'bug #41613: --compress --line-buffer with --tagstring'; bug #41613: --compress --line-buffer with --tagstring diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' Good: --line-buffer matters -echo 'bug #41613: --compress --line-buffer - no newline'; -bug #41613: --compress --line-buffer - no newline - echo 'pipe compress tagstring' -pipe compress tagstring - perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat; echo "K" -1 OK - echo 'pipe compress notagstring' -pipe compress notagstring - perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat; echo "K" -OK - echo 'pipe nocompress tagstring' -pipe nocompress tagstring - perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat; echo "K" -1 OK - echo 'pipe nocompress notagstring' -pipe nocompress notagstring - perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat; echo "K" -OK - echo 'nopipe compress tagstring' -nopipe compress tagstring - $NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" -1 OK - echo 'nopipe compress notagstring' -nopipe compress notagstring - $NICEPAR --compress --line-buffer echo {} O ::: -n; echo "K" -OK - echo 'nopipe nocompress tagstring' -nopipe nocompress tagstring - $NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" -1 OK - echo 'nopipe nocompress notagstring' -nopipe nocompress notagstring - $NICEPAR --line-buffer echo {} O ::: -n; echo "K" -OK echo 'bug #41412: --timeout + --delay causes deadlock'; bug #41412: --timeout + --delay causes deadlock seq 10 | parallel -j10 --timeout 1 --delay .3 echo; diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 9d51f862..e231f867 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -440,30 +440,30 @@ Computer:jobs running/jobs completed/%of started jobs/Average seconds to complet seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --timeout=12 --progress --auto-kill) BASE64 parallel --joblog /tmp/log exit ::: 1 2 3 0 - cat /tmp/log -Seq Host Starttime JobRuntime Send Receive Exitval Signal Command -1 : TIMESTAMP 9.999 0 0 1 0 exit 1 -2 : TIMESTAMP 9.999 0 0 2 0 exit 2 -3 : TIMESTAMP 9.999 0 0 3 0 exit 3 -4 : TIMESTAMP 9.999 0 0 0 0 exit 0 + cat /tmp/log; parallel --joblog /tmp/log exit ::: 1 2 3 0 - cat /tmp/log - parallel --resume --joblog /tmp/log exit ::: 1 2 3 0 0 0 - cat /tmp/log -Seq Host Starttime JobRuntime Send Receive Exitval Signal Command -1 : TIMESTAMP 9.999 0 0 1 0 exit 1 -2 : TIMESTAMP 9.999 0 0 2 0 exit 2 -3 : TIMESTAMP 9.999 0 0 3 0 exit 3 -4 : TIMESTAMP 9.999 0 0 0 0 exit 0 -Seq Host Starttime JobRuntime Send Receive Exitval Signal Command -1 : TIMESTAMP 9.999 0 0 1 0 exit 1 -2 : TIMESTAMP 9.999 0 0 2 0 exit 2 -3 : TIMESTAMP 9.999 0 0 3 0 exit 3 -4 : TIMESTAMP 9.999 0 0 0 0 exit 0 -5 : TIMESTAMP 9.999 0 0 0 0 exit 0 -6 : TIMESTAMP 9.999 0 0 0 0 exit 0 + cat /tmp/log; parallel --resume --joblog /tmp/log exit ::: 1 2 3 0 0 0 + cat /tmp/log; parallel --resume-failed --joblog /tmp/log exit ::: 1 2 3 0 0 0 - cat /tmp/log + cat /tmp/log; + parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3 +Seq Host Starttime JobRuntime Send Receive Exitval Signal Command +1 : TIMESTAMP 9.999 0 0 1 0 exit 1 +2 : TIMESTAMP 9.999 0 0 2 0 exit 2 +3 : TIMESTAMP 9.999 0 0 3 0 exit 3 +4 : TIMESTAMP 9.999 0 0 0 0 exit 0 +Seq Host Starttime JobRuntime Send Receive Exitval Signal Command +1 : TIMESTAMP 9.999 0 0 1 0 exit 1 +2 : TIMESTAMP 9.999 0 0 2 0 exit 2 +3 : TIMESTAMP 9.999 0 0 3 0 exit 3 +4 : TIMESTAMP 9.999 0 0 0 0 exit 0 +Seq Host Starttime JobRuntime Send Receive Exitval Signal Command +1 : TIMESTAMP 9.999 0 0 1 0 exit 1 +2 : TIMESTAMP 9.999 0 0 2 0 exit 2 +3 : TIMESTAMP 9.999 0 0 3 0 exit 3 +4 : TIMESTAMP 9.999 0 0 0 0 exit 0 +5 : TIMESTAMP 9.999 0 0 0 0 exit 0 +6 : TIMESTAMP 9.999 0 0 0 0 exit 0 Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 1 : TIMESTAMP 9.999 0 0 1 0 exit 1 2 : TIMESTAMP 9.999 0 0 2 0 exit 2 @@ -474,7 +474,6 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 1 : TIMESTAMP 9.999 0 0 1 0 exit 1 2 : TIMESTAMP 9.999 0 0 2 0 exit 2 3 : TIMESTAMP 9.999 0 0 3 0 exit 3 - parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3 0 0 1 @@ -507,12 +506,9 @@ completed 1 completed 2 completed 0 tried 1 -tried 2 tried 0 tried 1 -tried 2 tried 1 -tried 2 parallel --load 100% echo load is less than {} job per cpu ::: 1 load is less than 1 job per cpu parallel --noswap echo the system is not swapping ::: now @@ -664,6 +660,7 @@ TEXTDOMAIN TEXTDOMAINDIR TIMEOUT TMPDIR +TRIES UPSTART_EVENTS UPSTART_INSTANCE UPSTART_JOB @@ -883,14 +880,12 @@ The second finished running in the foreground First started The first finished sem --id my_id -u 'echo Second started; sleep 10; echo The second finished' -Second started sem --jobs 3 --id my_id -u 'echo First started; sleep 5; echo The first finished' && sem --jobs 3 --id my_id -u 'echo Second started; sleep 6; echo The second finished' && sem --jobs 3 --id my_id -u 'echo Third started; sleep 7; echo The third finished' && sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 8; echo The fourth finished' && sem --wait --id my_id First started -Second started The first finished Third started The second finished @@ -936,7 +931,7 @@ This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. parallel --version -GNU parallel 20150403 +GNU parallel 20150415 Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange and Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later @@ -948,7 +943,7 @@ Web site: http://www.gnu.org/software/parallel When using programs that use GNU Parallel to process data for publication please cite as described in 'parallel --bibtex'. parallel --minversion 20130722 && echo Your version is at least 20130722. -20150403 +20150415 Your version is at least 20130722. parallel --bibtex Academic tradition requires you to cite works you base your article on.