Merge tag 'det'

Conflicts:
	src/parallel
This commit is contained in:
Ole Tange 2015-04-15 22:36:52 +02:00
commit 1d1d35cfdd
10 changed files with 362 additions and 145 deletions

View file

@ -231,7 +231,7 @@ New in this release:
* GNU Parallel was cited in: Experimental study on the Wind Farm Substation Cable Installation Problem http://i11www.iti.uni-karlsruhe.de/_media/teaching/theses/ma-schmitz-14.pdf
* <<afventer>> Comparing the CarbonTracker and TM5-4DVar data assimilation systems for CO2 surface flux inversions http://www.atmos-chem-phys-discuss.net/15/8883/2015/acpd-15-8883-2015-discussion.html
* <<afventer - rykket >> Comparing the CarbonTracker and TM5-4DVar data assimilation systems for CO2 surface flux inversions http://www.atmos-chem-phys-discuss.net/15/8883/2015/acpd-15-8883-2015-discussion.html
* <<afventer opdatering>> CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf
@ -247,10 +247,16 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
* GNU Parallel was used in: http://search.cpan.org/~ajpage/Bio-Roary-2.0.7/lib/Bio/Roary/JobRunner/Parallel.pm
* GNU Parallel was used in: Rapid haploid SNP calling https://github.com/tseemann/snippy
* << afventer svar fra Rachel >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/
* Pictures and Metadata http://www.ozzy.no/2015/02/05/pictures-and-metadata/
* Task automation with bash and parallel https://biowize.wordpress.com/2015/03/23/task-automation-with-bash-and-parallel/
* How To: Speed Up File Transfers in Linux using RSync with GNU Parallel http://www.yourownlinux.com/2015/04/speed-up-file-transfers-using-rsync-with-gnu-parallel.html
* Mirroring a Git Repository https://avacariu.me/articles/mirroring-a-git-repository.html
* Add an ENVI header to JAXA Global Mangrove Watch PALSAR tiles https://spectraldifferences.wordpress.com/2015/03/24/add-an-envi-header-to-jaxa-global-mangrove-watch-palsar-tiles/

View file

@ -1051,7 +1051,7 @@ sub parse_options {
sub init_globals {
# Defaults:
$Global::version = 20150409;
$Global::version = 20150416;
$Global::progname = 'parallel';
$Global::infinity = 2**31;
$Global::debug = 0;
@ -2986,7 +2986,7 @@ sub reaper {
$job or next;
$job->set_exitstatus($? >> 8);
$job->set_exitsignal($? & 127);
debug("run", "died (", $job->exitstatus(), "): ", $job->seq());
debug("run", "seq ",$job->seq()," died (", $job->exitstatus(), ")");
$job->set_endtime(::now());
if($stiff == $Global::tty_taken) {
# The process that died had the tty => release it
@ -3031,6 +3031,72 @@ sub reaper {
sub __USAGE__ {}
sub killall {
# Kill all jobs
# Send all jobs TERM
# Wait
# Send all jobs TERM
# Wait
# Send all jobs KILL
# Send all (grand*)children KILL
$Global::start_no_new_jobs ||= 1;
# pids of the all children and (grand*)children
# before we start the blood bath
my @family_pids = family_pids(keys %Global::running);
# Send jobs TERM
kill "TERM", keys %Global::running;
# Wait up to 200 ms
# Send jobs TERM (again)
my $sleepsum = 0;
my $sleep = 0;
for (; kill(0, keys %Global::running) and $sleepsum < 200;
$sleepsum += $sleep) {
# This can change %Global::running
$sleep = ::reap_usleep($sleep);
}
kill "TERM", keys %Global::running;
# Wait up to 200 ms
# Send jobs KILL
$sleepsum = 0;
$sleep = 0;
for (; kill(0, keys %Global::running) and $sleepsum < 200;
$sleepsum += $sleep) {
# This can change %Global::running
$sleep = ::reap_usleep($sleep);
}
kill "KILL", keys %Global::running;
# Send all (grand*)children KILL (if there are any left)
kill "KILL", @family_pids;
}
sub family_pids {
# Find the pids with this->pid as (grand)*parent
# Input:
# @parents = pids of parents
# Returns:
# @pids = pids of (grand)*children
my @parents = @_;
my @pids;
my ($children_of_ref, $parent_of_ref, $name_of_ref) = ::pid_table();
my @more = @parents;
# While more (grand)*children
while(@more) {
my @m;
push @pids, @more;
for my $parent (@more) {
if($children_of_ref->{$parent}) {
# add the children of this parent
push @m, @{$children_of_ref->{$parent}};
}
}
@more = @m;
}
return (@pids);
}
sub wait_and_exit {
# If we do not wait, we sometimes get segfault
# Returns: N/A
@ -3038,9 +3104,7 @@ sub wait_and_exit {
unlink keys %Global::unlink;
if($error) {
# Kill all without printing
for my $job (values %Global::running) {
$job->kill();
}
killall();
}
for (keys %Global::unkilled_children) {
kill 9, $_;
@ -3359,13 +3423,14 @@ sub multiply_binary_prefix {
}
{
my ($disk_full_fh, $b8193, $name);
my ($disk_full_fh, $b8193, $error_printed);
sub exit_if_disk_full {
# Checks if $TMPDIR is full by writing 8kb to a tmpfile
# If the disk is full: Exit immediately.
# Returns:
# N/A
if(not $disk_full_fh) {
my $name;
($disk_full_fh, $name) = ::tmpfile(SUFFIX => ".df");
# Separate unlink due to NFS dealing badly with File::Temp
unlink $name;
@ -3392,8 +3457,11 @@ sub multiply_binary_prefix {
or
tell $disk_full_fh != 8193) {
# On raspbian the disk can be full except for 10 chars.
::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n");
::error("Change \$TMPDIR with --tmpdir or use --compress.\n");
if(not $error_printed) {
::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n");
::error("Change \$TMPDIR with --tmpdir or use --compress.\n");
$error_printed = 1;
}
::wait_and_exit(255);
}
truncate $disk_full_fh, 0;
@ -3507,6 +3575,9 @@ sub which {
# Filter for SysV-style `ps`
my $sysv = q( ps -ef | perl -ane '1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;).
q(s/^.{$s}//; print "@F[1,2] $_"' );
# Crazy msys: ' is not accepted on the cmd line, but " are treated as '
my $msys = q( ps -ef | perl -ane "1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;).
q(s/^.{$s}//; print qq{@F[1,2] $_}" );
# BSD-style `ps`
my $bsd = q(ps -o pid,ppid,command -ax);
%pid_parentpid_cmd =
@ -3521,7 +3592,7 @@ sub which {
'hpux' => $sysv,
'linux' => $sysv,
'mirbsd' => $bsd,
'msys' => $sysv,
'msys' => $msys,
'MSWin32' => $sysv,
'netbsd' => $bsd,
'nto' => $sysv,
@ -4441,8 +4512,10 @@ sub compute_number_of_processes {
# The child takes one process slot
# It will be killed later
$SIG{TERM} = $Global::original_sig{TERM};
sleep 10000000;
exit(0);
# Exec 'sleep' to save RAM
# 32000 seconds should be plenty, and should be supported
# on all platforms.
exec("sleep","32000");
} else {
# Failed to spawn
$max_system_proc_reached = 1;
@ -5702,8 +5775,8 @@ sub filter_through_compress {
# The tmpfile is used to tell the reader that the writer has started,
# so unlink it to start with.
unlink $self->fh($fdno,'name');
my $wpid = open(my $fdw,"|-", empty_input_detector().
"| ($opt::compress_program) >>".
my $wpid = open(my $fdw,"|-", "(".empty_input_detector().
"| ($opt::compress_program)) >>".
$self->fh($fdno,'name')) || die $?;
$self->set_fh($fdno,'w',$fdw);
$self->set_fh($fdno,'wpid',$wpid);
@ -5927,6 +6000,7 @@ sub kill {
my $self = shift;
my @signals = @_;
my @family_pids = $self->family_pids();
# Record this jobs as failed
$self->set_exitstatus(-1);
# Send two TERMs to give time to clean up
@ -6999,7 +7073,7 @@ sub print {
}
next;
}
::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):");
::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):\n");
if($opt::files) {
$self->files_print($fdno,$in_fh,$out_fd);
} elsif($opt::linebuffer) {
@ -7024,14 +7098,18 @@ sub files_print {
my $self = shift;
my ($fdno,$in_fh,$out_fd) = @_;
# If --compress: $in_fh must be closed first.
# If the job is dead: close printing fh. Needed for --compress
close $self->fh($fdno,"w");
if($? and $opt::compress) {
::error($opt::decompress_program." failed.\n");
::error($opt::compress_program." failed.\n");
$self->set_exitstatus(255);
}
if($opt::compress) {
# Kill the decompressor which will not be needed
CORE::kill "TERM", $self->fh($fdno,"rpid");
}
close $in_fh;
# TODO if $?: die(?)
if($opt::pipe and $self->virgin()) {
# Nothing was printed to this job:
# cleanup unused tmp files if --files was set
@ -7054,7 +7132,7 @@ sub linebuffer_print {
# If the job is dead: close printing fh. Needed for --compress
close $self->fh($fdno,"w");
if($? and $opt::compress) {
::error($opt::decompress_program." failed.\n");
::error($opt::compress_program." failed.\n");
$self->set_exitstatus(255);
}
if($opt::compress) {
@ -7123,7 +7201,10 @@ sub linebuffer_print {
} else {
# decompress done: close fh
close $in_fh;
# TODO if $?: die(?)
if($? and $opt::compress) {
::error($opt::decompress_program." failed.\n");
$self->set_exitstatus(255);
}
}
}
}
@ -7134,7 +7215,7 @@ sub tag_print {
my $buf;
close $self->fh($fdno,"w");
if($? and $opt::compress) {
::error($opt::decompress_program." failed.\n");
::error($opt::compress_program." failed.\n");
$self->set_exitstatus(255);
}
seek $in_fh, 0, 0;
@ -7163,6 +7244,10 @@ sub tag_print {
$self->add_returnsize($outputlength);
}
close $in_fh;
if($? and $opt::compress) {
::error($opt::decompress_program." failed.\n");
$self->set_exitstatus(255);
}
}
sub normal_print {
@ -7171,7 +7256,7 @@ sub normal_print {
my $buf;
close $self->fh($fdno,"w");
if($? and $opt::compress) {
::error($opt::decompress_program." failed.\n");
::error($opt::compress_program." failed.\n");
$self->set_exitstatus(255);
}
seek $in_fh, 0, 0;
@ -7193,6 +7278,10 @@ sub normal_print {
$self->add_returnsize($outputlength);
}
close $in_fh;
if($? and $opt::compress) {
::error($opt::decompress_program." failed.\n");
$self->set_exitstatus(255);
}
}
sub print_joblog {

View file

@ -7,7 +7,7 @@ testsuite: 3
make stopvm
1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
time bash Start.sh - mem || true
TRIES=1 time bash Start.sh - mem || true
touch ~/.parallel/will-cite
make stopvm

View file

@ -84,6 +84,56 @@ echo '### bug #44546: If --compress-program fails: fail'
parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
parallel --compress-program false echo \;ls ::: /no-existing; echo $?
echo 'bug #41613: --compress --line-buffer - no newline';
echo 'pipe compress tagstring'
perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat; echo "K"
echo 'pipe compress notagstring'
perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat; echo "K"
echo 'pipe nocompress tagstring'
perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat; echo "K"
echo 'pipe nocompress notagstring'
perl -e 'print "O"'| parallel --pipe --line-buffer cat; echo "K"
echo 'nopipe compress tagstring'
parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K"
echo 'nopipe compress notagstring'
parallel --compress --line-buffer echo {} O ::: -n; echo "K"
echo 'nopipe nocompress tagstring'
parallel --tagstring {#} --line-buffer echo {} O ::: -n; echo "K"
echo 'nopipe nocompress notagstring'
parallel --line-buffer echo {} O ::: -n; echo "K"
echo 'Compress with failing (de)compressor'
parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag true true
parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag false true
parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false
parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag true false
parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: true true
parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: false true
parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false
parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: true false
parallel -k --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: line-buffer true true
parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: line-buffer false true
parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false
parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag line-buffer true false
parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag line-buffer true true
parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag line-buffer false true
parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag line-buffer false false
parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag line-buffer true false
parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: files true true | parallel rm
parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: files false true | parallel rm
parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm
parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: files true false | parallel rm
echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
# The first line dumps core if run from make file. Why?!
stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file
stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
echo 'bug #41613: --compress --line-buffer no newline';
perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo
echo '### bug #44614: --pipepart --header off by one'
seq 10 >/tmp/parallel_44616;
parallel --pipepart -a /tmp/parallel_44616 -k --block 5 'echo foo; cat';

View file

@ -8,16 +8,6 @@ NICEPAR="nice nice parallel"
export NICEPAR
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1
echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
# The first line dumps core if run from make file. Why?!
stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file
stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
echo 'bug #41613: --compress --line-buffer no newline';
perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo
echo 'bug #41613: --compress --line-buffer no --tagstring';
diff
<(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'|
@ -40,24 +30,6 @@ echo 'bug #41613: --compress --line-buffer with --tagstring';
>/dev/null
|| (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working'
echo 'bug #41613: --compress --line-buffer - no newline';
echo 'pipe compress tagstring'
perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat; echo "K"
echo 'pipe compress notagstring'
perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat; echo "K"
echo 'pipe nocompress tagstring'
perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat; echo "K"
echo 'pipe nocompress notagstring'
perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat; echo "K"
echo 'nopipe compress tagstring'
$NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K"
echo 'nopipe compress notagstring'
$NICEPAR --compress --line-buffer echo {} O ::: -n; echo "K"
echo 'nopipe nocompress tagstring'
$NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n; echo "K"
echo 'nopipe nocompress notagstring'
$NICEPAR --line-buffer echo {} O ::: -n; echo "K"
echo 'bug #41412: --timeout + --delay causes deadlock';
seq 10 | parallel -j10 --timeout 1 --delay .3 echo;
parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3;

View file

@ -16,7 +16,6 @@ echo '### Tests on polarhome machines'
echo 'Setup on polarhome machines'
stdout parallel -kj0 ssh -oLogLevel=quiet {} mkdir -p bin ::: $POLAR &
test_empty_cmd() {
echo '### Test if empty command in process list causes problems'
perl -e '$0=" ";sleep 1' &
@ -36,7 +35,8 @@ copy_and_test() {
perl -pe 's:/[a-z0-9_]+.arg:/XXXXXXXX.arg:gi; s/\d\d\d\d/0000/gi;'
}
export -f copy_and_test
stdout parallel -j0 -k --retries 5 --timeout 80 --delay 0.1 --tag -v copy_and_test {} ::: $POLAR
# 20150414 --timeout 80 -> 40
stdout parallel -j0 -k --retries 5 --timeout 40 --delay 0.1 --tag -v copy_and_test {} ::: $POLAR
cat /tmp/test_empty_cmd
rm /tmp/test_empty_cmd

View file

@ -15,6 +15,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
s/zenity/zenity --timeout=12/;
s:/usr/bin/time:/usr/bin/time -f %e:;
s:ignored_vars:ignored_vars|sort:;
# Remove \n to join all joblogs into the previous block
s:cat /tmp/log\n:cat /tmp/log;:;
# When parallelized: Sleep to make sure the abc-files are made
/%head1/ and $_.="sleep .3\n\n"x10;
' |
@ -44,6 +46,10 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
s/\+ wc.*\n//;
# + command_X | (Bash outputs these in random order)
s/.*command_[ABC].*\n//;
# Due to multiple jobs "Second started" often ends up wrong
s/Second started\n//;
# Due to multiple jobs "tried 2" often ends up wrong
s/tried 2\n//;
'
# 3+3 .par files (from --files), 1 .tms-file from tmux attach
ls /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx 2>/dev/null | wc -l

View file

@ -104,30 +104,177 @@ echo '### bug #44546: If --compress-program fails: fail'
### bug #44546: If --compress-program fails: fail
parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
1
parallel: Error: false -dc failed.
parallel: Error: false -dc failed.
parallel: Error: false failed.
parallel: Error: false failed.
parallel --tag --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
1
parallel: Error: false -dc failed.
parallel: Error: false -dc failed.
parallel: Error: false failed.
parallel: Error: false failed.
(parallel --files --tag --line-buffer --compress-program false echo \;sleep 1\;ls ::: /no-existing; echo $?) | tail -n1
1
parallel: Error: false -dc failed.
parallel: Error: false -dc failed.
parallel: Error: false -dc failed.
parallel: Error: false -dc failed.
parallel: Error: false failed.
parallel: Error: false failed.
parallel: Error: false failed.
parallel: Error: false failed.
parallel --tag --compress-program false echo \;ls ::: /no-existing; echo $?
1
parallel: Error: false -dc failed.
parallel: Error: false -dc failed.
parallel: Error: false failed.
parallel: Error: false failed.
parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
1
parallel: Error: false -dc failed.
parallel: Error: false -dc failed.
parallel: Error: false failed.
parallel: Error: false failed.
parallel --compress-program false echo \;ls ::: /no-existing; echo $?
1
parallel: Error: false -dc failed.
parallel: Error: false -dc failed.
parallel: Error: false failed.
parallel: Error: false failed.
echo 'bug #41613: --compress --line-buffer - no newline'; echo 'pipe compress tagstring'
bug #41613: --compress --line-buffer - no newline
pipe compress tagstring
perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat; echo "K"
1 OK
echo 'pipe compress notagstring'
pipe compress notagstring
perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat; echo "K"
OK
echo 'pipe nocompress tagstring'
pipe nocompress tagstring
perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat; echo "K"
1 OK
echo 'pipe nocompress notagstring'
pipe nocompress notagstring
perl -e 'print "O"'| parallel --pipe --line-buffer cat; echo "K"
OK
echo 'nopipe compress tagstring'
nopipe compress tagstring
parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K"
1 OK
echo 'nopipe compress notagstring'
nopipe compress notagstring
parallel --compress --line-buffer echo {} O ::: -n; echo "K"
OK
echo 'nopipe nocompress tagstring'
nopipe nocompress tagstring
parallel --tagstring {#} --line-buffer echo {} O ::: -n; echo "K"
1 OK
echo 'nopipe nocompress notagstring'
nopipe nocompress notagstring
parallel --line-buffer echo {} O ::: -n; echo "K"
OK
echo 'Compress with failing (de)compressor'
Compress with failing (de)compressor
parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag true true
tag tag
true true
true true
parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag false true
tag tag
false false
true true
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false
tag tag
false false
false false
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag true false
tag tag
true true
false false
parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: true true
true
true
parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: false true
false
true
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false
false
false
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: true false
true
false
parallel -k --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: line-buffer true true
line-buffer
true
true
parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: line-buffer false true
line-buffer
false
true
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false
line-buffer
false
false
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag line-buffer true false
tag tag
line-buffer line-buffer
true true
false false
parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag line-buffer true true
tag tag
line-buffer line-buffer
true true
true true
parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag line-buffer false true
tag tag
line-buffer line-buffer
false false
true true
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag line-buffer false false
tag tag
line-buffer line-buffer
false false
false false
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag line-buffer true false
tag tag
line-buffer line-buffer
true true
false false
parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: files true true | parallel rm
parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: files false true | parallel rm
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel: Error: cat;false failed.
parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: files true false | parallel rm
echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
bug #44250: pxz complains File format not recognized but decompresses anyway
# The first line dumps core if run from make file. Why?!
stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
Segmentation fault (core dumped)
parallel: Error: pxz failed.
stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file
ls: cannot access /OK-if-missing-file: No such file or directory
stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
echo 'bug #41613: --compress --line-buffer no newline'; perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo
bug #41613: --compress --line-buffer no newline
It worked
echo '### bug #44614: --pipepart --header off by one'
### bug #44614: --pipepart --header off by one
seq 10 >/tmp/parallel_44616; parallel --pipepart -a /tmp/parallel_44616 -k --block 5 'echo foo; cat'; parallel --pipepart -a /tmp/parallel_44616 -k --block 2 --regexp --recend 3'\n' 'echo foo; cat'; rm /tmp/parallel_44616

View file

@ -1,17 +1,3 @@
echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
bug #44250: pxz complains File format not recognized but decompresses anyway
# The first line dumps core if run from make file. Why?!
stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
Segmentation fault (core dumped)
parallel: Error: pxz -dc failed.
stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file
ls: cannot access /OK-if-missing-file: No such file or directory
stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
echo 'bug #41613: --compress --line-buffer no newline';
bug #41613: --compress --line-buffer no newline
perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo
It worked
echo 'bug #41613: --compress --line-buffer no --tagstring';
bug #41613: --compress --line-buffer no --tagstring
diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working'
@ -20,40 +6,6 @@ echo 'bug #41613: --compress --line-buffer with --tagstring';
bug #41613: --compress --line-buffer with --tagstring
diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working'
Good: --line-buffer matters
echo 'bug #41613: --compress --line-buffer - no newline';
bug #41613: --compress --line-buffer - no newline
echo 'pipe compress tagstring'
pipe compress tagstring
perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat; echo "K"
1 OK
echo 'pipe compress notagstring'
pipe compress notagstring
perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat; echo "K"
OK
echo 'pipe nocompress tagstring'
pipe nocompress tagstring
perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat; echo "K"
1 OK
echo 'pipe nocompress notagstring'
pipe nocompress notagstring
perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat; echo "K"
OK
echo 'nopipe compress tagstring'
nopipe compress tagstring
$NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K"
1 OK
echo 'nopipe compress notagstring'
nopipe compress notagstring
$NICEPAR --compress --line-buffer echo {} O ::: -n; echo "K"
OK
echo 'nopipe nocompress tagstring'
nopipe nocompress tagstring
$NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n; echo "K"
1 OK
echo 'nopipe nocompress notagstring'
nopipe nocompress notagstring
$NICEPAR --line-buffer echo {} O ::: -n; echo "K"
OK
echo 'bug #41412: --timeout + --delay causes deadlock';
bug #41412: --timeout + --delay causes deadlock
seq 10 | parallel -j10 --timeout 1 --delay .3 echo;

View file

@ -440,30 +440,30 @@ Computer:jobs running/jobs completed/%of started jobs/Average seconds to complet
seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --timeout=12 --progress --auto-kill)
BASE64 parallel --joblog /tmp/log exit ::: 1 2 3 0
cat /tmp/log
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
cat /tmp/log;
parallel --joblog /tmp/log exit ::: 1 2 3 0
cat /tmp/log
parallel --resume --joblog /tmp/log exit ::: 1 2 3 0 0 0
cat /tmp/log
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
5 : TIMESTAMP 9.999 0 0 0 0 exit 0
6 : TIMESTAMP 9.999 0 0 0 0 exit 0
cat /tmp/log; parallel --resume --joblog /tmp/log exit ::: 1 2 3 0 0 0
cat /tmp/log;
parallel --resume-failed --joblog /tmp/log exit ::: 1 2 3 0 0 0
cat /tmp/log
cat /tmp/log;
parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
5 : TIMESTAMP 9.999 0 0 0 0 exit 0
6 : TIMESTAMP 9.999 0 0 0 0 exit 0
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
@ -474,7 +474,6 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3
0
0
1
@ -507,12 +506,9 @@ completed 1
completed 2
completed 0
tried 1
tried 2
tried 0
tried 1
tried 2
tried 1
tried 2
parallel --load 100% echo load is less than {} job per cpu ::: 1
load is less than 1 job per cpu
parallel --noswap echo the system is not swapping ::: now
@ -664,6 +660,7 @@ TEXTDOMAIN
TEXTDOMAINDIR
TIMEOUT
TMPDIR
TRIES
UPSTART_EVENTS
UPSTART_INSTANCE
UPSTART_JOB
@ -883,14 +880,12 @@ The second finished running in the foreground
First started
The first finished
sem --id my_id -u 'echo Second started; sleep 10; echo The second finished'
Second started
sem --jobs 3 --id my_id -u 'echo First started; sleep 5; echo The first finished' &&
sem --jobs 3 --id my_id -u 'echo Second started; sleep 6; echo The second finished' &&
sem --jobs 3 --id my_id -u 'echo Third started; sleep 7; echo The third finished' &&
sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 8; echo The fourth finished' &&
sem --wait --id my_id
First started
Second started
The first finished
Third started
The second finished
@ -936,7 +931,7 @@ This helps funding further development; and it won't cost you a cent.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
parallel --version
GNU parallel 20150403
GNU parallel 20150415
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange
and Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
@ -948,7 +943,7 @@ Web site: http://www.gnu.org/software/parallel
When using programs that use GNU Parallel to process data for publication
please cite as described in 'parallel --bibtex'.
parallel --minversion 20130722 && echo Your version is at least 20130722.
20150403
20150415
Your version is at least 20130722.
parallel --bibtex
Academic tradition requires you to cite works you base your article on.