tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 05:57:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed bug #51299: --retry-failed with command with newline.

Browse source
This commit is contained in:
Ole Tange 2017-06-23 23:00:05 +02:00
parent 215aa6039f
commit 144164d516
14 changed files with 1592 additions and 1528 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
doc/release_new_version
View file

@ -196,9 +196,9 @@ file:///home/tange/privat/parallel/doc/release_new_version
from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
Subject: GNU Parallel 20170622 ('Manchester') released <<[stable]>>
Subject: GNU Parallel 20170622 ('Grenfell') released <<[stable]>>
GNU Parallel 20170622 ('Manchester') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
GNU Parallel 20170622 ('Grenfell') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>>

22
src/parallel
View file

@ -1,10 +1,5 @@
#!/usr/bin/env perl
# XDG_RUNTIME_DIR/parallel = TMPDIR
# XDG_CONFIG_HOME/parallel = ~/.parallel/config
# XDG_CONFIG_DIRS
# XDG_CACHE_HOME = ~/parallel/tmp + semaphores
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017
# Ole Tange and Free Software Foundation, Inc.
#
@ -928,7 +923,7 @@ sub options_hash {
"linebuffer|linebuffered|line-buffer|line-buffered|lb" => \$opt::linebuffer,
"tmux" => \$opt::tmux,
"tmuxpane" => \$opt::tmuxpane,
"null|0" => \$opt::0,
"null|0" => \$opt::null,
"quote|q" => \$opt::q,
# Replacement strings
"parens=s" => \$opt::parens,
@ -1113,7 +1108,7 @@ sub parse_options {
$Global::cshell = $Global::shell =~ m:/csh:;
if(defined $opt::X) { $Global::ContextReplace = 1; }
if(defined $opt::silent) { $Global::verbose = 0; }
if(defined $opt::0) { $/ = "\0"; }
if(defined $opt::null) { $/ = "\0"; }
if(defined $opt::d) { $/ = unquote_printf($opt::d) }
if(defined $opt::tagstring) {
$opt::tagstring = unquote_printf($opt::tagstring);
@ -1263,7 +1258,7 @@ sub parse_options {
if($opt::max_lines eq "-0") {
# -l -0 (swallowed -0)
$opt::max_lines = 1;
$opt::0 = 1;
$opt::null = 1;
$/ = "\0";
} elsif ($opt::max_lines == 0) {
# If not given (or if 0 is given) => 1
@ -1697,8 +1692,14 @@ sub open_joblog {
my ($outfh,$name) = ::tmpfile(SUFFIX => ".arg");
unlink($name);
# Put args into argfile
# Replace \0 with ' ' as used in print_joblog()
print $outfh map { s/\0/ /g; $_,$/ } @group;
if(grep /\0/, @group) {
# force --null to deal with \n in commandlines
::warning("Command lines contain newline. Forcing --null.");
$opt::null = 1;
$/ = "\0";
}
# Replace \0 with '\n' as used in print_joblog()
print $outfh map { s/\0/\n/g; $_,$/ } @group;
seek $outfh, 0, 0;
exit_if_disk_full();
# Set filehandle to -a
@ -4108,6 +4109,7 @@ sub citation {
"}",
"",
"(Feel free to use \\nocite{Tange2011a})",
"",
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
"",

1
testsuite/Makefile
View file

@ -44,7 +44,6 @@ prereqlocal: installparallel
echo | pv -qL 10 || (echo pv is required for testsuite; /bin/false)
echo | script -c echo -q /dev/null || (echo script is required for testsuite; /bin/false)
niceload true || (echo niceload is required for testsuite; /bin/false)
which cpuburn || (echo cpuburn is required for testsuite; /bin/false)
which timeout || (echo timeout is required for testsuite; /bin/false)
which autossh || (echo autossh is required for testsuite; /bin/false)
which lsh || (echo lsh is required for testsuite; /bin/false)

25
testsuite/tests-to-run/parallel-local-3s.sh
View file

@ -127,5 +127,30 @@ par_sqlworker_hostname() {
perl -pe "s/$hostname/<hostname>/g"
}
par_commandline_with_newline() {
echo 'bug #51299: --retry-failed with command with newline'
echo 'The format must remain the same'
(
parallel --jl - 'false "command
with
newlines"' ::: a b | sort
echo resume
parallel --resume --jl - 'false "command
with
newlines"' ::: a b c | sort
echo resume-failed
parallel --resume-failed --jl - 'false "command
with
newlines"' ::: a b c d | sort
echo retry-failed
parallel --retry-failed --jl - 'false "command
with
newlines"' ::: a b c d e | sort
) | perl -pe 's/\0/<null>/g;s/\d+/./g'
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | sort | parallel -j6 --tag -k '{} 2>&1'

54
testsuite/tests-to-run/parallel-local-race01.sh
View file

@ -1,37 +1,5 @@
#!/bin/bash
par_hostgroup() {
echo '### --hostgroup force ncpu'
parallel --delay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.4{} ::: {1..8} | sort
echo '### --hostgroup two group arg'
parallel -k --sshdelay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.3{} ::: {1..8}@g1+g2 | sort
echo '### --hostgroup one group arg'
parallel --delay 0.2 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.4{} ::: {1..8}@g2
echo '### --hostgroup multiple group arg + unused group'
parallel --delay 0.2 --hgrp -S @g1/1/parallel@lo -S @g1/3/lo -S @g3/100/tcsh@lo whoami\;sleep 0.8{} ::: {1..8}@g1+g2 | sort
echo '### --hostgroup two groups @'
parallel -k --hgrp -S @g1/parallel@lo -S @g2/lo --tag whoami\;echo ::: parallel@g1 tange@g2
echo '### --hostgroup'
parallel -k --hostgroup -S @grp1/lo echo ::: no_group explicit_group@grp1 implicit_group@lo
echo '### --hostgroup --sshlogin with @'
parallel -k --hostgroups -S parallel@lo echo ::: no_group implicit_group@parallel@lo
echo '### --hostgroup -S @group'
parallel -S @g1/ -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
echo '### --hostgroup -S @group1 -Sgrp2'
parallel -S @g1/ -S @g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
echo '### --hostgroup -S @group1+grp2'
parallel -S @g1+g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
}
par_distribute_input_by_ability() {
echo "### bug #48290: round-robin does not distribute data based on business"
echo "### Distribute input to jobs that are ready"
@ -59,28 +27,6 @@ par_print_before_halt_on_error() {
parallel -j0 -k --tag mytest ::: -2 -1 0 1 2
}
par_testhalt() {
testhalt_false() {
echo '### testhalt --halt '$1;
(yes 0 | head -n 10; seq 10) |
stdout parallel -kj4 --delay 0.23 --halt $1 \
'echo job {#}; sleep {= $_=0.3*($_+1+seq()) =}; exit {}'; echo $?;
}
testhalt_true() {
echo '### testhalt --halt '$1;
(seq 10; yes 0 | head -n 10) |
stdout parallel -kj4 --delay 0.17 --halt $1 \
'echo job {#}; sleep {= $_=0.3*($_+1+seq()) =}; exit {}'; echo $?;
};
export -f testhalt_false;
export -f testhalt_true;
stdout parallel -kj0 --delay 0.11 --tag testhalt_{4} {1},{2}={3} \
::: now soon ::: fail success done ::: 0 1 2 30% 70% ::: true false |
# Remove lines that only show up now and then
perl -ne '/Starting no more jobs./ or print'
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | sort |
# parallel --joblog /tmp/jl-`basename $0` -j10 --tag -k '{} 2>&1'

89
testsuite/tests-to-run/parallel-local-race02.sh Normal file
View file

@ -0,0 +1,89 @@
#!/bin/bash
# These fail regularly
#par_ctrlz_should_suspend_children() {
echo 'bug #46120: Suspend should suspend (at least local) children'
echo 'it should burn 1.9 CPU seconds, but no more than that'
echo 'The 5 second sleep will make it be killed by timeout when it fgs'
stdout bash -i -c 'stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | grep -q CPUTIME=1 &
sleep 1.9;
kill -TSTP -$!;
sleep 5;
fg;
echo Zero=OK $?' | grep -v '\[1\]' | grep -v 'SHA256'
stdout bash -i -c 'echo 1 | stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" | grep -q CPUTIME=1 &
sleep 1.9;
kill -TSTP -$!;
sleep 5;
fg;
echo Zero=OK $?' | grep -v '\[1\]' | grep -v 'SHA256'
echo Control case: Burn for 2.9 seconds
stdout bash -i -c 'stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | grep -q CPUTIME=1 &
sleep 2.9;
kill -TSTP -$!;
sleep 5;
fg;
echo 1=OK $?' | grep -v '\[1\]' | grep -v 'SHA256'
#}
par_testhalt() {
testhalt_false() {
echo '### testhalt --halt '$1;
(yes 0 | head -n 10; seq 10) |
stdout parallel -kj4 --delay 0.23 --halt $1 \
'echo job {#}; sleep {= $_=0.3*($_+1+seq()) =}; exit {}'; echo $?;
}
testhalt_true() {
echo '### testhalt --halt '$1;
(seq 10; yes 0 | head -n 10) |
stdout parallel -kj4 --delay 0.17 --halt $1 \
'echo job {#}; sleep {= $_=0.3*($_+1+seq()) =}; exit {}'; echo $?;
};
export -f testhalt_false;
export -f testhalt_true;
stdout parallel -kj0 --delay 0.11 --tag testhalt_{4} {1},{2}={3} \
::: now soon ::: fail success done ::: 0 1 2 30% 70% ::: true false |
# Remove lines that only show up now and then
perl -ne '/Starting no more jobs./ or print'
}
par_hostgroup() {
echo '### --hostgroup force ncpu'
parallel --delay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.4{} ::: {1..8} | sort
echo '### --hostgroup two group arg'
parallel -k --sshdelay 0.1 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.3{} ::: {1..8}@g1+g2 | sort
echo '### --hostgroup one group arg'
parallel --delay 0.2 --hgrp -S @g1/1/parallel@lo -S @g2/3/lo whoami\;sleep 0.4{} ::: {1..8}@g2
echo '### --hostgroup multiple group arg + unused group'
parallel --delay 0.2 --hgrp -S @g1/1/parallel@lo -S @g1/3/lo -S @g3/100/tcsh@lo whoami\;sleep 0.8{} ::: {1..8}@g1+g2 | sort
echo '### --hostgroup two groups @'
parallel -k --hgrp -S @g1/parallel@lo -S @g2/lo --tag whoami\;echo ::: parallel@g1 tange@g2
echo '### --hostgroup'
parallel -k --hostgroup -S @grp1/lo echo ::: no_group explicit_group@grp1 implicit_group@lo
echo '### --hostgroup --sshlogin with @'
parallel -k --hostgroups -S parallel@lo echo ::: no_group implicit_group@parallel@lo
echo '### --hostgroup -S @group'
parallel -S @g1/ -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
echo '### --hostgroup -S @group1 -Sgrp2'
parallel -S @g1/ -S @g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
echo '### --hostgroup -S @group1+grp2'
parallel -S @g1+g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | sort |
# parallel --joblog /tmp/jl-`basename $0` -j10 --tag -k '{} 2>&1'
parallel --joblog /tmp/jl-`basename $0` -j1 --tag -k '{} 2>&1'

26
testsuite/tests-to-run/parallel-local4.sh
View file

@ -1,30 +1,4 @@
#!/bin/bash
echo 'bug #46120: Suspend should suspend (at least local) children'
echo 'it should burn 1.9 CPU seconds, but no more than that'
echo 'The 5 second sleep will make it be killed by timeout when it fgs'
stdout bash -i -c 'stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | grep -q CPUTIME=1 &
sleep 1.9;
kill -TSTP -$!;
sleep 5;
fg;
echo Zero=OK $?' | grep -v '\[1\]' | grep -v 'SHA256'
stdout bash -i -c 'echo 1 | stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" | grep -q CPUTIME=1 &
sleep 1.9;
kill -TSTP -$!;
sleep 5;
fg;
echo Zero=OK $?' | grep -v '\[1\]' | grep -v 'SHA256'
echo Control case: Burn for 2.9 seconds
stdout bash -i -c 'stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | grep -q CPUTIME=1 &
sleep 2.9;
kill -TSTP -$!;
sleep 5;
fg;
echo 1=OK $?' | grep -v '\[1\]' | grep -v 'SHA256'
cat <<'EOF' | sed -e 's/;$/; /;' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
echo '### -L -n with pipe'

24
testsuite/wanted-results/parallel-local-3s
View file

@ -4,6 +4,29 @@ par_children_receive_sig Got TERM
par_children_receive_sig Got TERM
par_children_receive_sig Got INT
par_children_receive_sig Got TERM
par_commandline_with_newline bug #51299: --retry-failed with command with newline
par_commandline_with_newline The format must remain the same
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" a
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" b
par_commandline_with_newline Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
par_commandline_with_newline resume
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" a
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" b
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" c
par_commandline_with_newline Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
par_commandline_with_newline resume-failed
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" a
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" b
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" c
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" d
par_commandline_with_newline Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
par_commandline_with_newline retry-failed
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" a
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" b
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" c
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" d
par_commandline_with_newline . : ... ... . . . . false "command<null>with<null>newlines" e
par_commandline_with_newline Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
par_dryrun_timeout_ungroup bug #51039: --dry-run --timeout 3600 -u breaks
par_dryrun_timeout_ungroup 1000 2000 8893
par_keeporder_roundrobin bug #50081: --keep-order --round-robin should give predictable results
@ -26,6 +49,7 @@ par_multiline_commands 1
par_multiline_commands finish 1
par_multiline_commands 2
par_multiline_commands finish 2
par_multiline_commands parallel: Warning: Command lines contain newline. Forcing --null.
par_multiline_commands 3
par_multiline_commands finish 3
par_pipepart_block ### --pipepart --block -# (# < 0)

1409
testsuite/wanted-results/parallel-local-race01
View file

File diff suppressed because it is too large Load diff

1428
testsuite/wanted-results/parallel-local-race02 Normal file
View file

File diff suppressed because it is too large Load diff

8
testsuite/wanted-results/parallel-local-ssh7
View file

@ -198,11 +198,11 @@ par_csh_man -j n Run n jobs in parallel
par_csh_man -k Keep same order
par_csh_man Academic tradition requires you to cite works you base your article on.
par_csh_man If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
par_csh_man If you use programs that use GNU Parallel to process data for an article in a
par_csh_man See 'man parallel' for details
par_csh_man This helps funding further development; AND IT WON'T COST YOU A CENT.
par_csh_man Unknown option: no-such-option
par_csh_man Usage:
par_csh_man When using programs that use GNU Parallel to process data for publication
par_csh_man With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
par_csh_man aliases work
par_csh_man aliases work
@ -217,7 +217,7 @@ par_csh_man exit value -1 should be 255
par_csh_man exit value 2 should be 2
par_csh_man parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))...
par_csh_man parallel [options] [command [arguments]] < list_of_arguments
par_csh_man please cite:
par_csh_man scientific publication, please cite:
par_csh_man too
par_csh_man too
par_csh_man too
@ -602,8 +602,8 @@ par_tcsh_man
par_tcsh_man See 'man parallel' for details
par_tcsh_man
par_tcsh_man Academic tradition requires you to cite works you base your article on.
par_tcsh_man When using programs that use GNU Parallel to process data for publication
par_tcsh_man please cite:
par_tcsh_man If you use programs that use GNU Parallel to process data for an article in a
par_tcsh_man scientific publication, please cite:
par_tcsh_man
par_tcsh_man O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
par_tcsh_man ;login: The USENIX Magazine, February 2011:42-47.

13
testsuite/wanted-results/parallel-local12
View file

@ -1,8 +1,8 @@
### Test stdin goes to first command only ("cat" as argument)
via pseudotty
Academic tradition requires you to cite works you base your article on.
When using programs that use GNU Parallel to process data for publication
please cite:
If you use programs that use GNU Parallel to process data for an article in a
scientific publication, please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
@ -23,8 +23,8 @@ NB: If this changes and the diff is printed to terminal, then
the terminal settings may be fucked up. Use 'reset' to get back.
ZZZZ
Academic tradition requires you to cite works you base your article on.
When using programs that use GNU Parallel to process data for publication
please cite:
If you use programs that use GNU Parallel to process data for an article in a
scientific publication, please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
@ -36,8 +36,8 @@ To silence this citation notice: run 'parallel --citation'.
10 files to edit
[?1049h[?1h=[?12;25h[?12l[?25h[?25l"file1" [New File]~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ [?12l[?25h[?25lE173: 9 more files to edit[?12l[?25h[?1l>[?1049lAcademic tradition requires you to cite works you base your article on.
When using programs that use GNU Parallel to process data for publication
please cite:
If you use programs that use GNU Parallel to process data for an article in a
scientific publication, please cite:
@article{Tange2011a,
title = {GNU Parallel - The Command-Line Power Tool},
@ -62,6 +62,5 @@ If you send a copy of your published article to tange@gnu.org, it will be
mentioned in the release notes of next version of GNU Parallel.
Type: 'will cite' and press enter.
>

13
testsuite/wanted-results/parallel-local4
View file

@ -1,16 +1,3 @@
bug #46120: Suspend should suspend (at least local) children
it should burn 1.9 CPU seconds, but no more than that
The 5 second sleep will make it be killed by timeout when it fgs
stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | grep --color=auto -q CPUTIME=1
Zero=OK 0
echo 1 | stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" | grep --color=auto -q CPUTIME=1
Zero=OK 0
Control case: Burn for 2.9 seconds
stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 -q perl -e "while(1){ }" ::: 1 | grep --color=auto -q CPUTIME=1
1=OK 1
echo '### -L -n with pipe'
### -L -n with pipe
seq 14 | parallel --pipe -k -L 3 -n 2 'cat;echo 6 Ln line record'

4
testsuite/wanted-results/parallel-local9
View file

@ -162,8 +162,8 @@ With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
See 'man parallel' for details
Academic tradition requires you to cite works you base your article on.
When using programs that use GNU Parallel to process data for publication
please cite:
If you use programs that use GNU Parallel to process data for an article in a
scientific publication, please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.