mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
--eta was broken. Now fixed.
Race condition in sem fixed with setsid(). Passes unittest.
This commit is contained in:
Ole Tange
parent
adff24a1a0
commit
0fbd1b2653
|
|
@ -1,16 +1,7 @@
|
|||
Bug: seq 1 10 | stdout parallel --eta echo
|
||||
|
||||
# Examples of sem
|
||||
|
||||
Weird bug - only interactive
|
||||
perlipc: Complete Dissociation of Child from Parent
|
||||
Setting $SIG{CHLD} to 'IGNORE'
|
||||
|
||||
echo '### BUG: Test --fg followed by --bg'
|
||||
parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20
|
||||
parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20
|
||||
parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20
|
||||
parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20
|
||||
sem --wait
|
||||
|
||||
find . -execdir sem -j100 'sleep 4; echo {}' \; ; sem --wait
|
||||
|
||||
|
||||
|
|
@ -18,46 +9,12 @@ fex syntax for splitting fields
|
|||
http://www.semicomplete.com/projects/fex/
|
||||
sql :foo 'select * from bar' | parallel --fex '|{1,2}' do_stuff {2} {1}
|
||||
|
||||
|
||||
Unittest: --colsep + multiple -a
|
||||
|
||||
Import sql
|
||||
|
||||
inputfile tabel, Split colonner til {n}
|
||||
sql :foo 'select * from bar' | parallel --colsep '\|' do_stuff {4} {1}
|
||||
|
||||
|
||||
|
||||
if(defined $::opt_colsep and defined @::opt_a and @::opt_a > 1) {
|
||||
# must be done after converting :::: to -a -a
|
||||
warn("--colsep incompatible with multiple argument files. Ignoring --colsep");
|
||||
$::opt_colsep = undef;
|
||||
}
|
||||
|
||||
if($::opt_colsep) {
|
||||
# read input either from -a or from stdin
|
||||
my $max_cols = 0;
|
||||
my @table;
|
||||
my $lineno = 0;
|
||||
$Global::input_is_filename = 0; # cheat get_next_arg into not quoting
|
||||
while get_next_arg {
|
||||
my @cols = split /$::opt_colsep/o, $_;
|
||||
if ($Global::trim) {
|
||||
for(@cols) { s/^\s+//; s/\s+$//; }
|
||||
}
|
||||
$max_cols = max($#cols+1,$max_cols);
|
||||
@table[$lineno++] = @cols;
|
||||
}
|
||||
$::opt_N = $max_cols;
|
||||
for ($lineno = 0; $lineno <= $#table; $lineno++) {
|
||||
if (not defined $table[$lineno][$max_col-1]) {
|
||||
# Make sure the table has the same columns for all rows
|
||||
$table[$lineno][$max_col-1] = "";
|
||||
}
|
||||
unget_arg(@table[$lineno]);
|
||||
}
|
||||
}
|
||||
|
||||
--autocolsep: Læs alle linjer.
|
||||
Prøv fastlængde: Find tegn, som står i alle linjer på de samme pladser. Risiko for falske pos
|
||||
Prøv fieldsep: Find eet tegn, som optræder det samme antal gange i alle linjer (tab sep)
|
||||
|
|
@ -117,6 +74,9 @@ find music-files -type f | parallel -j+0 lame {} -o {.}.mp3
|
|||
find music-files -type f | parallel -j+0 -S :,computer1.examle.com,computer2.example.com \
|
||||
--eta --trc {.}.mp3 lame {} -o {.}.mp3
|
||||
|
||||
# Colsep
|
||||
# sem
|
||||
|
||||
|
||||
=head1 YouTube video
|
||||
|
||||
|
|
|
|||
|
|
@ -91,7 +91,9 @@ download at: http://ftp.gnu.org/gnu/parallel/
|
|||
New in this release:
|
||||
|
||||
* Counting semaphore functionality: start a job in the background. If
|
||||
N jobs are already running, wait for one to complete.
|
||||
N jobs are already running, wait for one to complete. Examples:
|
||||
sem 'sleep 2; echo foo'; sem 'sleep 1; echo bar'; sem --wait
|
||||
sem -j2 'sleep 2; echo foo'; sem -j2 'sleep 1; echo bar'; sem --wait
|
||||
|
||||
* With --colsep a table can be used as input. Example:
|
||||
cat tab_sep_table | parallel --colsep '\t' echo col1 {1} col2 {2}
|
||||
|
|
|
|||
58
src/parallel
58
src/parallel
|
|
@ -1265,11 +1265,11 @@ To run:
|
|||
|
||||
you can run:
|
||||
|
||||
parallel -a table_file.tsv --colsep '\t' cmd -o {2} -i {1}
|
||||
B<parallel -a table_file.tsv --colsep '\t' cmd -o {2} -i {1}>
|
||||
|
||||
Note: The default for GNU B<parallel> is to remove the spaces around the columns. To keep the spaces:
|
||||
|
||||
parallel -a table_file.tsv --trim n --colsep '\t' cmd -o {2} -i {1}
|
||||
B<parallel -a table_file.tsv --trim n --colsep '\t' cmd -o {2} -i {1}>
|
||||
|
||||
|
||||
=head1 EXAMPLE: Working as cat | sh. Ressource inexpensive jobs and evaluation
|
||||
|
|
@ -2077,7 +2077,7 @@ B<xjobs>(1), B<prll>(1), B<dxargs>(1), B<mdm>(1)
|
|||
use IPC::Open3;
|
||||
use Symbol qw(gensym);
|
||||
use IO::File;
|
||||
use POSIX ":sys_wait_h";
|
||||
use POSIX qw/:sys_wait_h setsid/;
|
||||
use File::Temp qw/tempfile tempdir/;
|
||||
use Getopt::Long;
|
||||
use strict;
|
||||
|
|
@ -2109,8 +2109,6 @@ sub acquire_semaphore {
|
|||
my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'});
|
||||
$sem->acquire();
|
||||
debug("run");
|
||||
$Global::argfile = open_or_exit("/dev/null");
|
||||
unget_arg("");
|
||||
if($Semaphore::fg) {
|
||||
# skip
|
||||
} else {
|
||||
|
|
@ -2118,14 +2116,11 @@ sub acquire_semaphore {
|
|||
# therefore release and re-acquire the semaphore
|
||||
$sem->release();
|
||||
if(fork()) {
|
||||
# TODO figure out the race condition that requires this sleep 1
|
||||
# sem --fg seq 21 30
|
||||
# sem --bg seq 31 40
|
||||
sleep(1);
|
||||
exit(0);
|
||||
} else {
|
||||
# child
|
||||
# Get a semaphore for this pid
|
||||
die "Can't start a new session: $!" if setsid() == -1;
|
||||
$sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'});
|
||||
$sem->acquire();
|
||||
}
|
||||
|
|
@ -2316,6 +2311,25 @@ sub parse_options {
|
|||
$Global::Xargs = 1;
|
||||
}
|
||||
|
||||
# Semaphore defaults
|
||||
# Must be done before computing number of processes and max_line_length as no args
|
||||
$Global::semaphore ||= ($0 =~ m:(^|/)sem$:); # called as 'sem'
|
||||
if($Global::semaphore) {
|
||||
# A semaphore does not take input from neither stdin nor file
|
||||
$Global::argfile = open_or_exit("/dev/null");
|
||||
unget_arg("");
|
||||
$Semaphore::timeout = $::opt_semaphoretimeout || 0;
|
||||
if(defined $::opt_semaphorename) {
|
||||
$Semaphore::name = $::opt_semaphorename;
|
||||
} else {
|
||||
$Semaphore::name = `tty`;
|
||||
chomp $Semaphore::name;
|
||||
}
|
||||
$Semaphore::fg = $::opt_fg;
|
||||
$Semaphore::wait = $::opt_wait;
|
||||
$Global::default_simultaneous_sshlogins = 1;
|
||||
}
|
||||
|
||||
if(defined $::opt_eta) {
|
||||
# must be done after opt_a
|
||||
$::opt_progress = $::opt_eta;
|
||||
|
|
@ -2345,22 +2359,6 @@ sub parse_options {
|
|||
print STDERR ("Warning: using -X or -m with --sshlogin may fail\n");
|
||||
}
|
||||
|
||||
# Semaphore defaults
|
||||
# Must be done before computing number of processes
|
||||
$Global::semaphore ||= ($0 =~ m:(^|/)sem$:); # called as 'sem'
|
||||
if($Global::semaphore) {
|
||||
$Semaphore::timeout = $::opt_semaphoretimeout || 0;
|
||||
if(defined $::opt_semaphorename) {
|
||||
$Semaphore::name = $::opt_semaphorename;
|
||||
} else {
|
||||
$Semaphore::name = `tty`;
|
||||
chomp $Semaphore::name;
|
||||
}
|
||||
$Semaphore::fg = $::opt_fg;
|
||||
$Semaphore::wait = $::opt_wait;
|
||||
$Global::default_simultaneous_sshlogins = 1;
|
||||
}
|
||||
|
||||
# Needs to be done after setting $Global::command and $Global::command_line_max_len
|
||||
# as '-m' influences the number of commands that needs to be run
|
||||
if(defined $::opt_P) {
|
||||
|
|
@ -2518,7 +2516,7 @@ sub trim {
|
|||
# lr|rl = both
|
||||
# Returns:
|
||||
# string with white space removed as needed
|
||||
my (@strings) = (@_);
|
||||
my (@strings) = map { defined $_ ? $_ : "" } (@_);
|
||||
my $arg;
|
||||
if($Global::trim eq "n") {
|
||||
# skip
|
||||
|
|
@ -3774,9 +3772,11 @@ sub more_arguments {
|
|||
# Returns:
|
||||
# whether there are more arguments to be processed or not
|
||||
my $fh = shift || $Global::argfile;
|
||||
return (@Global::unget_argv or @{$Global::unget_line{$fh}} or
|
||||
@{$Global::unget_col{$fh}} or @Global::unget_arg or not
|
||||
eof $fh);
|
||||
return (
|
||||
@Global::unget_argv or
|
||||
(defined $Global::unget_line{$fh} and @{$Global::unget_line{$fh}}) or
|
||||
(defined $Global::unget_col{$fh} and @{$Global::unget_col{$fh}})
|
||||
or @Global::unget_arg or not eof $fh);
|
||||
}
|
||||
|
||||
sub get_line_from_fh {
|
||||
|
|
|
|||
|
|
@ -24,16 +24,17 @@ sem -u -j2 'echo job2; sleep 0.5; echo job2'
|
|||
sem --wait
|
||||
echo done
|
||||
|
||||
echo '### Test similar example as from man page'
|
||||
for i in 0.5 0.1 0.2 0.3 0.4 ; do
|
||||
echo $i
|
||||
sem -j+0 sleep $i ";" echo done $i
|
||||
echo '### Test similar example as from man page - run 2 jobs simultaneously'
|
||||
echo 'Expect done: 1 2 5 3 4'
|
||||
for i in 5 1 2 3 4 ; do
|
||||
echo Scheduling $i
|
||||
sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i
|
||||
done
|
||||
sem --wait
|
||||
|
||||
echo '### BUG: Test --fg followed by --bg'
|
||||
parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20
|
||||
parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20
|
||||
parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20
|
||||
parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20
|
||||
echo '### Test --fg followed by --bg'
|
||||
parallel -u --fg --semaphore seq 1 10 '|' pv -qL 30
|
||||
parallel -u --bg --semaphore seq 11 20 '|' pv -qL 30
|
||||
parallel -u --fg --semaphore seq 21 30 '|' pv -qL 30
|
||||
parallel -u --bg --semaphore seq 31 40 '|' pv -qL 30
|
||||
sem --wait
|
||||
|
|
@ -37,6 +37,12 @@ echo '### Test of tab as colsep'
|
|||
printf 'def\tabc\njkl\tghi' | parallel -k --colsep '\t' echo {2} {1}
|
||||
parallel -k -a <(printf 'def\tabc\njkl\tghi') --colsep '\t' echo {2} {1}
|
||||
|
||||
echo '### Test of multiple -a plus colsep'
|
||||
parallel -k -a <(printf 'def\njkl\n') -a <(printf 'abc\tghi\nmno\tpqr') --colsep '\t' echo {2} {1}
|
||||
|
||||
echo '### Test of multiple -a no colsep'
|
||||
parallel -k -a <(printf 'ghi\npqr\n') -a <(printf 'abc\tdef\njkl\tmno') echo {2} {1}
|
||||
|
||||
echo '### Test of quoting after colsplit'
|
||||
parallel --colsep % echo {2} {1} ::: '>/dev/null%>/tmp/null'
|
||||
|
||||
|
|
|
|||
7
unittest/tests-to-run/test30.sh
Normal file
7
unittest/tests-to-run/test30.sh
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo '### Test of --eta'
|
||||
seq 1 10 | stdout parallel --eta "sleep 1; echo {}" | wc -l
|
||||
|
||||
echo '### Test of --progress'
|
||||
seq 1 10 | stdout parallel --progress "sleep 1; echo {}" | wc -l
|
||||
|
|
@ -54,18 +54,24 @@ job2
|
|||
job1
|
||||
job2
|
||||
done
|
||||
### Test similar example as from man page
|
||||
0.5
|
||||
0.1
|
||||
0.2
|
||||
done 0.1
|
||||
0.3
|
||||
done 0.5
|
||||
0.4
|
||||
done 0.2
|
||||
done 0.3
|
||||
done 0.4
|
||||
### BUG: Test --fg followed by --bg
|
||||
### Test similar example as from man page - run 2 jobs simultaneously
|
||||
Expect done: 1 2 5 3 4
|
||||
Scheduling 5
|
||||
Scheduling 1
|
||||
starting 5
|
||||
Scheduling 2
|
||||
starting 1
|
||||
done 1
|
||||
Scheduling 3
|
||||
starting 2
|
||||
done 2
|
||||
Scheduling 4
|
||||
starting 3
|
||||
done 5
|
||||
starting 4
|
||||
done 3
|
||||
done 4
|
||||
### Test --fg followed by --bg
|
||||
1
|
||||
2
|
||||
3
|
||||
|
|
|
|||
|
|
@ -38,6 +38,14 @@ abc def
|
|||
ghi jkl
|
||||
abc def
|
||||
ghi jkl
|
||||
### Test of multiple -a plus colsep
|
||||
abc def
|
||||
ghi jkl
|
||||
mno
|
||||
pqr
|
||||
### Test of multiple -a no colsep
|
||||
abc def ghi
|
||||
jkl mno pqr
|
||||
### Test of quoting after colsplit
|
||||
>/tmp/null >/dev/null
|
||||
### Test of --colsep as regexp
|
||||
|
|
|
|||
4
unittest/wanted-results/test30
Normal file
4
unittest/wanted-results/test30
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
### Test of --eta
|
||||
16
|
||||
### Test of --progress
|
||||
16
|
||||
Loading…
Reference in a new issue