diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index f11e9ca7..8c72af4e 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,16 +1,7 @@ +Bug: seq 1 10 | stdout parallel --eta echo + # Examples of sem -Weird bug - only interactive -perlipc: Complete Dissociation of Child from Parent -Setting $SIG{CHLD} to 'IGNORE' - -echo '### BUG: Test --fg followed by --bg' -parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20 -parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20 -parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20 -parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20 -sem --wait - find . -execdir sem -j100 'sleep 4; echo {}' \; ; sem --wait @@ -18,46 +9,12 @@ fex syntax for splitting fields http://www.semicomplete.com/projects/fex/ sql :foo 'select * from bar' | parallel --fex '|{1,2}' do_stuff {2} {1} - -Unittest: --colsep + multiple -a - Import sql inputfile tabel, Split colonner til {n} sql :foo 'select * from bar' | parallel --colsep '\|' do_stuff {4} {1} - -if(defined $::opt_colsep and defined @::opt_a and @::opt_a > 1) { - # must be done after converting :::: to -a -a - warn("--colsep incompatible with multiple argument files. Ignoring --colsep"); - $::opt_colsep = undef; -} - -if($::opt_colsep) { - # read input either from -a or from stdin - my $max_cols = 0; - my @table; - my $lineno = 0; - $Global::input_is_filename = 0; # cheat get_next_arg into not quoting - while get_next_arg { - my @cols = split /$::opt_colsep/o, $_; - if ($Global::trim) { - for(@cols) { s/^\s+//; s/\s+$//; } - } - $max_cols = max($#cols+1,$max_cols); - @table[$lineno++] = @cols; - } - $::opt_N = $max_cols; - for ($lineno = 0; $lineno <= $#table; $lineno++) { - if (not defined $table[$lineno][$max_col-1]) { - # Make sure the table has the same columns for all rows - $table[$lineno][$max_col-1] = ""; - } - unget_arg(@table[$lineno]); - } -} - --autocolsep: Læs alle linjer. Prøv fastlængde: Find tegn, som står i alle linjer på de samme pladser. Risiko for falske pos Prøv fieldsep: Find eet tegn, som optræder det samme antal gange i alle linjer (tab sep) @@ -117,6 +74,9 @@ find music-files -type f | parallel -j+0 lame {} -o {.}.mp3 find music-files -type f | parallel -j+0 -S :,computer1.examle.com,computer2.example.com \ --eta --trc {.}.mp3 lame {} -o {.}.mp3 +# Colsep +# sem + =head1 YouTube video diff --git a/doc/release_new_version b/doc/release_new_version index d794df47..38812d59 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -91,7 +91,9 @@ download at: http://ftp.gnu.org/gnu/parallel/ New in this release: * Counting semaphore functionality: start a job in the background. If - N jobs are already running, wait for one to complete. + N jobs are already running, wait for one to complete. Examples: + sem 'sleep 2; echo foo'; sem 'sleep 1; echo bar'; sem --wait + sem -j2 'sleep 2; echo foo'; sem -j2 'sleep 1; echo bar'; sem --wait * With --colsep a table can be used as input. Example: cat tab_sep_table | parallel --colsep '\t' echo col1 {1} col2 {2} diff --git a/src/parallel b/src/parallel index a288e04e..ae38ab32 100755 --- a/src/parallel +++ b/src/parallel @@ -1265,11 +1265,11 @@ To run: you can run: - parallel -a table_file.tsv --colsep '\t' cmd -o {2} -i {1} +B Note: The default for GNU B is to remove the spaces around the columns. To keep the spaces: - parallel -a table_file.tsv --trim n --colsep '\t' cmd -o {2} -i {1} +B =head1 EXAMPLE: Working as cat | sh. Ressource inexpensive jobs and evaluation @@ -2077,8 +2077,8 @@ B(1), B(1), B(1), B(1) use IPC::Open3; use Symbol qw(gensym); use IO::File; -use POSIX ":sys_wait_h"; -use File::Temp qw/ tempfile tempdir /; +use POSIX qw/:sys_wait_h setsid/; +use File::Temp qw/tempfile tempdir/; use Getopt::Long; use strict; @@ -2109,8 +2109,6 @@ sub acquire_semaphore { my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'}); $sem->acquire(); debug("run"); - $Global::argfile = open_or_exit("/dev/null"); - unget_arg(""); if($Semaphore::fg) { # skip } else { @@ -2118,14 +2116,11 @@ sub acquire_semaphore { # therefore release and re-acquire the semaphore $sem->release(); if(fork()) { - # TODO figure out the race condition that requires this sleep 1 - # sem --fg seq 21 30 - # sem --bg seq 31 40 - sleep(1); exit(0); } else { # child # Get a semaphore for this pid + die "Can't start a new session: $!" if setsid() == -1; $sem = Semaphore->new($Semaphore::name,$Global::host{':'}{'max_no_of_running'}); $sem->acquire(); } @@ -2316,6 +2311,25 @@ sub parse_options { $Global::Xargs = 1; } + # Semaphore defaults + # Must be done before computing number of processes and max_line_length as no args + $Global::semaphore ||= ($0 =~ m:(^|/)sem$:); # called as 'sem' + if($Global::semaphore) { + # A semaphore does not take input from neither stdin nor file + $Global::argfile = open_or_exit("/dev/null"); + unget_arg(""); + $Semaphore::timeout = $::opt_semaphoretimeout || 0; + if(defined $::opt_semaphorename) { + $Semaphore::name = $::opt_semaphorename; + } else { + $Semaphore::name = `tty`; + chomp $Semaphore::name; + } + $Semaphore::fg = $::opt_fg; + $Semaphore::wait = $::opt_wait; + $Global::default_simultaneous_sshlogins = 1; + } + if(defined $::opt_eta) { # must be done after opt_a $::opt_progress = $::opt_eta; @@ -2345,22 +2359,6 @@ sub parse_options { print STDERR ("Warning: using -X or -m with --sshlogin may fail\n"); } - # Semaphore defaults - # Must be done before computing number of processes - $Global::semaphore ||= ($0 =~ m:(^|/)sem$:); # called as 'sem' - if($Global::semaphore) { - $Semaphore::timeout = $::opt_semaphoretimeout || 0; - if(defined $::opt_semaphorename) { - $Semaphore::name = $::opt_semaphorename; - } else { - $Semaphore::name = `tty`; - chomp $Semaphore::name; - } - $Semaphore::fg = $::opt_fg; - $Semaphore::wait = $::opt_wait; - $Global::default_simultaneous_sshlogins = 1; - } - # Needs to be done after setting $Global::command and $Global::command_line_max_len # as '-m' influences the number of commands that needs to be run if(defined $::opt_P) { @@ -2518,7 +2516,7 @@ sub trim { # lr|rl = both # Returns: # string with white space removed as needed - my (@strings) = (@_); + my (@strings) = map { defined $_ ? $_ : "" } (@_); my $arg; if($Global::trim eq "n") { # skip @@ -3774,9 +3772,11 @@ sub more_arguments { # Returns: # whether there are more arguments to be processed or not my $fh = shift || $Global::argfile; - return (@Global::unget_argv or @{$Global::unget_line{$fh}} or - @{$Global::unget_col{$fh}} or @Global::unget_arg or not - eof $fh); + return ( + @Global::unget_argv or + (defined $Global::unget_line{$fh} and @{$Global::unget_line{$fh}}) or + (defined $Global::unget_col{$fh} and @{$Global::unget_col{$fh}}) + or @Global::unget_arg or not eof $fh); } sub get_line_from_fh { diff --git a/unittest/tests-to-run/sem01.sh b/unittest/tests-to-run/sem01.sh index de175934..a9439a55 100755 --- a/unittest/tests-to-run/sem01.sh +++ b/unittest/tests-to-run/sem01.sh @@ -24,16 +24,17 @@ sem -u -j2 'echo job2; sleep 0.5; echo job2' sem --wait echo done -echo '### Test similar example as from man page' -for i in 0.5 0.1 0.2 0.3 0.4 ; do - echo $i - sem -j+0 sleep $i ";" echo done $i +echo '### Test similar example as from man page - run 2 jobs simultaneously' +echo 'Expect done: 1 2 5 3 4' +for i in 5 1 2 3 4 ; do + echo Scheduling $i + sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i done sem --wait -echo '### BUG: Test --fg followed by --bg' -parallel -u --fg --semaphore seq 1 10 '|' pv -qL 20 -parallel -u --bg --semaphore seq 11 20 '|' pv -qL 20 -parallel -u --fg --semaphore seq 21 30 '|' pv -qL 20 -parallel -u --bg --semaphore seq 31 40 '|' pv -qL 20 -sem --wait \ No newline at end of file +echo '### Test --fg followed by --bg' +parallel -u --fg --semaphore seq 1 10 '|' pv -qL 30 +parallel -u --bg --semaphore seq 11 20 '|' pv -qL 30 +parallel -u --fg --semaphore seq 21 30 '|' pv -qL 30 +parallel -u --bg --semaphore seq 31 40 '|' pv -qL 30 +sem --wait diff --git a/unittest/tests-to-run/test29.sh b/unittest/tests-to-run/test29.sh index 155d6039..4254b60c 100644 --- a/unittest/tests-to-run/test29.sh +++ b/unittest/tests-to-run/test29.sh @@ -37,6 +37,12 @@ echo '### Test of tab as colsep' printf 'def\tabc\njkl\tghi' | parallel -k --colsep '\t' echo {2} {1} parallel -k -a <(printf 'def\tabc\njkl\tghi') --colsep '\t' echo {2} {1} +echo '### Test of multiple -a plus colsep' +parallel -k -a <(printf 'def\njkl\n') -a <(printf 'abc\tghi\nmno\tpqr') --colsep '\t' echo {2} {1} + +echo '### Test of multiple -a no colsep' +parallel -k -a <(printf 'ghi\npqr\n') -a <(printf 'abc\tdef\njkl\tmno') echo {2} {1} + echo '### Test of quoting after colsplit' parallel --colsep % echo {2} {1} ::: '>/dev/null%>/tmp/null' diff --git a/unittest/tests-to-run/test30.sh b/unittest/tests-to-run/test30.sh new file mode 100644 index 00000000..8bbe3558 --- /dev/null +++ b/unittest/tests-to-run/test30.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +echo '### Test of --eta' +seq 1 10 | stdout parallel --eta "sleep 1; echo {}" | wc -l + +echo '### Test of --progress' +seq 1 10 | stdout parallel --progress "sleep 1; echo {}" | wc -l diff --git a/unittest/wanted-results/sem01 b/unittest/wanted-results/sem01 index 3fa79f69..ebb89452 100644 --- a/unittest/wanted-results/sem01 +++ b/unittest/wanted-results/sem01 @@ -54,18 +54,24 @@ job2 job1 job2 done -### Test similar example as from man page -0.5 -0.1 -0.2 -done 0.1 -0.3 -done 0.5 -0.4 -done 0.2 -done 0.3 -done 0.4 -### BUG: Test --fg followed by --bg +### Test similar example as from man page - run 2 jobs simultaneously +Expect done: 1 2 5 3 4 +Scheduling 5 +Scheduling 1 +starting 5 +Scheduling 2 +starting 1 +done 1 +Scheduling 3 +starting 2 +done 2 +Scheduling 4 +starting 3 +done 5 +starting 4 +done 3 +done 4 +### Test --fg followed by --bg 1 2 3 diff --git a/unittest/wanted-results/test29 b/unittest/wanted-results/test29 index 783f982d..ec52aa39 100644 --- a/unittest/wanted-results/test29 +++ b/unittest/wanted-results/test29 @@ -38,6 +38,14 @@ abc def ghi jkl abc def ghi jkl +### Test of multiple -a plus colsep +abc def +ghi jkl +mno +pqr +### Test of multiple -a no colsep +abc def ghi +jkl mno pqr ### Test of quoting after colsplit >/tmp/null >/dev/null ### Test of --colsep as regexp diff --git a/unittest/wanted-results/test30 b/unittest/wanted-results/test30 new file mode 100644 index 00000000..c0f7b83f --- /dev/null +++ b/unittest/wanted-results/test30 @@ -0,0 +1,4 @@ +### Test of --eta +16 +### Test of --progress +16