From 847841aa1106666c514a6ee3e2bf4e54c6e15dc0 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 5 Sep 2010 12:22:08 +0200 Subject: [PATCH] BSD xargs -o (open /dev/tty) is now default for the job running in foreground. Useful for: ls | parallel -Xuj1 vi. Unittest for tty commands using the command 'script'. --- doc/FUTURE_IDEAS | 17 +++++++++++------ doc/release_new_version | 11 +++++++---- src/parallel | 22 +++++++++++++++++++--- src/sem.pod | 5 ++++- src/sql | 13 ++++++++++--- unittest/Makefile | 1 + unittest/actual-results/test25 | 22 +++++++++++++--------- unittest/tests-to-run/sql01.sh | 9 ++++++--- unittest/tests-to-run/test15.sh | 21 ++++++++++++++++++--- unittest/tests-to-run/test25.sh | 19 +++++++++++++++++-- unittest/wanted-results/test15 | 6 ++++-- unittest/wanted-results/test25 | 22 +++++++++++++--------- 12 files changed, 123 insertions(+), 45 deletions(-) diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index 4c1f19d6..32feaa5f 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,5 +1,14 @@ -sql: Added unittest, --shebang, --version, --help. -parallel: bugfix in unittest. +== SQL == + +/etc/sql/aliases +~/.sql/aliases + + +== STDIN == + +BSD xargs -o (open /dev/tty) is now default for the job running in foreground. +Useful for: ls | parallel -Xuj1 vi. +Unittest for tty commands using the command 'script'. == FEX == @@ -7,10 +16,6 @@ fex syntax for splitting fields http://www.semicomplete.com/projects/fex/ sql :foo 'select * from bar' | parallel --fex '|{1,2}' do_stuff {2} {1} -== SQL == -Example: -sql :foo 'select * from bar' | parallel --colsep '\|' do_stuff {4} {1} - == Build == build.opensuse.org diff --git a/doc/release_new_version b/doc/release_new_version index 49c24504..4d5d3703 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -103,7 +103,7 @@ cc:Peter Simons , Sandro Cazzaniga , Tim Cuthbertson , Ludovic Courtès , Markus Ammer , Pavel Nuzhdin , Phil Sung , Michael Shigorin , - Andrew McFague + Andrew McFague , Steven M. Christensen Subject: GNU Parallel 2010XXXX released @@ -112,9 +112,6 @@ download at: http://ftp.gnu.org/gnu/parallel/ New in this release: -* sql - a small script to access sql bases from the command line which - is a handy companion to parallel --colsep - * Using --shebang GNU Parallel can be used as the parser for a script. E.g: #!/usr/bin/parallel --shebang traceroute (followed by lines of hosts) @@ -124,6 +121,9 @@ New in this release: * Alt Linux package of GNU Parallel. Thanks to Michael Shigorin +* Sunfreeware package of GNU Parallel. Thanks to Steven M. Christensen + + * Untested CentOS, Fedora, Mandriva, RedHat, and SUSE packages available through OpenSUSE build service: https://build.opensuse.org/package/show?package=parallel&project=home%3Atange @@ -134,6 +134,9 @@ New in this release: * First 1000 views of the intro video +* sql - a small script to access sql bases from the command line which + is a handy companion to parallel --colsep + = About GNU Parallel = GNU Parallel is a shell tool for executing jobs in parallel using one diff --git a/src/parallel b/src/parallel index 9f7041f3..92b54273 100755 --- a/src/parallel +++ b/src/parallel @@ -1937,10 +1937,14 @@ B<6> find */ -... | fmt 960 1024 | xapply -f -i /dev/tty 'vi' - B<6> sh <(find */ -... | parallel -s 1024 echo vi) +B<6> find */ -... | parallel -s 1024 -Xuj1 vi + B<7> find ... | xapply -f -5 -i /dev/tty 'vi' - - - - - B<7> sh <(find ... |parallel -n5 echo vi) +B<7> find ... |parallel -n5 -uj1 vi + B<8> xapply -fn "" /etc/passwd B<8> parallel -k echo < /etc/passwd @@ -2193,7 +2197,7 @@ sub acquire_semaphore { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20100822; + $Global::version = 20100901; $Global::progname = 'parallel'; $Global::debug = 0; $Global::verbose = 0; @@ -3404,6 +3408,7 @@ sub init_run_jobs { $Global::total_running = 0; $Global::total_started = 0; $Global::total_completed = 0; + $Global::tty_taken = 0; $SIG{USR1} = \&list_running_jobs; $SIG{USR2} = \&toggle_progress; $Global::original_sigterm = $SIG{TERM}; @@ -3731,13 +3736,20 @@ sub start_job { if(@::opt_a and $Private::job_start_sequence == 1) { # Give STDIN to the first job if using -a $pid = open3("<&STDIN", ">&STDOUT", ">&STDERR", $command) || - die("open3 failed. Report a bug to \n"); + die("open3 (with -a) failed. Report a bug to \n"); # Re-open to avoid complaining open STDIN, "<&", $Global::original_stdin or die "Can't dup \$Global::original_stdin: $!"; + } elsif (not $Global::tty_taken and -c "/dev/tty" and + open(DEVTTY, "/dev/tty")) { + # Give /dev/tty to the command if no one else is using it + $pid = open3("<&DEVTTY", ">&STDOUT", ">&STDERR", $command) || + die("open3 (with /dev/tty) failed. Report a bug to \n"); + $Global::tty_taken = $pid; + close DEVTTY; } else { $pid = open3(gensym, ">&STDOUT", ">&STDERR", $command) || - die("open3 failed. Report a bug to \n"); + die("open3 (with gensym) failed. Report a bug to \n"); } debug("started: $command\n"); open STDOUT, ">&", $Global::original_stdout @@ -4278,6 +4290,10 @@ sub reaper { $Global::running{$stiff} or next; $Global::running{$stiff}{'exitstatus'} = $? >> 8; debug("died ($Global::running{$stiff}{'exitstatus'}): $Global::running{$stiff}{'seq'}"); + if($stiff == $Global::tty_taken) { + # The process that died had the tty => release it + $Global::tty_taken = 0; + } # Force printing now if the job failed and we are going to exit my $print_now = ($Global::running{$stiff}{'exitstatus'} and $::opt_halt_on_error and $::opt_halt_on_error == 2); diff --git a/src/sem.pod b/src/sem.pod index bbec5ca1..2ca64cee 100755 --- a/src/sem.pod +++ b/src/sem.pod @@ -72,9 +72,12 @@ Wait for all commands to complete. =head1 EXAMPLE: Gzipping *.log +Run one gzip process per CPU core. Block until a CPU core becomes +available. + for i in `ls *.log` ; do echo $i - sem gzip $i ";" echo done + sem -j+0 gzip $i ";" echo done done sem --wait diff --git a/src/sql b/src/sql index ce6e4764..df2d0744 100755 --- a/src/sql +++ b/src/sql @@ -6,9 +6,10 @@ sql - execute a command on a database determined by a dburl =head1 SYNOPSIS -B [B<-hnr>] [B<--table-size>] [B<--db-size>] [B<-p> I] [B<-s> I] I [I] +B [B<-hnr>] [B<--table-size>] [B<--db-size>] [B<-p> +I] [B<-s> I] I [I] -B<#!/usr/bin/sql> --shebang [options] I +B<#!/usr/bin/sql> B<--shebang> [options] I =head1 DESCRIPTION @@ -203,6 +204,12 @@ Then do: chmod 755 demosql; ./demosql +=head2 Use --colsep to process multiple columns + +Use GNU B's B<--colsep> to separate columns: + + sql :mydburl 'select * from bar' | parallel --colsep '\t' do_stuff {4} {1} + =head1 REPORTING BUGS @@ -401,7 +408,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20100822; + $Global::version = 20100901; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/unittest/Makefile b/unittest/Makefile index 12588db3..11fe161e 100644 --- a/unittest/Makefile +++ b/unittest/Makefile @@ -9,6 +9,7 @@ unittest: ../src/parallel tests-to-run/* wanted-results/* stdout gawk | mop || (echo gawk is required for unittest; /bin/false) expect -c 'spawn cat; puts "expect is installed"' || (echo expect is required for unittest; /bin/false) echo | pv -qL 10 || (echo pv is required for unittest; /bin/false) + echo | script -c echo -q /dev/null || (echo script is required for unittest; /bin/false) time sh Start.sh date diff --git a/unittest/actual-results/test25 b/unittest/actual-results/test25 index 27b593f8..8740e781 100644 --- a/unittest/actual-results/test25 +++ b/unittest/actual-results/test25 @@ -23,21 +23,25 @@ echo a a echo b b -### Test stdin goes to first command only +### Test stdin goes to first command only ("-" as argument) cat - via first cat cat - -cat -via cat -echo b -b -cat -via cat -echo b -b +via pseudotty +### Test stdin goes to first command only ("cat" as argument) echo a a cat +via pseudotty +### Test stdin goes to first command only +cat +via cat +echo b +b +cat +via cat +echo b +b ### Bug made 4 5 go before 1 2 3 1 2 diff --git a/unittest/tests-to-run/sql01.sh b/unittest/tests-to-run/sql01.sh index 4c687ebc..144fc223 100755 --- a/unittest/tests-to-run/sql01.sh +++ b/unittest/tests-to-run/sql01.sh @@ -49,9 +49,12 @@ echo :sqlunittest mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest >> sql :sqlunittest "SELECT 'Yes it does' as 'Test if .dburl.aliases works';" echo "### Test --noheaders --no-headers -n" -sql -n :sqlunittest 'select * from unittest' | parallel --colsep '\t' echo {2} {1} -sql --noheaders :sqlunittest 'select * from unittest' | parallel --colsep '\t' echo {2} {1} -sql --no-headers :sqlunittest 'select * from unittest' | parallel --colsep '\t' echo {2} {1} +sql -n :sqlunittest 'select * from unittest order by id' \ +| parallel -k --colsep '\t' echo {2} {1} +sql --noheaders :sqlunittest 'select * from unittest order by id' \ +| parallel -k --colsep '\t' echo {2} {1} +sql --no-headers :sqlunittest 'select * from unittest order by id' \ +| parallel -k --colsep '\t' echo {2} {1} echo "### Test --sep -s"; sql --no-headers -s : pg:/// 'select 1,2' | parallel --colsep ':' echo {2} {1} diff --git a/unittest/tests-to-run/test15.sh b/unittest/tests-to-run/test15.sh index ec8931d5..372a9d13 100755 --- a/unittest/tests-to-run/test15.sh +++ b/unittest/tests-to-run/test15.sh @@ -2,6 +2,8 @@ # Test xargs compatibility + + echo '### Test -p --interactive' cat >/tmp/parallel-script-for-expect <<_EOF #!/bin/bash @@ -88,12 +90,25 @@ cd input-files/test15 echo 'xargs Expect: 3 1 2' echo 3 | xargs -P 1 -n 1 -a files cat - -echo 'parallel Expect: 3 1 2' -echo 3 | parallel -k -P 2 -n 1 -a files cat - +echo 'parallel Expect: 3 1 via psedotty 2' +cat >/tmp/parallel-script-for-script </tmp/parallel-script-for-script2 </tmp/parallel-script-for-script </tmp/parallel-script-for-script2 <