From 8bec8de5df773ef4da86e827afcca7aef70eca63 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 21 Aug 2016 15:37:59 +0200 Subject: [PATCH] --tmuxpane initial version. Passes testsuite. --- doc/release_new_version | 5 +- src/parallel | 25 +++++-- src/parallel.pod | 7 ++ testsuite/Makefile | 3 +- testsuite/input-files/test15/files | 1 - testsuite/tests-to-run/niceload01.sh | 4 +- testsuite/tests-to-run/parallel-local-100s.sh | 1 + testsuite/tests-to-run/parallel-local-10s.sh | 4 +- testsuite/tests-to-run/parallel-local-ssh3.sh | 4 +- testsuite/tests-to-run/parallel-polarhome.sh | 7 +- testsuite/tests-to-run/sem02.sh | 13 +++- testsuite/tests-to-run/sql03.sh | 2 +- testsuite/tests-to-run/test15.sh | 4 +- testsuite/wanted-results/niceload01 | 4 +- testsuite/wanted-results/parallel-install | 2 +- testsuite/wanted-results/parallel-local-0.3s | 10 +-- testsuite/wanted-results/parallel-local-100s | 4 +- testsuite/wanted-results/parallel-local-10s | 6 +- testsuite/wanted-results/parallel-local-ssh3 | 2 - testsuite/wanted-results/parallel-polarhome | 13 +++- testsuite/wanted-results/parallel-tutorial | 2 +- testsuite/wanted-results/sql03 | 20 +++--- testsuite/wanted-results/test13 | 4 +- testsuite/wanted-results/test15 | 68 +++---------------- 24 files changed, 102 insertions(+), 113 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index b999e67c..5ca5ecfe 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -219,9 +219,9 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20160722 ('Og Nomekop/Munich/Erdogan') released <<[stable]>> +Subject: GNU Parallel 20160822 ('Og Nomekop') released <<[stable]>> -GNU Parallel 20160722 ('Munich/Erdogan') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20160822 ('Og Nomekop') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ <> @@ -234,6 +234,7 @@ New in this release: * GNU Parallel was cited in: Exome sequencing of geographically diverse barley landraces and wild relatives gives insights into environmental adaptation http://www.nature.com/ng/journal/vaop/ncurrent/full/ng.3612.html?WT.feed_name=subjects_genetics#references +* GNU Parallel was cited in: Vision-Enhanced Lidar Odometry and Mapping https://www.ri.cmu.edu/pub_files/2016/8/DLL-thesis.pdf * <> diff --git a/src/parallel b/src/parallel index fde99d47..ac444eae 100755 --- a/src/parallel +++ b/src/parallel @@ -799,6 +799,7 @@ sub options_hash { "ungroup|u" => \$opt::ungroup, "linebuffer|linebuffered|line-buffer|line-buffered|lb" => \$opt::linebuffer, "tmux" => \$opt::tmux, + "tmuxpane" => \$opt::tmuxpane, "null|0" => \$opt::0, "quote|q" => \$opt::q, # Replacement strings @@ -997,6 +998,7 @@ sub parse_options { $opt::nice ||= 0; if(defined $opt::help) { die_usage(); } if(defined $opt::sqlandworker) { $opt::sqlmaster = $opt::sqlworker = $opt::sqlandworker; } + if(defined $opt::tmuxpane) { $opt::tmux = $opt::tmuxpane; } if(defined $opt::colsep) { $Global::trim = 'lr'; } if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; } if(defined $opt::trim) { $Global::trim = $opt::trim; } @@ -1551,7 +1553,7 @@ sub find_compression_program { # $decompress_program = decompress program with options # Search for these. Sorted by speed on 20 core - # parallel -j1 --joblog jl --arg-sep , parallel --compress-program \'{3}" "-{2}\' cat ::: gz '>'/dev/null , 1 2 3 , {1..3} , zstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 + # parallel -j1 --joblog jl --arg-sep , parallel --compress-program \'{3}" "-{2}\' cat ::: gz '>'/dev/null , 1 2 3 , {1..3} , zstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrzip # sort -nk4 jl my @prg = qw(lz4 lzop pbzip2 pigz plzip lzip gzip pxz lzma xz bzip2); for my $p (@prg) { @@ -1617,6 +1619,9 @@ sub read_options { exec "$0 --internal-pipe-means-argfiles @options @parser $script ::: @ARGV"; } } + if($ARGV[0] =~ / --shebang(-?wrap)? /) { + ::warning("--shebang and --shebang-wrap must be the first argument.\n"); + } Getopt::Long::Configure("bundling","require_order"); my @ARGV_copy = @ARGV; @@ -5023,8 +5028,10 @@ sub loadavg { if($update_loadavg_file) { ::debug("load", "Updating loadavg file", $self->{'loadavg_file'}, "\n"); $self->{'last_loadavg_update'} = time; - -e $ENV{'HOME'}."/.parallel" or mkdir $ENV{'HOME'}."/.parallel"; - -e $ENV{'HOME'}."/.parallel/tmp" or mkdir $ENV{'HOME'}."/.parallel/tmp"; + -w $ENV{'HOME'}."/.parallel" or mkdir $ENV{'HOME'}."/.parallel" + or ::die_bug("Cannot write to $ENV{'HOME'}/.parallel"); + -w $ENV{'HOME'}."/.parallel/tmp" or mkdir $ENV{'HOME'}."/.parallel/tmp" + or ::die_bug("Cannot write to $ENV{'HOME'}/.parallel/tmp"); my $cmd = ""; if($self->{'string'} ne ":") { $cmd = $self->sshcommand() . " " . $self->serverlogin() . " " . @@ -7851,6 +7858,12 @@ sub print_dryrun_and_verbose { ::error("tmux does not support sockets with path > 100."); ::wait_and_exit(255); } + if($opt::tmuxpane) { + # Move the command into a pane in window 0 + $actual_command = 'tmux joinp -t :0 ; '. + 'tmux select-layout -t :0 tiled ; '. + $actual_command; + } my $visual_command = $self->replaced(); my $title = $visual_command; if($visual_command =~ /\0/) { @@ -10445,11 +10458,11 @@ sub remove_dead_locks { $d =~ m:$lockdir/([0-9]+)\@([-\._a-z0-9]+)$:o or next; my ($pid, $host) = ($1, $2); if($host eq ::hostname()) { - if(not kill 0, $pid) { + if(kill 0, $pid) { + ::debug("sem", "Alive: $pid $d\n"); + } else { ::debug("sem", "Dead: $d\n"); ::rm($d); - } else { - ::debug("sem", "Alive: $d\n"); } } } diff --git a/src/parallel.pod b/src/parallel.pod index f576cdd2..276794c2 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2105,6 +2105,13 @@ Use B for output. Start a B session and run each job in a window in that session. No other output will be produced. +=item B<--tmuxpane> (alpha testing) + +Use B for output but put output into panes in the first window. +Useful if you want to monitor the progress of less than 100 concurrent +jobs. + + =item B<--timeout> I Time out for command. If the command runs for longer than I diff --git a/testsuite/Makefile b/testsuite/Makefile index f267a0d5..44b71c00 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -53,7 +53,8 @@ prereqremote: installparallel startvm startvm: # Make sure we can reach the virtual machines - sudo ifconfig wlp13s0:0 192.168.1.72 || sudo ifconfig wlan0:0 192.168.1.72 + #sudo ifconfig wlp13s0:0 192.168.1.72 || sudo ifconfig wlan0:0 192.168.1.72 + sudo ifconfig enp14s0 192.168.1.72 || sudo ifconfig eth0:0 192.168.1.72 # If they are already running: Don't fail VBoxManage startvm CentOS3-root:centos3 || VBoxManage controlvm CentOS3-root:centos3 resume || true VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true diff --git a/testsuite/input-files/test15/files b/testsuite/input-files/test15/files index 1191247b..d00491fd 100644 --- a/testsuite/input-files/test15/files +++ b/testsuite/input-files/test15/files @@ -1,2 +1 @@ 1 -2 diff --git a/testsuite/tests-to-run/niceload01.sh b/testsuite/tests-to-run/niceload01.sh index 9bcd9e74..1d1db3b3 100755 --- a/testsuite/tests-to-run/niceload01.sh +++ b/testsuite/tests-to-run/niceload01.sh @@ -23,8 +23,8 @@ freepl >/dev/null & cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 echo '### --rm and --runmem' - niceload -H --rm 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--rm - niceload -H --runmem 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--runmem + niceload -H --rm 1g free -g | perl -ane '/Mem:/ and print $F[5],"\n"' | grep '[1-9]' >/dev/null && echo OK--rm + niceload -H --runmem 1g free -g | perl -ane '/Mem:/ and print $F[5],"\n"' | grep '[1-9]' >/dev/null && echo OK--runmem echo '### -N and --noswap. Must give 0' niceload -H -N vmstat 1 2 | tail -n1 | awk '{print "-N " $7*$8}' diff --git a/testsuite/tests-to-run/parallel-local-100s.sh b/testsuite/tests-to-run/parallel-local-100s.sh index e2eb6f0a..061b8c11 100644 --- a/testsuite/tests-to-run/parallel-local-100s.sh +++ b/testsuite/tests-to-run/parallel-local-100s.sh @@ -46,6 +46,7 @@ no_mem_leak() { min30000=$(parallel measure {} 100000 ::: 30000 30000 30000 30000 | sort -n | head -n 1) if [ $max1000 -gt $min30000 ] ; then + # Make sure there are a few sleeps max1000=$(parallel measure {} 100 ::: 1000 1000 1000 1000 | sort -n | tail -n 1) min30000=$(parallel measure {} 100 ::: 30000 30000 30000 30000 | diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index 1c6ead2e..747a8275 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -112,7 +112,7 @@ echo '**' echo "### Test max line length -m -I" - seq 1 60000 | parallel -I :: -m -j1 echo a::b::c | sort >/tmp/114-a$$; + seq 1 60000 | parallel -I :: -km -j1 echo a::b::c | sort >/tmp/114-a$$; md5sum /tmp/114-b$$; + seq 1 60000 | parallel -I :: -kX -j1 echo a::b::c | sort >/tmp/114-b$$; md5sum & /dev/stdout echo '^^^^^^^^ Not found is OK' diff --git a/testsuite/tests-to-run/parallel-polarhome.sh b/testsuite/tests-to-run/parallel-polarhome.sh index 67e74ea2..d81e87c4 100755 --- a/testsuite/tests-to-run/parallel-polarhome.sh +++ b/testsuite/tests-to-run/parallel-polarhome.sh @@ -6,8 +6,8 @@ P_ALL="vax freebsd solaris openbsd netbsd debian alpha aix redhat hpux ultrix mi P_NOTWORKING="vax alpha openstep" P_NOTWORKING_YET="ultrix irix" -P_WORKING="pidora raspbian solaris openindiana aix hpux qnx debian-ppc suse solaris-x86 mandriva ubuntu scosysv unixware centos miros macosx redhat netbsd openbsd freebsd debian" -P_TEMPORARILY_BROKEN="minix tru64 syllable hurd hpux-ia64 dragonfly" +P_WORKING="tru64 syllable pidora raspbian solaris openindiana aix hpux qnx debian-ppc suse solaris-x86 mandriva ubuntu scosysv unixware centos miros macosx redhat netbsd openbsd freebsd debian" +P_TEMPORARILY_BROKEN="minix hurd hpux-ia64 dragonfly" P="$P_WORKING" POLAR=`parallel -k echo {}.polarhome.com ::: $P` @@ -15,7 +15,8 @@ S_POLAR=`parallel -k echo -S 1/{}.polarhome.com ::: $P` # 20150414 --timeout 80 -> 40 # 20151219 --retries 5 -> 2 -TIMEOUT=10 +# 20160821 --timeout 10 -> 100 (DNS problems) +TIMEOUT=130 RETRIES=4 echo '### Tests on polarhome machines' diff --git a/testsuite/tests-to-run/sem02.sh b/testsuite/tests-to-run/sem02.sh index e7bb274a..f69a4f18 100644 --- a/testsuite/tests-to-run/sem02.sh +++ b/testsuite/tests-to-run/sem02.sh @@ -1,9 +1,18 @@ #!/bin/bash +minipv() { + sleep=$1 + while read line; do + echo "$line" + sleep $sleep + done +} +export -f minipv + echo '### Test id = --id `tty`' -parallel --id `tty` -u --semaphore seq 1 10 '|' pv -qL 20 +parallel --id `tty` --lb --semaphore seq 1 10 '|' minipv 0.1 echo '### Test default id = --id `tty`' -parallel -u --semaphore seq 11 20 '|' pv -qL 100 +parallel --lb --semaphore seq 11 20 '|' minipv 0.01 echo '### Test --semaphorename `tty`' parallel --semaphorename `tty` --semaphore --wait echo done diff --git a/testsuite/tests-to-run/sql03.sh b/testsuite/tests-to-run/sql03.sh index b99b1784..61ba3dec 100644 --- a/testsuite/tests-to-run/sql03.sh +++ b/testsuite/tests-to-run/sql03.sh @@ -66,7 +66,7 @@ echo "### Test oracle with \n arguments on the command line" sql :oraunittest 'select 1 from dual;\nselect 2 from dual;\x0aselect 3 from dual;' echo "### Test --show-tables" -sql --show-tables :oraunittest +sql --show-tables :oraunittest | sort echo "### Test --show-databases" sql --show-databases :oraunittest diff --git a/testsuite/tests-to-run/test15.sh b/testsuite/tests-to-run/test15.sh index 7b9f009d..cf527c7d 100755 --- a/testsuite/tests-to-run/test15.sh +++ b/testsuite/tests-to-run/test15.sh @@ -42,7 +42,7 @@ EOF cd input-files/test15 -echo 'xargs Expect: 3 1 2' +echo 'xargs Expect: 3 1' echo 3 | xargs -P 1 -n 1 -a files cat - echo 'parallel Expect: 3 1 via psedotty 2' cat >/tmp/parallel-script-for-script </tmp/parallel-script-for-script2 </dev/null && echo OK--rm + niceload -H --rm 1g free -g | perl -ane '/Mem:/ and print $F[5],"\n"' | grep '[1-9]' >/dev/null && echo OK--rm OK--rm - niceload -H --runmem 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--runmem + niceload -H --runmem 1g free -g | perl -ane '/Mem:/ and print $F[5],"\n"' | grep '[1-9]' >/dev/null && echo OK--runmem OK--runmem echo '### -N and --noswap. Must give 0' ### -N and --noswap. Must give 0 diff --git a/testsuite/wanted-results/parallel-install b/testsuite/wanted-results/parallel-install index f3b4105d..912de29a 100644 --- a/testsuite/wanted-results/parallel-install +++ b/testsuite/wanted-results/parallel-install @@ -13,7 +13,7 @@ test -n "" \ -exec chmod u+rwx,go+rx {} \; -o \ ! -type d ! -perm -444 -links 1 -exec chmod a+r {} \; -o \ ! -type d ! -perm -400 -exec chmod a+r {} \; -o \ - ! -type d ! -perm -444 -exec /bin/bash /mnt/4tb/home/tange/privat/parallel/install-sh -c -m a+r {} {} \; \ + ! -type d ! -perm -444 -exec /bin/bash /home/tange/privat/parallel/install-sh -c -m a+r {} {} \; \ || chmod -R a+r "parallel-00000000" tardir=parallel-00000000 && ${TAR-tar} chof - "$tardir" | GZIP=--best gzip -c >parallel-00000000.tar.gz make[0]: Leaving directory '/mnt/4tb/home/tange/privat/parallel' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 4d51f0df..d8d29af6 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -297,23 +297,23 @@ bug #45993: --wd ... should also work when run locally parallel: Error: Cannot change into non-executable dir /bi: No such file or directory parallel --wd /bin 'pwd; echo $OLDPWD; echo' ::: OK /bin -/mnt/4tb/home/tange/privat/parallel/testsuite +/home/tange/privat/parallel/testsuite OK parallel --wd / 'pwd; echo $OLDPWD; echo' ::: OK / -/mnt/4tb/home/tange/privat/parallel/testsuite +/home/tange/privat/parallel/testsuite OK parallel --wd /tmp 'pwd; echo $OLDPWD; echo' ::: OK /tmp -/mnt/4tb/home/tange/privat/parallel/testsuite +/home/tange/privat/parallel/testsuite OK parallel --wd ... 'pwd; echo $OLDPWD; echo' ::: OK | perl -pe 's/\d+/0/g' /home/tange/.parallel/tmp/aspire-0-0 -/mnt/0tb/home/tange/privat/parallel/testsuite +/home/tange/privat/parallel/testsuite OK parallel --wd . 'pwd; echo $OLDPWD; echo' ::: OK /home/tange/privat/parallel/testsuite -/mnt/4tb/home/tange/privat/parallel/testsuite +/home/tange/privat/parallel/testsuite OK echo '**' ** diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index f400de76..1c4271c8 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -3,8 +3,8 @@ par_mem_leak ### test for mem leak par_mem_leak no mem leak detected par_outside_file_handle_limit 2>&1 par_outside_file_handle_limit ### Test Force outside the file handle limit, 2009-02-17 Gave fork error -par_outside_file_handle_limit parallel: Warning: Only enough file handles to run 251 jobs in parallel. -par_outside_file_handle_limit parallel: Warning: Running 'parallel -j0 -N 251 --pipe parallel -j0' or +par_outside_file_handle_limit parallel: Warning: Only enough file handles to run 252 jobs in parallel. +par_outside_file_handle_limit parallel: Warning: Running 'parallel -j0 -N 252 --pipe parallel -j0' or par_outside_file_handle_limit parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help. par_outside_file_handle_limit Start par_outside_file_handle_limit end diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index bb7c4cf9..f29e7a45 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -1,7 +1,7 @@ echo '### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922' ### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922 seq 1000000 > /tmp/num1000000; stdout parallel --pipepart --progress -a /tmp/num1000000 --block 10k -j0 true |grep 1:local -1:local / 8 / 251 +1:local / 8 / 252 echo '**' ** testhalt() { echo '### testhalt --halt '$1; (yes 0 | head -n 10; seq 10) | stdout parallel -kj4 --halt $1 'sleep {= $_=$_*0.3+1 =}; exit {}'; echo $?; (seq 10; yes 0 | head -n 10) | stdout parallel -kj4 --halt $1 'sleep {= $_=$_*0.3+1 =}; exit {}'; echo $?; }; export -f testhalt; parallel -kj0 testhalt ::: now,fail=0 now,fail=1 now,fail=2 now,fail=30% now,fail=70% soon,fail=0 soon,fail=1 soon,fail=2 soon,fail=30% soon,fail=70% now,success=0 now,success=1 now,success=2 now,success=30% now,success=70% soon,success=0 soon,success=1 soon,success=2 soon,success=30% now,success=70% @@ -862,12 +862,12 @@ echo '**' ** echo "### Test max line length -m -I" ### Test max line length -m -I - seq 1 60000 | parallel -I :: -m -j1 echo a::b::c | sort >/tmp/114-a$$; md5sum /tmp/114-a$$; md5sum /tmp/114-b$$; md5sum /tmp/114-b$$; md5sum