tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: More general --tmux length detection.

Browse source
This commit is contained in:
Ole Tange 2015-03-17 20:44:09 +01:00
parent 8990c7bb42
commit 815d00ac38
6 changed files with 53 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README
View file

@ -89,8 +89,8 @@ will love you for it.
When using programs that use GNU Parallel to process data for When using programs that use GNU Parallel to process data for
publication please cite: publication please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool, O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47. ;login: The USENIX Magazine, February 2011:42-47.
= New versions = = New versions =

57
src/parallel
View file

@ -2248,7 +2248,7 @@ sub progress {
$last_column_time = time; $last_column_time = time;
$columns = $ENV{'COLUMNS'}; $columns = $ENV{'COLUMNS'};
if(not $columns) { if(not $columns) {
my $resize = qx{ resize 2>/dev/null }; my $resize = qx{ sh -c 'resize 2>/dev/null' };
$resize =~ /COLUMNS=(\d+);/ and do { $columns = $1; }; $resize =~ /COLUMNS=(\d+);/ and do { $columns = $1; };
} }
$columns ||= 80; $columns ||= 80;
@ -2697,6 +2697,7 @@ sub parallelized_host_filtering {
# will make it less likely to overload the ssh daemon. # will make it less likely to overload the ssh daemon.
# --retries 3: If the ssh daemon it overloaded, try 3 times # --retries 3: If the ssh daemon it overloaded, try 3 times
# -s 16000: Half of the max line on UnixWare # -s 16000: Half of the max line on UnixWare
# TODO sh -c wrapper to work in csh
my $cmd = "cat $tmpfile | $0 -j0 --timeout 5 -s 16000 --joblog - --plain --delay 0.1 --retries 3 --tag --tagstring {1} -0 --colsep '\t' -k eval {2} 2>/dev/null"; my $cmd = "cat $tmpfile | $0 -j0 --timeout 5 -s 16000 --joblog - --plain --delay 0.1 --retries 3 --tag --tagstring {1} -0 --colsep '\t' -k eval {2} 2>/dev/null";
::debug("init", $cmd, "\n"); ::debug("init", $cmd, "\n");
my @out; my @out;
@ -4502,6 +4503,7 @@ sub simultaneous_sshlogin {
my $sshcmd = $self->sshcommand(); my $sshcmd = $self->sshcommand();
my $serverlogin = $self->serverlogin(); my $serverlogin = $self->serverlogin();
my $sshdelay = $opt::sshdelay ? "sleep $opt::sshdelay;" : ""; my $sshdelay = $opt::sshdelay ? "sleep $opt::sshdelay;" : "";
# TODO sh -c wrapper to work for csh
my $cmd = "$sshdelay$sshcmd $serverlogin echo simultaneouslogin </dev/null 2>&1 &"x$wanted_processes; my $cmd = "$sshdelay$sshcmd $serverlogin echo simultaneouslogin </dev/null 2>&1 &"x$wanted_processes;
::debug("init", "Trying $wanted_processes logins at $serverlogin\n"); ::debug("init", "Trying $wanted_processes logins at $serverlogin\n");
open (my $simul_fh, "-|", "($cmd)|grep simultaneouslogin | wc -l") or open (my $simul_fh, "-|", "($cmd)|grep simultaneouslogin | wc -l") or
@ -4717,7 +4719,7 @@ sub no_of_cores {
sub nproc { sub nproc {
# Returns: # Returns:
# Number of cores using `nproc` # Number of cores using `nproc`
my $no_of_cores = `nproc 2>/dev/null`; my $no_of_cores = qx{ sh -c 'nproc 2>/dev/null' };
return $no_of_cores; return $no_of_cores;
} }
@ -4792,6 +4794,7 @@ sub no_of_cpus_freebsd {
# Returns: # Returns:
# Number of physical CPUs on FreeBSD # Number of physical CPUs on FreeBSD
# undef if not FreeBSD # undef if not FreeBSD
# TODO sh -c wrapper to work in csh
my $no_of_cpus = my $no_of_cpus =
(`sysctl -a dev.cpu 2>/dev/null | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }'` (`sysctl -a dev.cpu 2>/dev/null | grep \%parent | awk '{ print \$2 }' | uniq | wc -l | awk '{ print \$1 }'`
or or
@ -4804,6 +4807,7 @@ sub no_of_cores_freebsd {
# Returns: # Returns:
# Number of CPU cores on FreeBSD # Number of CPU cores on FreeBSD
# undef if not FreeBSD # undef if not FreeBSD
# TODO sh -c wrapper to work in csh
my $no_of_cores = my $no_of_cores =
(`sysctl hw.ncpu 2>/dev/null | awk '{ print \$2 }'` (`sysctl hw.ncpu 2>/dev/null | awk '{ print \$2 }'`
or or
@ -4816,7 +4820,7 @@ sub no_of_cpus_netbsd {
# Returns: # Returns:
# Number of physical CPUs on NetBSD # Number of physical CPUs on NetBSD
# undef if not NetBSD # undef if not NetBSD
my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`; my $no_of_cpus = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' };
chomp $no_of_cpus; chomp $no_of_cpus;
return $no_of_cpus; return $no_of_cpus;
} }
@ -4825,7 +4829,7 @@ sub no_of_cores_netbsd {
# Returns: # Returns:
# Number of CPU cores on NetBSD # Number of CPU cores on NetBSD
# undef if not NetBSD # undef if not NetBSD
my $no_of_cores = `sysctl -n hw.ncpu 2>/dev/null`; my $no_of_cores = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' };
chomp $no_of_cores; chomp $no_of_cores;
return $no_of_cores; return $no_of_cores;
} }
@ -4834,7 +4838,7 @@ sub no_of_cpus_openbsd {
# Returns: # Returns:
# Number of physical CPUs on OpenBSD # Number of physical CPUs on OpenBSD
# undef if not OpenBSD # undef if not OpenBSD
my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`; my $no_of_cpus = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' };
chomp $no_of_cpus; chomp $no_of_cpus;
return $no_of_cpus; return $no_of_cpus;
} }
@ -4843,7 +4847,7 @@ sub no_of_cores_openbsd {
# Returns: # Returns:
# Number of CPU cores on OpenBSD # Number of CPU cores on OpenBSD
# undef if not OpenBSD # undef if not OpenBSD
my $no_of_cores = `sysctl -n hw.ncpu 2>/dev/null`; my $no_of_cores = qx{ sh -c 'sysctl -n hw.ncpu 2>/dev/null' };
chomp $no_of_cores; chomp $no_of_cores;
return $no_of_cores; return $no_of_cores;
} }
@ -4852,7 +4856,7 @@ sub no_of_cpus_hurd {
# Returns: # Returns:
# Number of physical CPUs on HURD # Number of physical CPUs on HURD
# undef if not HURD # undef if not HURD
my $no_of_cpus = `nproc`; my $no_of_cpus = qx{ nproc };
chomp $no_of_cpus; chomp $no_of_cpus;
return $no_of_cpus; return $no_of_cpus;
} }
@ -4870,6 +4874,7 @@ sub no_of_cpus_darwin {
# Returns: # Returns:
# Number of physical CPUs on Mac Darwin # Number of physical CPUs on Mac Darwin
# undef if not Mac Darwin # undef if not Mac Darwin
# TODO sh -c wrapper to work in csh
my $no_of_cpus = my $no_of_cpus =
(`sysctl -n hw.physicalcpu 2>/dev/null` (`sysctl -n hw.physicalcpu 2>/dev/null`
or or
@ -4881,6 +4886,7 @@ sub no_of_cores_darwin {
# Returns: # Returns:
# Number of CPU cores on Mac Darwin # Number of CPU cores on Mac Darwin
# undef if not Mac Darwin # undef if not Mac Darwin
# TODO sh -c wrapper to work in csh
my $no_of_cores = my $no_of_cores =
(`sysctl -n hw.logicalcpu 2>/dev/null` (`sysctl -n hw.logicalcpu 2>/dev/null`
or or
@ -4961,7 +4967,7 @@ sub no_of_cpus_hpux {
# Number of physical CPUs on HP-UX # Number of physical CPUs on HP-UX
# undef if not HP-UX # undef if not HP-UX
my $no_of_cpus = my $no_of_cpus =
qx{/usr/bin/mpsched -s 2>&1 | grep 'Locality Domain Count' | awk '{ print \$4 }'}; qx{ sh -c '/usr/bin/mpsched -s 2>&1' | grep 'Locality Domain Count' | awk '{ print \$4 }'};
return $no_of_cpus; return $no_of_cpus;
} }
@ -4970,7 +4976,7 @@ sub no_of_cores_hpux {
# Number of CPU cores on HP-UX # Number of CPU cores on HP-UX
# undef if not HP-UX # undef if not HP-UX
my $no_of_cores = my $no_of_cores =
qx{/usr/bin/mpsched -s 2>&1 | perl -ne '/Processor Count\\D+(\\d+)/ and print "\$1\n"'}; qx{ sh -c '/usr/bin/mpsched -s 2>&1' | perl -ne '/Processor Count\\D+(\\d+)/ and print "\$1\n"'};
return $no_of_cores; return $no_of_cores;
} }
@ -5188,7 +5194,7 @@ sub cleanup_cmd {
$dir .= $_."/"; $dir .= $_."/";
unshift @rmdir, ::shell_quote_file($dir); unshift @rmdir, ::shell_quote_file($dir);
} }
my $rmdir = @rmdir ? "rmdir @rmdir >&/dev/null;" : ""; my $rmdir = @rmdir ? "sh -c 'rmdir @rmdir 2>/dev/null';" : "";
if(defined $opt::workdir and $opt::workdir eq "...") { if(defined $opt::workdir and $opt::workdir eq "...") {
$rmdir .= "rm -rf " . ::shell_quote_file($workdir).';'; $rmdir .= "rm -rf " . ::shell_quote_file($workdir).';';
} }
@ -6697,6 +6703,7 @@ sub print_dryrun_and_verbose {
unlink $tmuxsocket; unlink $tmuxsocket;
::status("See output with: $ENV{'TMUX'} -S $tmuxsocket attach\n"); ::status("See output with: $ENV{'TMUX'} -S $tmuxsocket attach\n");
} }
# TODO sh -c wrapper for >&
$tmux = $ENV{'TMUX'}." -S $tmuxsocket new-session -s p$$ -d 'sleep .2' >&/dev/null;" . $tmux = $ENV{'TMUX'}." -S $tmuxsocket new-session -s p$$ -d 'sleep .2' >&/dev/null;" .
$ENV{'TMUX'}." -S $tmuxsocket new-window -t p$$ -n $title"; $ENV{'TMUX'}." -S $tmuxsocket new-window -t p$$ -n $title";
@ -8112,26 +8119,16 @@ sub tmux_length {
my $len = shift; my $len = shift;
if($opt::tmux) { if($opt::tmux) {
$ENV{'TMUX'} ||= "tmux"; $ENV{'TMUX'} ||= "tmux";
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmb"); my @out;
$Global::unlink{$tmpfile}=1; for my $l (1, 2020, 16320, 100000, $len) {
close $fh; my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmb");
unlink $tmpfile; close $fh;
my $b2020 = "x"x2020; $Global::unlink{$tmpfile} = 1;
my $b16320 = "x"x16320; unlink $tmpfile;
my $b100000 = "x"x100000; my $tmuxcmd = "sh -c '".$ENV{'TMUX'}." -S $tmpfile new-session -d -n echo $l".
my $blen = "x"x$len; ("x"x$l). " 2>/dev/null' && echo $l";
my @out = qx{ push @out, qx{ $tmuxcmd };
sleep .2; }
echo 1;
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo 2020$b2020 2>/dev/null && echo 2020;
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo 16320$b16320 2>/dev/null && echo 16320;
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo 100000$b100000 2>/dev/null && echo 100000;
},
# $len can be > 131072 which will confuse bash, so run it on its own
qx{
$ENV{'TMUX'} -S $tmpfile new-session -d -n echo $len$blen 2>/dev/null && echo $len;
};
unlink $tmpfile;
::debug("tmux","tmux-length ",@out); ::debug("tmux","tmux-length ",@out);
chomp @out; chomp @out;
# The arguments is given 3 times on the command line # The arguments is given 3 times on the command line

17
testsuite/Makefile
View file

@ -2,13 +2,19 @@ testsuite: 3
true true
3: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote 3: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote
TRIES=3 time sh Start.sh || true TRIES=3 time sh Start.sh - mem || true
touch ~/.parallel/will-cite touch ~/.parallel/will-cite
date date
make stopvm make stopvm
1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote 1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
time sh Start.sh || true time sh Start.sh - mem || true
touch ~/.parallel/will-cite
date
make stopvm
mem: ../src/parallel tests-to-run/*mem* wanted-results/*mem* startdb prereqlocal prereqremote
time sh Start.sh mem NONE || true
touch ~/.parallel/will-cite touch ~/.parallel/will-cite
date date
make stopvm make stopvm
@ -49,9 +55,10 @@ startvm:
VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true
stopvm: stopvm:
VBoxManage controlvm CentOS3-root:centos3 savestate # || true - because this should not fail if the VM is not running
VBoxManage controlvm RedHat9-root:redhat9 savestate VBoxManage controlvm CentOS3-root:centos3 savestate || true
VBoxManage controlvm OracleXE savestate VBoxManage controlvm RedHat9-root:redhat9 savestate || true
VBoxManage controlvm OracleXE savestate || true
installparallel: ../src/parallel installparallel: ../src/parallel
cd .. && make -j && sudo make -j install cd .. && make -j && sudo make -j install

2
testsuite/Start.sh
View file

@ -11,12 +11,14 @@ if [ "$TRIES" = "3" ] ; then
# Try a failing test thrice # Try a failing test thrice
echo Retrying 3 times echo Retrying 3 times
ls -t tests-to-run/*${1}*.sh | ls -t tests-to-run/*${1}*.sh |
grep -v ${2} |
perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh: ' \ perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh: ' \
>$SHFILE >$SHFILE
else else
# Run a failing test once # Run a failing test once
echo Not retrying echo Not retrying
ls -t tests-to-run/*${1}*.sh | ls -t tests-to-run/*${1}*.sh |
grep -v ${2} |
perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh:' \ perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh:' \
>$SHFILE >$SHFILE
fi fi

6
testsuite/tests-to-run/parallel-local-0.3s.sh
View file

@ -62,10 +62,16 @@ echo '### bug #43817: Some JP char cause problems in positional replacement stri
echo '**' echo '**'
echo '### --rpl % that is a substring of longer --rpl %D'
parallel --plus --rpl '%' parallel --plus --rpl '%'
--rpl '%D $_=::shell_quote(::dirname($_));' --rpl '%B s:.*/::;s:\.[^/.]+$::;' --rpl '%E s:.*\.::' --rpl '%D $_=::shell_quote(::dirname($_));' --rpl '%B s:.*/::;s:\.[^/.]+$::;' --rpl '%E s:.*\.::'
'echo {}=%;echo %D={//};echo %B={/.};echo %E={+.};echo %D/%B.%E={}' ::: a.b/c.d/e.f 'echo {}=%;echo %D={//};echo %B={/.};echo %E={+.};echo %D/%B.%E={}' ::: a.b/c.d/e.f
echo '**' echo '**'
echo '### Disk full'
cat /dev/zero >/mnt/ram/out;
parallel --tmpdir /mnt/ram echo ::: OK;
rm /mnt/ram/out
EOF EOF

8
testsuite/wanted-results/parallel-local-mem
View file

@ -1,8 +1,8 @@
make[2]: Entering directory `/home/tange/privat/parallel/testsuite' make[1]: Entering directory `/home/tange/privat/parallel/testsuite'
VBoxManage controlvm CentOS3-root:centos3 savestate VBoxManage controlvm CentOS3-root:centos3 savestate
VBoxManage controlvm RedHat9-root:redhat9 savestate VBoxManage controlvm RedHat9-root:redhat9 savestate
VBoxManage controlvm OracleXE savestate VBoxManage controlvm OracleXE savestate
make[2]: Leaving directory `/home/tange/privat/parallel/testsuite' make[1]: Leaving directory `/home/tange/privat/parallel/testsuite'
echo '### Trouble reading a record > 2 GB for certain versions of Perl (substr($a,0,2G+1)="fails")' echo '### Trouble reading a record > 2 GB for certain versions of Perl (substr($a,0,2G+1)="fails")'
### Trouble reading a record > 2 GB for certain versions of Perl (substr($a,0,2G+1)="fails") ### Trouble reading a record > 2 GB for certain versions of Perl (substr($a,0,2G+1)="fails")
echo '### perl -e $buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print length $buf' echo '### perl -e $buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print length $buf'
@ -49,7 +49,7 @@ echo '### -L >4GB'
parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001 parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001
parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003 parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003
parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647 parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647
make[2]: Entering directory `/home/tange/privat/parallel/testsuite' make[1]: Entering directory `/home/tange/privat/parallel/testsuite'
# Make sure we can reach the virtual machines # Make sure we can reach the virtual machines
sudo ifconfig wlan0:0 192.168.1.72 sudo ifconfig wlan0:0 192.168.1.72
# If they are already running: Don't fail # If they are already running: Don't fail
@ -59,4 +59,4 @@ VM "CentOS3-root:centos3" has been successfully started.
VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true VBoxManage startvm RedHat9-root:redhat9 || VBoxManage controlvm RedHat9-root:redhat9 resume || true
Waiting for VM "RedHat9-root:redhat9" to power on... Waiting for VM "RedHat9-root:redhat9" to power on...
VM "RedHat9-root:redhat9" has been successfully started. VM "RedHat9-root:redhat9" has been successfully started.
make[2]: Leaving directory `/home/tange/privat/parallel/testsuite' make[1]: Leaving directory `/home/tange/privat/parallel/testsuite'