diff --git a/src/parallel b/src/parallel index 58795118..9aac6ac6 100755 --- a/src/parallel +++ b/src/parallel @@ -82,7 +82,7 @@ you use this option, stdin is given to the first process run. Otherwise, stdin is redirected from /dev/null. -=item B<--cleanup> (not implemented) +=item B<--cleanup> Remove transfered files. B<--cleanup> will remove the transfered files on the remote server after processing is done. @@ -263,8 +263,8 @@ on remote machines). =item B<--number-of-cpus> -Print the number of CPUs and exit (used by GNU B itself to -determine the number of CPUs on remote machines). +Print the number of physical CPUs and exit (used by GNU B +itself to determine the number of physical CPUs on remote machines). =item B<--number-of-cores> (not implemented) @@ -298,7 +298,7 @@ default. If the standard input only contains whitespace, do not run the command. -=item B<--return> I (not implemented) +=item B<--return> I Transfer files from remote servers. B<--return> is used with B<--sshlogin> when the arguments are files on the remote servers. When @@ -415,7 +415,7 @@ it. See also B<-v>. -=item B<--transfer> (not implemented) +=item B<--transfer> Transfer files to remote servers. B<--transfer> is used with B<--sshlogin> when the arguments are files and should be transfered to @@ -441,7 +441,7 @@ B<--transfer> is often used with B<--return> and B<--cleanup>. B<--transfer> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>. -=item B<--trc> I (not implemented) +=item B<--trc> I Transfer, Return, Cleanup. Short hand for: @@ -453,7 +453,8 @@ Transfer, Return, Cleanup. Short hand for: =item B<-u> Ungroup output. Output is printed as soon as possible. This may cause -output from different commands to be mixed. Can be reversed with B<-g>. +output from different commands to be mixed. GNU B runs +faster with B<-u>. Can be reversed with B<-g>. =item B<--extensionreplace> I @@ -463,14 +464,16 @@ output from different commands to be mixed. Can be reversed with B<-g>. Use the replacement string I instead of {.} for input line without extension. -=item B<--use-cpus-instead-of-cores> (not implemented) +=item B<--use-cpus-instead-of-cores> -Count the number of CPUs instead of cores. When computing how many -jobs to run in parallel relative to the number of cores you can ask -GNU B to instead look at the number of CPUs. This will make sense -for computers that have hyperthreading as two jobs running on one CPU -with hyperthreading will run slower than two jobs running on two CPUs. -Most users will not need this option. +Count the number of physical CPUs instead of cores. When computing how +many jobs to run in parallel relative to the number of cores you can +ask GNU B to instead look at the number of physical +CPUs. This will make sense for computers that have hyperthreading as +two jobs running on one CPU with hyperthreading will run slower than +two jobs running on two physical CPUs. Some multi-core CPUs can run +faster if only one thread is running per physical CPU. Most users will +not need this option. =item B<-v> @@ -713,7 +716,7 @@ B<(echo foss.org.my; echo debian.org; echo freenetproject.org) | parallel -k tra This will make sure the traceroute to foss.org.my will be printed first. -=head1 EXAMPLE: Using remote computers (not implemented) +=head1 EXAMPLE: Using remote computers To run commands on a remote computer SSH needs to be set up and you must be able to login without entering a password (B may be @@ -762,7 +765,7 @@ server has 8 CPU cores. seq 1 10 | parallel --sshlogin 8/server.example.com echo -=head1 EXAMPLE: Transferring of files (not implemented) +=head1 EXAMPLE: Transferring of files To recompress gzipped files with B using a remote server run: @@ -1158,7 +1161,7 @@ drain_job_queue(); sub parse_options { # Defaults: - $Global::version = 20100516; + $Global::version = 20100529; $Global::progname = 'parallel'; $Global::debug = 0; $Global::verbose = 0; @@ -1193,6 +1196,7 @@ sub parse_options { "max-line-length-allowed" => \$::opt_max_line_length_allowed, "number-of-cpus" => \$::opt_number_of_cpus, "number-of-cores" => \$::opt_number_of_cores, + "use-cpus-instead-of-cores" => \$::opt_use_cpus_instead_of_cores, "sshlogin|S=s" => \@::opt_sshlogin, "sshloginfile=s" => \$::opt_sshloginfile, "return=s" => \@::opt_return, @@ -1261,7 +1265,7 @@ sub parse_options { } if(defined $::opt_a) { - if(not open(ARGFILE,"<".$::opt_a)) { + if(not open(ARGFILE,"<",$::opt_a)) { print STDERR "$Global::progname: Cannot open input file `$::opt_a': No such file or directory\n"; exit(-1); } @@ -1634,6 +1638,8 @@ sub processes_available_by_system_limit { } sub simultaneous_sshlogin { + # Using $sshlogin try to see if we can do $wanted_processes + # simultaneous logins my $sshlogin = shift; my $wanted_processes = shift; my ($sshcmd,$serverlogin) = sshcommand_of_sshlogin($sshlogin); @@ -1693,14 +1699,14 @@ sub user_requested_processes { if($opt_P =~ /^\+(\d+)$/) { # E.g. -P +2 my $j = $1; - $processes = $j + no_of_cpus_sshlogin($sshlogin); + $processes = $j + no_of_processing_units_sshlogin($sshlogin); } elsif ($opt_P =~ /^-(\d+)$/) { # E.g. -P -2 my $j = $1; - $processes = no_of_cpus_sshlogin($sshlogin) - $j; + $processes = no_of_processing_units_sshlogin($sshlogin) - $j; } elsif ($opt_P =~ /^(\d+)\%$/) { my $j = $1; - $processes = no_of_cpus_sshlogin($sshlogin) * $j / 100; + $processes = no_of_processing_units_sshlogin($sshlogin) * $j / 100; } elsif ($opt_P =~ /^(\d+)$/) { $processes = $1; if($processes == 0) { @@ -1717,21 +1723,26 @@ sub user_requested_processes { return $processes; } -sub no_of_cores { - # TODO This should return number of cores and not the number of CPUs - return no_of_cpus(@_); -} - -sub no_of_cpus_sshlogin { - # Number of CPUs at this sshlogin +sub no_of_processing_units_sshlogin { + # Number of processing units (CPUs or cores) at this sshlogin my $sshlogin = shift; my ($sshcmd,$serverlogin) = sshcommand_of_sshlogin($sshlogin); if(not $Global::host{$sshlogin}{'ncpus'}) { if($serverlogin eq ":") { - $Global::host{$sshlogin}{'ncpus'} = no_of_cpus(); + if($::opt_use_cpus_instead_of_cores) { + $Global::host{$sshlogin}{'ncpus'} = no_of_cpus(); + } else { + $Global::host{$sshlogin}{'ncpus'} = no_of_cores(); + } } else { - my $ncpu = qx(echo|$sshcmd $serverlogin parallel --number-of-cpus); - chomp($ncpu); + my $ncpu; + if($::opt_use_cpus_instead_of_cores) { + $ncpu = qx(echo|$sshcmd $serverlogin parallel --number-of-cpus); + chomp($ncpu); + } else { + $ncpu = qx(echo|$sshcmd $serverlogin parallel --number-of-cores); + chomp($ncpu); + } if($ncpu =~ /^[0-9]+$/) { $Global::host{$sshlogin}{'ncpus'} = $ncpu; } else { @@ -1746,37 +1757,90 @@ sub no_of_cpus_sshlogin { sub no_of_cpus { if(not $Global::no_of_cpus) { local $/="\n"; # If delimiter is set, then $/ will be wrong - my $no_of_cpus = (no_of_cpus_gnu_linux() + my $no_of_cpus = (0 + || no_of_cpus_freebsd() || no_of_cpus_darwin() - || no_of_cpus_solaris()); + || no_of_cpus_solaris() + || no_of_cpus_gnu_linux() + ); if($no_of_cpus) { $Global::no_of_cpus = $no_of_cpus; } else { - warn("Cannot figure out no of cpus. Using 1"); + warn("Cannot figure out number of cpus. Using 1"); $Global::no_of_cpus = 1; } } return $Global::no_of_cpus; } +sub no_of_cores { + if(not $Global::no_of_cores) { + local $/="\n"; # If delimiter is set, then $/ will be wrong + my $no_of_cores = (0 + || no_of_cores_freebsd() + || no_of_cores_darwin() + || no_of_cores_solaris() + || no_of_cores_gnu_linux() + ); + if($no_of_cores) { + $Global::no_of_cores = $no_of_cores; + } else { + warn("Cannot figure out number of CPU cores. Using 1"); + $Global::no_of_cores = 1; + } + } + return $Global::no_of_cores; +} + sub no_of_cpus_gnu_linux { my $no_of_cpus; if(-e "/proc/cpuinfo") { $no_of_cpus = 0; + my %seen; open(IN,"cat /proc/cpuinfo|") || return undef; while() { - /^processor.*[:]/ and $no_of_cpus++; + if(/^physical id.*[:](.*)/ and not $seen{$1}++) { + $no_of_cpus++; + } } close IN; } return $no_of_cpus; } +sub no_of_cores_gnu_linux { + my $no_of_cores; + if(-e "/proc/cpuinfo") { + $no_of_cores = 0; + open(IN,"cat /proc/cpuinfo|") || return undef; + while() { + /^processor.*[:]/ and $no_of_cores++; + } + close IN; + } + return $no_of_cores; +} + sub no_of_cpus_darwin { - my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`; + my $no_of_cpus = `sysctl -a hw 2>/dev/null | grep -w physicalcpu | awk '{ print \$2 }'`; return $no_of_cpus; } +sub no_of_cores_darwin { + my $no_of_cores = `sysctl -a hw 2>/dev/null | grep -w logicalcpu | awk '{ print \$2 }'`; + return $no_of_cores; +} + +sub no_of_cpus_freebsd { + my $no_of_cpus = `sysctl hw.ncpu 2>/dev/null | awk '{ print \$2 }'`; + return $no_of_cpus; +} + +sub no_of_cores_freebsd { + my $no_of_cores = `sysctl -a hw 2>/dev/null | grep -w logicalcpu | awk '{ print \$2 }'`; + return $no_of_cores; +} + sub no_of_cpus_solaris { if(-x "/usr/sbin/psrinfo") { my @psrinfo = `/usr/sbin/psrinfo`; @@ -1793,6 +1857,22 @@ sub no_of_cpus_solaris { return undef; } +sub no_of_cores_solaris { + if(-x "/usr/sbin/psrinfo") { + my @psrinfo = `/usr/sbin/psrinfo`; + if($#psrinfo >= 0) { + return $#psrinfo +1; + } + } + if(-x "/usr/sbin/prtconf") { + my @prtconf = `/usr/sbin/prtconf | grep cpu..instance`; + if($#prtconf >= 0) { + return $#prtconf +1; + } + } + return undef; +} + # # General useful library functions # @@ -2364,6 +2444,3 @@ $main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder = 0; # No: space Will make it hard to do: 8/server1,server2 # Maybe: / 8//usr/bin/myssh,//usr/bin/ssh # %/=:_^ - -# Check transfer of newline file to sshlogin with own ssh and more than 9 simultaneously -# TODO Unittest myssh: parallel --trc -j10000% -S "myssh -l tange nlv.pi.dk" echo diff --git a/unittest/actual-results/test17 b/unittest/actual-results/test17 index 96e0eda9..955663cf 100644 --- a/unittest/actual-results/test17 +++ b/unittest/actual-results/test17 @@ -45,7 +45,6 @@ file19 file20 good if no file ls: cannot access tmp/parallel.file*: No such file or directory -### --transfer - file with newline ### --transfer --cleanup - abspath file1 file2 @@ -93,7 +92,6 @@ file20 good if no file ls: cannot access tmp/parallel.file*: No such file or directory OK -### --transfer --cleanup - file with newline ### --return - abspath /tmp/parallel.file10.out /tmp/parallel.file13.out @@ -136,7 +134,6 @@ tmp/parallel.file9.out tmp/parallel.file>fire.out tmp/parallel.file : & ) \n*.jpg.out tmp/parallel.file/sub dir.out -### --return - file with newline ### --return - multiple files tmp/parallel.file10.file.done tmp/parallel.file10.out @@ -266,7 +263,6 @@ tmp/parallel.file/sub dir.out good if no file ls: cannot access tmp/parallel.file*: No such file or directory OK -### --return --cleanup - file with newline ### --return --cleanup - multiple returns /tmp/parallel.file10.file.done /tmp/parallel.file10.out @@ -399,7 +395,6 @@ OK good if no file ls: cannot access tmp/parallel.file*: No such file or directory OK -### --transfer --return --cleanup - file with newline ### --transfer --return --cleanup - multiple files /tmp/parallel.file10.file.done /tmp/parallel.file10.out @@ -531,7 +526,6 @@ tmp/parallel.file/sub dir.out good if no file ls: cannot access tmp/parallel.file*: No such file or directory OK -### --trc - file with newline ### --trc - multiple files /tmp/parallel.file10.file.done /tmp/parallel.file10.out diff --git a/unittest/actual-results/test19 b/unittest/actual-results/test19 new file mode 100644 index 00000000..608bafed --- /dev/null +++ b/unittest/actual-results/test19 @@ -0,0 +1,202 @@ +### Test --transfer --return --cleanup - files with newline +### --transfer - file with newline +newline +newline +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory +### --transfer --cleanup - file with newline +newline +newline +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory +### --return - file with newline +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline2.out +### --return --cleanup - file with newline +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline2.out +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --transfer --return --cleanup - file with newline +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline2.out +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --trc - file with newline +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline2.out +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --trc - multiple file with newline +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline1.out2 +tmp/parallel.file. +newline2.out +tmp/parallel.file. +newline2.out2 +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### Test use special ssh +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline1.out2 +tmp/parallel.file. +newline2.out +tmp/parallel.file. +newline2.out2 +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +Input for ssh +parallel-server1 rsync --server -lDrRze.iLsf . ./ +parallel-server1 cat tmp/parallel.file.' +'newline2 > tmp/parallel.file.' +'newline2.out;cat tmp/parallel.file.' +'newline2 > tmp/parallel.file.' +'newline2.out2 +parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.' +'newline2.out +parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.' +'newline2.out2 +parallel-server1 rm -f tmp/parallel.file.' +'newline2 +-l parallel parallel-server2 rsync --server -lDrRze.iLsf . ./ +parallel@parallel-server2 cat tmp/parallel.file.' +'newline1 > tmp/parallel.file.' +'newline1.out;cat tmp/parallel.file.' +'newline1 > tmp/parallel.file.' +'newline1.out2 +-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.' +'newline1.out +-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.' +'newline1.out2 +parallel@parallel-server2 rm -f tmp/parallel.file.' +'newline1 +### Test use special ssh with > 9 simultaneous +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 diff --git a/unittest/actual-results/test20 b/unittest/actual-results/test20 new file mode 100644 index 00000000..e80c2ed8 --- /dev/null +++ b/unittest/actual-results/test20 @@ -0,0 +1,8 @@ +### Test --number-of-cpus +1 +### Test --number-of-cores +2 +### Test --use-cpus-instead-of-cores +Cores should complete first on machines with less than 4 physical CPUs +cores done +CPUs done diff --git a/unittest/tests-to-run/test17.sh b/unittest/tests-to-run/test17.sh index 2177e334..227f850b 100755 --- a/unittest/tests-to-run/test17.sh +++ b/unittest/tests-to-run/test17.sh @@ -8,6 +8,8 @@ SERVER2=parallel-server2 echo '### Test --transfer --return --cleanup' rm -rf /tmp/parallel.file* +stdout ssh $SERVER1 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' +stdout ssh parallel@$SERVER2 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' (seq 1 3;echo '>fire';seq 5 10; echo ' : & ) \n*.jpg'; echo '/./sub dir'; seq 13 20) >/tmp/test17 # Create some weirdly files in /tmp mkdir -p /tmp/parallel.file @@ -21,7 +23,7 @@ stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*' cat /tmp/test17abs | $PAR -k --transfer --sshlogin $SERVER1,parallel@$SERVER2 cat {}";"rm {} # One of these should give the empty dir /tmp/parallel.file echo good if no file -stdout ssh $SERVER1 ls '/tmp/parallel.file*' +stdout ssh $SERVER1 ls '/tmp/parallel.file*' # The other: No such file or directory stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*' @@ -36,10 +38,6 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' # Should give: No such file or directory stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' -echo '### --transfer - file with newline' -## TODO - - echo '### --transfer --cleanup - abspath' cat /tmp/test17abs | $PAR -k --transfer --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {} echo good if no file @@ -56,9 +54,6 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK # Should give: No such file or directory stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK -echo '### --transfer --cleanup - file with newline' -# TODO - echo '### --return - abspath' rm -rf /tmp/parallel.file*out cat /tmp/test17abs | $PAR -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 echo {} ">"{.}.out @@ -69,9 +64,6 @@ rm -rf /tmp/parallel.file*out cat /tmp/test17rel | $PAR -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'echo {} ">"{.}.out ls tmp/parallel.file*out tmp/parallel.file/*out -echo '### --return - file with newline' -# TODO - echo '### --return - multiple files' rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done cat /tmp/test17rel | $PAR -k --return {.}.out --return {}.done \ @@ -98,10 +90,6 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK # Should give: No such file or directory stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK -echo '### --return --cleanup - file with newline' -# -# TODO - echo '### --return --cleanup - multiple returns' rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done cat /tmp/test17rel | $PAR -k --return {.}.out --return {}.done --cleanup \ @@ -133,9 +121,6 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK # Should give: No such file or directory stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK -echo '### --transfer --return --cleanup - file with newline' -# TODO - echo '### --transfer --return --cleanup - multiple files' rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done cat /tmp/test17rel | $PAR -k --transfer --return {.}.out --return {}.done --cleanup \ @@ -165,9 +150,6 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK # Should give: No such file or directory stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK -echo '### --trc - file with newline' -# TODO - echo '### --trc - multiple files' rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done cat /tmp/test17abs | $PAR -k --trc {.}.out --trc {}.done \ diff --git a/unittest/tests-to-run/test19.sh b/unittest/tests-to-run/test19.sh new file mode 100644 index 00000000..a845e859 --- /dev/null +++ b/unittest/tests-to-run/test19.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +# TODO return multiple + +PAR=parallel + +SERVER1=parallel-server1 +SERVER2=parallel-server2 + +echo '### Test --transfer --return --cleanup - files with newline' + +rm -rf /tmp/parallel.file* +stdout ssh $SERVER1 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' +stdout ssh parallel@$SERVER2 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' + +cd / +echo '### --transfer - file with newline' +echo newline > '/tmp/parallel.file. +newline1' +echo newline > '/tmp/parallel.file. +newline2' +find tmp/parallel*newline* -print0 | $PAR -0 -k --transfer --sshlogin $SERVER1,parallel@$SERVER2 cat {}";"rm {} +# Should give: No such file or directory +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' + +echo '### --transfer --cleanup - file with newline' +echo newline > '/tmp/parallel.file. +newline1' +echo newline > '/tmp/parallel.file. +newline2' +find tmp/parallel*newline* -print0 | $PAR -0 -k --transfer --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {} +# Should give: No such file or directory +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' + +echo '### --return - file with newline' +echo newline > '/tmp/parallel.file. +newline1' +echo newline > '/tmp/parallel.file. +newline2' +find tmp/parallel*newline* -print0 | $PAR -0 -k --return {}.out --sshlogin $SERVER1,parallel@$SERVER2 echo remote '>' {}.out +ls tmp/parallel*newline*out +rm tmp/parallel*newline*out +# Cleanup remote +stdout ssh $SERVER1 rm -rf 'tmp/parallel.file*' +stdout ssh parallel@$SERVER2 rm -rf 'tmp/parallel.file*' + +echo '### --return --cleanup - file with newline' +echo newline > '/tmp/parallel.file. +newline1' +echo newline > '/tmp/parallel.file. +newline2' +find tmp/parallel*newline* -print0 | $PAR -0 -k --return {}.out --cleanup --sshlogin $SERVER1,parallel@$SERVER2 echo remote '>' {}.out +ls tmp/parallel*newline*out +rm tmp/parallel*newline*out +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK + +echo '### --transfer --return --cleanup - file with newline' +echo newline > '/tmp/parallel.file. +newline1' +echo newline > '/tmp/parallel.file. +newline2' +find tmp/parallel*newline* -print0 | $PAR -0 -k --transfer --return {}.out --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {} '>' {}.out +ls tmp/parallel*newline*out +rm tmp/parallel*newline*out +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK + +echo '### --trc - file with newline' +echo newline > '/tmp/parallel.file. +newline1' +echo newline > '/tmp/parallel.file. +newline2' +find tmp/parallel*newline* -print0 | $PAR -0 -k --trc {}.out --sshlogin $SERVER1,parallel@$SERVER2 cat {} '>' {}.out +ls tmp/parallel*newline*out +rm tmp/parallel*newline*out +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK + +echo '### --trc - multiple file with newline' +echo newline > '/tmp/parallel.file. +newline1' +echo newline > '/tmp/parallel.file. +newline2' +find tmp/parallel*newline* -print0 | $PAR -0 -k --trc {}.out --trc {}.out2 --sshlogin $SERVER1,parallel@$SERVER2 cat {} '>' {}.out';'cat {} '>' {}.out2 +ls tmp/parallel*newline*out* +rm tmp/parallel*newline*out* +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK + +echo '### Test use special ssh' +echo 'ssh "$@"; echo "$@" >>/tmp/myssh1-run' >/tmp/myssh1 +echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2 +chmod 755 /tmp/myssh1 /tmp/myssh2 +rm -rf /tmp/myssh1-run /tmp/myssh2-run +echo newline > '/tmp/parallel.file. +newline1' +echo newline > '/tmp/parallel.file. +newline2' +find tmp/parallel*newline* -print0 | $PAR -0 -k -j1 --trc {}.out --trc {}.out2 \ + --sshlogin "/tmp/myssh1 $SERVER1, /tmp/myssh2 parallel@$SERVER2" \ + cat {} '>' {}.out';'cat {} '>' {}.out2 +ls tmp/parallel*newline*out* +rm tmp/parallel*newline*out* +echo good if no file +stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK +# Should give: No such file or directory +stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK +echo Input for ssh +cat /tmp/myssh1-run /tmp/myssh2-run +rm /tmp/myssh1-run /tmp/myssh2-run + +echo '### Test use special ssh with > 9 simultaneous' +echo 'ssh "$@"; echo "$@" >>/tmp/myssh1-run' >/tmp/myssh1 +echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2 +chmod 755 /tmp/myssh1 /tmp/myssh2 +seq 1 100 | $PAR --sshlogin "/tmp/myssh1 $SERVER1, /tmp/myssh2 parallel@$SERVER2" \ + -j10000% -k echo diff --git a/unittest/tests-to-run/test20.sh b/unittest/tests-to-run/test20.sh new file mode 100644 index 00000000..bdd7f8e8 --- /dev/null +++ b/unittest/tests-to-run/test20.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +PAR=parallel + +echo '### Test --number-of-cpus' +$PAR --number-of-cpus + +echo '### Test --number-of-cores' +$PAR --number-of-cores + +echo '### Test --use-cpus-instead-of-cores' +(seq 1 4 | $PAR --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done & +(seq 1 4 | $PAR -j100% sleep) && echo cores done & +echo 'Cores should complete first on machines with less than 4 physical CPUs' +wait + + diff --git a/unittest/wanted-results/test17 b/unittest/wanted-results/test17 index 1a34189c..955663cf 100644 --- a/unittest/wanted-results/test17 +++ b/unittest/wanted-results/test17 @@ -45,7 +45,6 @@ file19 file20 good if no file ls: cannot access tmp/parallel.file*: No such file or directory -### --transfer - file with newline ### --transfer --cleanup - abspath file1 file2 @@ -91,9 +90,8 @@ file18 file19 file20 good if no file -ls: cannot access /tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory OK -### --transfer --cleanup - file with newline ### --return - abspath /tmp/parallel.file10.out /tmp/parallel.file13.out @@ -116,68 +114,67 @@ OK /tmp/parallel.file : & ) \n*.jpg.out /tmp/parallel.file/sub dir.out ### --return - relpath -/tmp/parallel.file10.out -/tmp/parallel.file13.out -/tmp/parallel.file14.out -/tmp/parallel.file15.out -/tmp/parallel.file16.out -/tmp/parallel.file17.out -/tmp/parallel.file18.out -/tmp/parallel.file19.out -/tmp/parallel.file1.out -/tmp/parallel.file20.out -/tmp/parallel.file2.out -/tmp/parallel.file3.out -/tmp/parallel.file5.out -/tmp/parallel.file6.out -/tmp/parallel.file7.out -/tmp/parallel.file8.out -/tmp/parallel.file9.out -/tmp/parallel.file>fire.out -/tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.out -### --return - file with newline +tmp/parallel.file10.out +tmp/parallel.file13.out +tmp/parallel.file14.out +tmp/parallel.file15.out +tmp/parallel.file16.out +tmp/parallel.file17.out +tmp/parallel.file18.out +tmp/parallel.file19.out +tmp/parallel.file1.out +tmp/parallel.file20.out +tmp/parallel.file2.out +tmp/parallel.file3.out +tmp/parallel.file5.out +tmp/parallel.file6.out +tmp/parallel.file7.out +tmp/parallel.file8.out +tmp/parallel.file9.out +tmp/parallel.file>fire.out +tmp/parallel.file : & ) \n*.jpg.out +tmp/parallel.file/sub dir.out ### --return - multiple files -/tmp/parallel.file10.file.done -/tmp/parallel.file10.out -/tmp/parallel.file13.file.done -/tmp/parallel.file13.out -/tmp/parallel.file14.file.done -/tmp/parallel.file14.out -/tmp/parallel.file15.file.done -/tmp/parallel.file15.out -/tmp/parallel.file16.file.done -/tmp/parallel.file16.out -/tmp/parallel.file17.file.done -/tmp/parallel.file17.out -/tmp/parallel.file18.file.done -/tmp/parallel.file18.out -/tmp/parallel.file19.file.done -/tmp/parallel.file19.out -/tmp/parallel.file1.file.done -/tmp/parallel.file1.out -/tmp/parallel.file20.file.done -/tmp/parallel.file20.out -/tmp/parallel.file2.file.done -/tmp/parallel.file2.out -/tmp/parallel.file3.file.done -/tmp/parallel.file3.out -/tmp/parallel.file5.file.done -/tmp/parallel.file5.out -/tmp/parallel.file6.file.done -/tmp/parallel.file6.out -/tmp/parallel.file7.file.done -/tmp/parallel.file7.out -/tmp/parallel.file8.file.done -/tmp/parallel.file8.out -/tmp/parallel.file9.file.done -/tmp/parallel.file9.out -/tmp/parallel.file>fire.file.done -/tmp/parallel.file>fire.out -/tmp/parallel.file : & ) \n*.jpg.file.done -/tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.file.done -/tmp/parallel.file/sub dir.out +tmp/parallel.file10.file.done +tmp/parallel.file10.out +tmp/parallel.file13.file.done +tmp/parallel.file13.out +tmp/parallel.file14.file.done +tmp/parallel.file14.out +tmp/parallel.file15.file.done +tmp/parallel.file15.out +tmp/parallel.file16.file.done +tmp/parallel.file16.out +tmp/parallel.file17.file.done +tmp/parallel.file17.out +tmp/parallel.file18.file.done +tmp/parallel.file18.out +tmp/parallel.file19.file.done +tmp/parallel.file19.out +tmp/parallel.file1.file.done +tmp/parallel.file1.out +tmp/parallel.file20.file.done +tmp/parallel.file20.out +tmp/parallel.file2.file.done +tmp/parallel.file2.out +tmp/parallel.file3.file.done +tmp/parallel.file3.out +tmp/parallel.file5.file.done +tmp/parallel.file5.out +tmp/parallel.file6.file.done +tmp/parallel.file6.out +tmp/parallel.file7.file.done +tmp/parallel.file7.out +tmp/parallel.file8.file.done +tmp/parallel.file8.out +tmp/parallel.file9.file.done +tmp/parallel.file9.out +tmp/parallel.file>fire.file.done +tmp/parallel.file>fire.out +tmp/parallel.file : & ) \n*.jpg.file.done +tmp/parallel.file : & ) \n*.jpg.out +tmp/parallel.file/sub dir.file.done +tmp/parallel.file/sub dir.out ### --return --cleanup - abspath /tmp/parallel.file10.file.done /tmp/parallel.file10.out @@ -223,50 +220,49 @@ good if no file ls: cannot access /tmp/parallel.file*: No such file or directory OK ### --return --cleanup - relpath -/tmp/parallel.file10.file.done -/tmp/parallel.file10.out -/tmp/parallel.file13.file.done -/tmp/parallel.file13.out -/tmp/parallel.file14.file.done -/tmp/parallel.file14.out -/tmp/parallel.file15.file.done -/tmp/parallel.file15.out -/tmp/parallel.file16.file.done -/tmp/parallel.file16.out -/tmp/parallel.file17.file.done -/tmp/parallel.file17.out -/tmp/parallel.file18.file.done -/tmp/parallel.file18.out -/tmp/parallel.file19.file.done -/tmp/parallel.file19.out -/tmp/parallel.file1.file.done -/tmp/parallel.file1.out -/tmp/parallel.file20.file.done -/tmp/parallel.file20.out -/tmp/parallel.file2.file.done -/tmp/parallel.file2.out -/tmp/parallel.file3.file.done -/tmp/parallel.file3.out -/tmp/parallel.file5.file.done -/tmp/parallel.file5.out -/tmp/parallel.file6.file.done -/tmp/parallel.file6.out -/tmp/parallel.file7.file.done -/tmp/parallel.file7.out -/tmp/parallel.file8.file.done -/tmp/parallel.file8.out -/tmp/parallel.file9.file.done -/tmp/parallel.file9.out -/tmp/parallel.file>fire.file.done -/tmp/parallel.file>fire.out -/tmp/parallel.file : & ) \n*.jpg.file.done -/tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.file.done -/tmp/parallel.file/sub dir.out +tmp/parallel.file10.file.done +tmp/parallel.file10.out +tmp/parallel.file13.file.done +tmp/parallel.file13.out +tmp/parallel.file14.file.done +tmp/parallel.file14.out +tmp/parallel.file15.file.done +tmp/parallel.file15.out +tmp/parallel.file16.file.done +tmp/parallel.file16.out +tmp/parallel.file17.file.done +tmp/parallel.file17.out +tmp/parallel.file18.file.done +tmp/parallel.file18.out +tmp/parallel.file19.file.done +tmp/parallel.file19.out +tmp/parallel.file1.file.done +tmp/parallel.file1.out +tmp/parallel.file20.file.done +tmp/parallel.file20.out +tmp/parallel.file2.file.done +tmp/parallel.file2.out +tmp/parallel.file3.file.done +tmp/parallel.file3.out +tmp/parallel.file5.file.done +tmp/parallel.file5.out +tmp/parallel.file6.file.done +tmp/parallel.file6.out +tmp/parallel.file7.file.done +tmp/parallel.file7.out +tmp/parallel.file8.file.done +tmp/parallel.file8.out +tmp/parallel.file9.file.done +tmp/parallel.file9.out +tmp/parallel.file>fire.file.done +tmp/parallel.file>fire.out +tmp/parallel.file : & ) \n*.jpg.file.done +tmp/parallel.file : & ) \n*.jpg.out +tmp/parallel.file/sub dir.file.done +tmp/parallel.file/sub dir.out good if no file -ls: cannot access /tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory OK -### --return --cleanup - file with newline ### --return --cleanup - multiple returns /tmp/parallel.file10.file.done /tmp/parallel.file10.out @@ -309,7 +305,7 @@ OK /tmp/parallel.file/sub dir.file.done /tmp/parallel.file/sub dir.out good if no file -ls: cannot access /tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory OK ### --transfer --return --cleanup - abspath /tmp/parallel.file10.file.done @@ -397,9 +393,8 @@ OK /tmp/parallel.file/sub dir.file.done /tmp/parallel.file/sub dir.out good if no file -ls: cannot access /tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory OK -### --transfer --return --cleanup - file with newline ### --transfer --return --cleanup - multiple files /tmp/parallel.file10.file.done /tmp/parallel.file10.out @@ -441,7 +436,7 @@ OK /tmp/parallel.file : & ) \n*.jpg.out /tmp/parallel.file/sub dir.file.done /tmp/parallel.file/sub dir.out -ls: cannot access /tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory OK ### --trc - abspath /tmp/parallel.file10.file.done @@ -488,50 +483,49 @@ good if no file ls: cannot access /tmp/parallel.file*: No such file or directory OK ### --trc - relpath -/tmp/parallel.file10.file.done -/tmp/parallel.file10.out -/tmp/parallel.file13.file.done -/tmp/parallel.file13.out -/tmp/parallel.file14.file.done -/tmp/parallel.file14.out -/tmp/parallel.file15.file.done -/tmp/parallel.file15.out -/tmp/parallel.file16.file.done -/tmp/parallel.file16.out -/tmp/parallel.file17.file.done -/tmp/parallel.file17.out -/tmp/parallel.file18.file.done -/tmp/parallel.file18.out -/tmp/parallel.file19.file.done -/tmp/parallel.file19.out -/tmp/parallel.file1.file.done -/tmp/parallel.file1.out -/tmp/parallel.file20.file.done -/tmp/parallel.file20.out -/tmp/parallel.file2.file.done -/tmp/parallel.file2.out -/tmp/parallel.file3.file.done -/tmp/parallel.file3.out -/tmp/parallel.file5.file.done -/tmp/parallel.file5.out -/tmp/parallel.file6.file.done -/tmp/parallel.file6.out -/tmp/parallel.file7.file.done -/tmp/parallel.file7.out -/tmp/parallel.file8.file.done -/tmp/parallel.file8.out -/tmp/parallel.file9.file.done -/tmp/parallel.file9.out -/tmp/parallel.file>fire.file.done -/tmp/parallel.file>fire.out -/tmp/parallel.file : & ) \n*.jpg.file.done -/tmp/parallel.file : & ) \n*.jpg.out -/tmp/parallel.file/sub dir.file.done -/tmp/parallel.file/sub dir.out +tmp/parallel.file10.file.done +tmp/parallel.file10.out +tmp/parallel.file13.file.done +tmp/parallel.file13.out +tmp/parallel.file14.file.done +tmp/parallel.file14.out +tmp/parallel.file15.file.done +tmp/parallel.file15.out +tmp/parallel.file16.file.done +tmp/parallel.file16.out +tmp/parallel.file17.file.done +tmp/parallel.file17.out +tmp/parallel.file18.file.done +tmp/parallel.file18.out +tmp/parallel.file19.file.done +tmp/parallel.file19.out +tmp/parallel.file1.file.done +tmp/parallel.file1.out +tmp/parallel.file20.file.done +tmp/parallel.file20.out +tmp/parallel.file2.file.done +tmp/parallel.file2.out +tmp/parallel.file3.file.done +tmp/parallel.file3.out +tmp/parallel.file5.file.done +tmp/parallel.file5.out +tmp/parallel.file6.file.done +tmp/parallel.file6.out +tmp/parallel.file7.file.done +tmp/parallel.file7.out +tmp/parallel.file8.file.done +tmp/parallel.file8.out +tmp/parallel.file9.file.done +tmp/parallel.file9.out +tmp/parallel.file>fire.file.done +tmp/parallel.file>fire.out +tmp/parallel.file : & ) \n*.jpg.file.done +tmp/parallel.file : & ) \n*.jpg.out +tmp/parallel.file/sub dir.file.done +tmp/parallel.file/sub dir.out good if no file -ls: cannot access /tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory OK -### --trc - file with newline ### --trc - multiple files /tmp/parallel.file10.file.done /tmp/parallel.file10.out diff --git a/unittest/wanted-results/test19 b/unittest/wanted-results/test19 new file mode 100644 index 00000000..608bafed --- /dev/null +++ b/unittest/wanted-results/test19 @@ -0,0 +1,202 @@ +### Test --transfer --return --cleanup - files with newline +### --transfer - file with newline +newline +newline +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory +### --transfer --cleanup - file with newline +newline +newline +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +ls: cannot access tmp/parallel.file*: No such file or directory +### --return - file with newline +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline2.out +### --return --cleanup - file with newline +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline2.out +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --transfer --return --cleanup - file with newline +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline2.out +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --trc - file with newline +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline2.out +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### --trc - multiple file with newline +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline1.out2 +tmp/parallel.file. +newline2.out +tmp/parallel.file. +newline2.out2 +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +### Test use special ssh +tmp/parallel.file. +newline1.out +tmp/parallel.file. +newline1.out2 +tmp/parallel.file. +newline2.out +tmp/parallel.file. +newline2.out2 +good if no file +ls: cannot access tmp/parallel.file*: No such file or directory +OK +ls: cannot access tmp/parallel.file*: No such file or directory +OK +Input for ssh +parallel-server1 rsync --server -lDrRze.iLsf . ./ +parallel-server1 cat tmp/parallel.file.' +'newline2 > tmp/parallel.file.' +'newline2.out;cat tmp/parallel.file.' +'newline2 > tmp/parallel.file.' +'newline2.out2 +parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.' +'newline2.out +parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.' +'newline2.out2 +parallel-server1 rm -f tmp/parallel.file.' +'newline2 +-l parallel parallel-server2 rsync --server -lDrRze.iLsf . ./ +parallel@parallel-server2 cat tmp/parallel.file.' +'newline1 > tmp/parallel.file.' +'newline1.out;cat tmp/parallel.file.' +'newline1 > tmp/parallel.file.' +'newline1.out2 +-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.' +'newline1.out +-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.' +'newline1.out2 +parallel@parallel-server2 rm -f tmp/parallel.file.' +'newline1 +### Test use special ssh with > 9 simultaneous +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 diff --git a/unittest/wanted-results/test20 b/unittest/wanted-results/test20 new file mode 100644 index 00000000..e80c2ed8 --- /dev/null +++ b/unittest/wanted-results/test20 @@ -0,0 +1,8 @@ +### Test --number-of-cpus +1 +### Test --number-of-cores +2 +### Test --use-cpus-instead-of-cores +Cores should complete first on machines with less than 4 physical CPUs +cores done +CPUs done