From fd44e4c793b2cc08c509c64220a532c63f2db7e8 Mon Sep 17 00:00:00 2001
From: Ole Tange <ole@tange.dk>
Date: Mon, 31 May 2010 00:30:01 +0200
Subject: [PATCH] --use-cpus-instead-of-cores implemented with unittests. More
 unittests for remote usage (--trc, myssh, and newline files)

---
 src/parallel                    | 155 ++++++++++++-----
 unittest/actual-results/test17  |   6 -
 unittest/actual-results/test19  | 202 ++++++++++++++++++++++
 unittest/actual-results/test20  |   8 +
 unittest/tests-to-run/test17.sh |  24 +--
 unittest/tests-to-run/test19.sh | 132 ++++++++++++++
 unittest/tests-to-run/test20.sh |  17 ++
 unittest/wanted-results/test17  | 298 ++++++++++++++++----------------
 unittest/wanted-results/test19  | 202 ++++++++++++++++++++++
 unittest/wanted-results/test20  |   8 +
 10 files changed, 834 insertions(+), 218 deletions(-)
 create mode 100644 unittest/actual-results/test19
 create mode 100644 unittest/actual-results/test20
 create mode 100644 unittest/tests-to-run/test19.sh
 create mode 100644 unittest/tests-to-run/test20.sh
 create mode 100644 unittest/wanted-results/test19
 create mode 100644 unittest/wanted-results/test20

diff --git a/src/parallel b/src/parallel
index 58795118..9aac6ac6 100755
--- a/src/parallel
+++ b/src/parallel
@@ -82,7 +82,7 @@ you use this option, stdin is given to the first process run.
 Otherwise, stdin is redirected from /dev/null.
 
 
-=item B<--cleanup> (not implemented)
+=item B<--cleanup>
 
 Remove transfered files. B<--cleanup> will remove the transfered files
 on the remote server after processing is done.
@@ -263,8 +263,8 @@ on remote machines).
 
 =item B<--number-of-cpus>
 
-Print the number of CPUs and exit (used by GNU B<parallel> itself to
-determine the number of CPUs on remote machines).
+Print the number of physical CPUs and exit (used by GNU B<parallel>
+itself to determine the number of physical CPUs on remote machines).
 
 
 =item B<--number-of-cores> (not implemented)
@@ -298,7 +298,7 @@ default.
 
 If the standard input only contains whitespace, do not run the command.
 
-=item B<--return> I<filename> (not implemented)
+=item B<--return> I<filename>
 
 Transfer files from remote servers. B<--return> is used with
 B<--sshlogin> when the arguments are files on the remote servers. When
@@ -415,7 +415,7 @@ it.
 See also B<-v>.
 
 
-=item B<--transfer> (not implemented)
+=item B<--transfer>
 
 Transfer files to remote servers. B<--transfer> is used with
 B<--sshlogin> when the arguments are files and should be transfered to
@@ -441,7 +441,7 @@ B<--transfer> is often used with B<--return> and B<--cleanup>.
 B<--transfer> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>.
 
 
-=item B<--trc> I<filename> (not implemented)
+=item B<--trc> I<filename>
 
 Transfer, Return, Cleanup. Short hand for:
 
@@ -453,7 +453,8 @@ Transfer, Return, Cleanup. Short hand for:
 =item B<-u>
 
 Ungroup output.  Output is printed as soon as possible. This may cause
-output from different commands to be mixed. Can be reversed with B<-g>.
+output from different commands to be mixed. GNU B<parallel> runs
+faster with B<-u>. Can be reversed with B<-g>.
 
 
 =item B<--extensionreplace> I<replace-str>
@@ -463,14 +464,16 @@ output from different commands to be mixed. Can be reversed with B<-g>.
 Use the replacement string I<replace-str> instead of {.} for input line without extension.
 
 
-=item B<--use-cpus-instead-of-cores> (not implemented)
+=item B<--use-cpus-instead-of-cores>
 
-Count the number of CPUs instead of cores. When computing how many
-jobs to run in parallel relative to the number of cores you can ask
-GNU B<parallel> to instead look at the number of CPUs. This will make sense
-for computers that have hyperthreading as two jobs running on one CPU
-with hyperthreading will run slower than two jobs running on two CPUs.
-Most users will not need this option.
+Count the number of physical CPUs instead of cores. When computing how
+many jobs to run in parallel relative to the number of cores you can
+ask GNU B<parallel> to instead look at the number of physical
+CPUs. This will make sense for computers that have hyperthreading as
+two jobs running on one CPU with hyperthreading will run slower than
+two jobs running on two physical CPUs. Some multi-core CPUs can run
+faster if only one thread is running per physical CPU. Most users will
+not need this option.
 
 
 =item B<-v>
@@ -713,7 +716,7 @@ B<(echo foss.org.my; echo debian.org; echo freenetproject.org) | parallel -k tra
 This will make sure the traceroute to foss.org.my will be printed
 first.
 
-=head1 EXAMPLE: Using remote computers (not implemented)
+=head1 EXAMPLE: Using remote computers
 
 To run commands on a remote computer SSH needs to be set up and you
 must be able to login without entering a password (B<ssh-agent> may be
@@ -762,7 +765,7 @@ server has 8 CPU cores.
   seq 1 10 | parallel --sshlogin 8/server.example.com echo
 
 
-=head1 EXAMPLE: Transferring of files (not implemented)
+=head1 EXAMPLE: Transferring of files
 
 To recompress gzipped files with B<bzip2> using a remote server run:
 
@@ -1158,7 +1161,7 @@ drain_job_queue();
 
 sub parse_options {
     # Defaults:
-    $Global::version = 20100516;
+    $Global::version = 20100529;
     $Global::progname = 'parallel';
     $Global::debug = 0;
     $Global::verbose = 0;
@@ -1193,6 +1196,7 @@ sub parse_options {
 	       "max-line-length-allowed" => \$::opt_max_line_length_allowed,
 	       "number-of-cpus" => \$::opt_number_of_cpus,
 	       "number-of-cores" => \$::opt_number_of_cores,
+	       "use-cpus-instead-of-cores" => \$::opt_use_cpus_instead_of_cores,
 	       "sshlogin|S=s" => \@::opt_sshlogin,
 	       "sshloginfile=s" => \$::opt_sshloginfile,
 	       "return=s" => \@::opt_return,
@@ -1261,7 +1265,7 @@ sub parse_options {
     }
 
     if(defined $::opt_a) {
-	if(not open(ARGFILE,"<".$::opt_a)) {
+	if(not open(ARGFILE,"<",$::opt_a)) {
 	    print STDERR "$Global::progname: Cannot open input file `$::opt_a': No such file or directory\n";
 	    exit(-1);
 	}
@@ -1634,6 +1638,8 @@ sub processes_available_by_system_limit {
 }
 
 sub simultaneous_sshlogin {
+    # Using $sshlogin try to see if we can do $wanted_processes
+    # simultaneous logins
     my $sshlogin = shift;
     my $wanted_processes = shift;
     my ($sshcmd,$serverlogin) = sshcommand_of_sshlogin($sshlogin);
@@ -1693,14 +1699,14 @@ sub user_requested_processes {
 	if($opt_P =~ /^\+(\d+)$/) {
 	    # E.g. -P +2
 	    my $j = $1;
-	    $processes = $j + no_of_cpus_sshlogin($sshlogin);
+	    $processes = $j + no_of_processing_units_sshlogin($sshlogin);
 	} elsif ($opt_P =~ /^-(\d+)$/) {
 	    # E.g. -P -2
 	    my $j = $1;
-	    $processes = no_of_cpus_sshlogin($sshlogin) - $j;
+	    $processes = no_of_processing_units_sshlogin($sshlogin) - $j;
 	} elsif ($opt_P =~ /^(\d+)\%$/) {
 	    my $j = $1;
-	    $processes = no_of_cpus_sshlogin($sshlogin) * $j / 100;
+	    $processes = no_of_processing_units_sshlogin($sshlogin) * $j / 100;
 	} elsif ($opt_P =~ /^(\d+)$/) {
 	    $processes = $1;
 	    if($processes == 0) {
@@ -1717,21 +1723,26 @@ sub user_requested_processes {
     return $processes;
 }
 
-sub no_of_cores {
-    # TODO This should return number of cores and not the number of CPUs
-    return no_of_cpus(@_);
-}
-
-sub no_of_cpus_sshlogin {
-    # Number of CPUs at this sshlogin
+sub no_of_processing_units_sshlogin {
+    # Number of processing units (CPUs or cores) at this sshlogin
     my $sshlogin = shift;
     my ($sshcmd,$serverlogin) = sshcommand_of_sshlogin($sshlogin);
     if(not $Global::host{$sshlogin}{'ncpus'}) {
 	if($serverlogin eq ":") {
-	    $Global::host{$sshlogin}{'ncpus'} = no_of_cpus();
+	    if($::opt_use_cpus_instead_of_cores) {
+		$Global::host{$sshlogin}{'ncpus'} = no_of_cpus();
+	    } else {
+		$Global::host{$sshlogin}{'ncpus'} = no_of_cores();
+	    }
 	} else {
-	    my $ncpu = qx(echo|$sshcmd $serverlogin parallel --number-of-cpus);
-	    chomp($ncpu);
+	    my $ncpu;
+	    if($::opt_use_cpus_instead_of_cores) {
+		$ncpu = qx(echo|$sshcmd $serverlogin parallel --number-of-cpus);
+		chomp($ncpu);
+	    } else {
+		$ncpu = qx(echo|$sshcmd $serverlogin parallel --number-of-cores);
+		chomp($ncpu);
+	    }
 	    if($ncpu =~ /^[0-9]+$/) {
 		$Global::host{$sshlogin}{'ncpus'} = $ncpu;
 	    } else {
@@ -1746,37 +1757,90 @@ sub no_of_cpus_sshlogin {
 sub no_of_cpus {
     if(not $Global::no_of_cpus) {
 	local $/="\n"; # If delimiter is set, then $/ will be wrong
-	my $no_of_cpus = (no_of_cpus_gnu_linux()
+	my $no_of_cpus = (0
+			  || no_of_cpus_freebsd()
 			  || no_of_cpus_darwin()
-			  || no_of_cpus_solaris());
+			  || no_of_cpus_solaris()
+			  || no_of_cpus_gnu_linux()
+	    );
 	if($no_of_cpus) {
 	    $Global::no_of_cpus = $no_of_cpus;
 	} else {
-	    warn("Cannot figure out no of cpus. Using 1");
+	    warn("Cannot figure out number of cpus. Using 1");
 	    $Global::no_of_cpus = 1;
 	}
     }
     return $Global::no_of_cpus;
 }
 
+sub no_of_cores {
+    if(not $Global::no_of_cores) {
+	local $/="\n"; # If delimiter is set, then $/ will be wrong
+	my $no_of_cores = (0
+			   || no_of_cores_freebsd()
+			   || no_of_cores_darwin()
+			   || no_of_cores_solaris()
+			   || no_of_cores_gnu_linux()
+	    );
+	if($no_of_cores) {
+	    $Global::no_of_cores = $no_of_cores;
+	} else {
+	    warn("Cannot figure out number of CPU cores. Using 1");
+	    $Global::no_of_cores = 1;
+	}
+    }
+    return $Global::no_of_cores;
+}
+
 sub no_of_cpus_gnu_linux {
     my $no_of_cpus;
     if(-e "/proc/cpuinfo") {
 	$no_of_cpus = 0;
+	my %seen;
 	open(IN,"cat /proc/cpuinfo|") || return undef;
 	while(<IN>) {
-	    /^processor.*[:]/ and $no_of_cpus++;
+	    if(/^physical id.*[:](.*)/ and not $seen{$1}++) {
+		$no_of_cpus++;
+	    }
 	}
 	close IN;
     }
     return $no_of_cpus;
 }
 
+sub no_of_cores_gnu_linux {
+    my $no_of_cores;
+    if(-e "/proc/cpuinfo") {
+	$no_of_cores = 0;
+	open(IN,"cat /proc/cpuinfo|") || return undef;
+	while(<IN>) {
+	    /^processor.*[:]/ and $no_of_cores++;
+	}
+	close IN;
+    }
+    return $no_of_cores;
+}
+
 sub no_of_cpus_darwin {
-    my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`;
+    my $no_of_cpus = `sysctl -a hw 2>/dev/null | grep -w physicalcpu | awk '{ print \$2 }'`;
     return $no_of_cpus;
 }
 
+sub no_of_cores_darwin {
+    my $no_of_cores = `sysctl -a hw  2>/dev/null | grep -w logicalcpu | awk '{ print \$2 }'`;
+    return $no_of_cores;
+}
+
+sub no_of_cpus_freebsd {
+    my $no_of_cpus = `sysctl hw.ncpu 2>/dev/null | awk '{ print \$2 }'`;
+    return $no_of_cpus;
+}
+
+sub no_of_cores_freebsd {
+    my $no_of_cores = `sysctl -a hw  2>/dev/null | grep -w logicalcpu | awk '{ print \$2 }'`;
+    return $no_of_cores;
+}
+
 sub no_of_cpus_solaris {
     if(-x "/usr/sbin/psrinfo") {
 	my @psrinfo = `/usr/sbin/psrinfo`;
@@ -1793,6 +1857,22 @@ sub no_of_cpus_solaris {
     return undef;
 }
 
+sub no_of_cores_solaris {
+    if(-x "/usr/sbin/psrinfo") {
+	my @psrinfo = `/usr/sbin/psrinfo`;
+	if($#psrinfo >= 0) {
+	    return $#psrinfo +1;
+	}
+    }
+    if(-x "/usr/sbin/prtconf") {
+	my @prtconf = `/usr/sbin/prtconf | grep cpu..instance`;
+	if($#prtconf >= 0) {
+	    return $#prtconf +1;
+	}
+    }
+    return undef;
+}
+
 #
 # General useful library functions
 #
@@ -2364,6 +2444,3 @@ $main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder = 0;
 # No: space Will make it hard to do: 8/server1,server2
 # Maybe: / 8//usr/bin/myssh,//usr/bin/ssh
 # %/=:_^
-
-# Check transfer of newline file to sshlogin with own ssh and more than 9 simultaneously
-# TODO Unittest myssh: parallel --trc -j10000% -S "myssh -l tange nlv.pi.dk" echo
diff --git a/unittest/actual-results/test17 b/unittest/actual-results/test17
index 96e0eda9..955663cf 100644
--- a/unittest/actual-results/test17
+++ b/unittest/actual-results/test17
@@ -45,7 +45,6 @@ file19
 file20
 good if no file
 ls: cannot access tmp/parallel.file*: No such file or directory
-### --transfer - file with newline
 ### --transfer --cleanup - abspath
 file1
 file2
@@ -93,7 +92,6 @@ file20
 good if no file
 ls: cannot access tmp/parallel.file*: No such file or directory
 OK
-### --transfer --cleanup - file with newline
 ### --return - abspath
 /tmp/parallel.file10.out
 /tmp/parallel.file13.out
@@ -136,7 +134,6 @@ tmp/parallel.file9.out
 tmp/parallel.file>fire.out
 tmp/parallel.file : & ) \n*.jpg.out
 tmp/parallel.file/sub dir.out
-### --return - file with newline
 ### --return - multiple files
 tmp/parallel.file10.file.done
 tmp/parallel.file10.out
@@ -266,7 +263,6 @@ tmp/parallel.file/sub dir.out
 good if no file
 ls: cannot access tmp/parallel.file*: No such file or directory
 OK
-### --return --cleanup - file with newline
 ### --return --cleanup - multiple returns
 /tmp/parallel.file10.file.done
 /tmp/parallel.file10.out
@@ -399,7 +395,6 @@ OK
 good if no file
 ls: cannot access tmp/parallel.file*: No such file or directory
 OK
-### --transfer --return --cleanup - file with newline
 ### --transfer --return --cleanup - multiple files
 /tmp/parallel.file10.file.done
 /tmp/parallel.file10.out
@@ -531,7 +526,6 @@ tmp/parallel.file/sub dir.out
 good if no file
 ls: cannot access tmp/parallel.file*: No such file or directory
 OK
-### --trc - file with newline
 ### --trc - multiple files
 /tmp/parallel.file10.file.done
 /tmp/parallel.file10.out
diff --git a/unittest/actual-results/test19 b/unittest/actual-results/test19
new file mode 100644
index 00000000..608bafed
--- /dev/null
+++ b/unittest/actual-results/test19
@@ -0,0 +1,202 @@
+### Test --transfer --return --cleanup - files with newline
+### --transfer - file with newline
+newline
+newline
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+ls: cannot access tmp/parallel.file*: No such file or directory
+### --transfer --cleanup - file with newline
+newline
+newline
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+ls: cannot access tmp/parallel.file*: No such file or directory
+### --return - file with newline
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline2.out
+### --return --cleanup - file with newline
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline2.out
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+### --transfer --return --cleanup - file with newline
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline2.out
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+### --trc - file with newline
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline2.out
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+### --trc - multiple file with newline
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline1.out2
+tmp/parallel.file.
+newline2.out
+tmp/parallel.file.
+newline2.out2
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+### Test use special ssh
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline1.out2
+tmp/parallel.file.
+newline2.out
+tmp/parallel.file.
+newline2.out2
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+Input for ssh
+parallel-server1 rsync --server -lDrRze.iLsf . ./
+parallel-server1 cat tmp/parallel.file.'
+'newline2 > tmp/parallel.file.'
+'newline2.out;cat tmp/parallel.file.'
+'newline2 > tmp/parallel.file.'
+'newline2.out2
+parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
+'newline2.out
+parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
+'newline2.out2
+parallel-server1 rm -f tmp/parallel.file.'
+'newline2
+-l parallel parallel-server2 rsync --server -lDrRze.iLsf . ./
+parallel@parallel-server2 cat tmp/parallel.file.'
+'newline1 > tmp/parallel.file.'
+'newline1.out;cat tmp/parallel.file.'
+'newline1 > tmp/parallel.file.'
+'newline1.out2
+-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
+'newline1.out
+-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
+'newline1.out2
+parallel@parallel-server2 rm -f tmp/parallel.file.'
+'newline1
+### Test use special ssh with > 9 simultaneous
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
+100
diff --git a/unittest/actual-results/test20 b/unittest/actual-results/test20
new file mode 100644
index 00000000..e80c2ed8
--- /dev/null
+++ b/unittest/actual-results/test20
@@ -0,0 +1,8 @@
+### Test --number-of-cpus
+1
+### Test --number-of-cores
+2
+### Test --use-cpus-instead-of-cores
+Cores should complete first on machines with less than 4 physical CPUs
+cores done
+CPUs done
diff --git a/unittest/tests-to-run/test17.sh b/unittest/tests-to-run/test17.sh
index 2177e334..227f850b 100755
--- a/unittest/tests-to-run/test17.sh
+++ b/unittest/tests-to-run/test17.sh
@@ -8,6 +8,8 @@ SERVER2=parallel-server2
 echo '### Test --transfer --return --cleanup'
 
 rm -rf /tmp/parallel.file*
+stdout ssh $SERVER1 rm -rf 'tmp/parallel.file*'  '/tmp/parallel.file*' 
+stdout ssh parallel@$SERVER2 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' 
 (seq 1 3;echo '>fire';seq 5 10; echo ' : & ) \n*.jpg'; echo '/./sub dir'; seq 13 20) >/tmp/test17
 # Create some weirdly files in /tmp
 mkdir -p /tmp/parallel.file
@@ -21,7 +23,7 @@ stdout ssh parallel@$SERVER2 'rm -rf tmp/parallel.file*'
 cat /tmp/test17abs | $PAR -k --transfer --sshlogin $SERVER1,parallel@$SERVER2 cat {}";"rm {}
 # One of these should give the empty dir /tmp/parallel.file
 echo good if no file
-stdout ssh $SERVER1 ls '/tmp/parallel.file*' 
+stdout ssh $SERVER1 ls '/tmp/parallel.file*'
 # The other: No such file or directory
 stdout ssh parallel@$SERVER2 ls '/tmp/parallel.file*'
 
@@ -36,10 +38,6 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*'
 # Should give: No such file or directory
 stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*'
 
-echo '### --transfer - file with newline'
-## TODO
-
-
 echo '### --transfer --cleanup - abspath'
 cat /tmp/test17abs | $PAR -k --transfer --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {}
 echo good if no file
@@ -56,9 +54,6 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
 stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
 
-echo '### --transfer --cleanup - file with newline'
-# TODO
-
 echo '### --return - abspath'
 rm -rf /tmp/parallel.file*out
 cat /tmp/test17abs | $PAR -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 echo {} ">"{.}.out
@@ -69,9 +64,6 @@ rm -rf /tmp/parallel.file*out
 cat /tmp/test17rel | $PAR -k --return {.}.out --sshlogin $SERVER1,parallel@$SERVER2 mkdir -p tmp ';'echo {} ">"{.}.out
 ls tmp/parallel.file*out tmp/parallel.file/*out
 
-echo '### --return - file with newline'
-# TODO
-
 echo '### --return - multiple files'
 rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
 cat /tmp/test17rel | $PAR -k --return {.}.out --return {}.done \
@@ -98,10 +90,6 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
 stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
 
-echo '### --return --cleanup - file with newline'
-#
-# TODO
-
 echo '### --return --cleanup - multiple returns'
 rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
 cat /tmp/test17rel | $PAR -k --return {.}.out --return {}.done --cleanup \
@@ -133,9 +121,6 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
 stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
 
-echo '### --transfer --return --cleanup - file with newline'
-# TODO
-
 echo '### --transfer --return --cleanup - multiple files'
 rm -rf tmp/parallel.file*out tmp/parallel.file/*out tmp/parallel.file*done tmp/parallel.file/*done
 cat /tmp/test17rel | $PAR -k --transfer --return {.}.out --return {}.done --cleanup \
@@ -165,9 +150,6 @@ stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
 # Should give: No such file or directory
 stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
 
-echo '### --trc - file with newline'
-# TODO
-
 echo '### --trc - multiple files'
 rm -rf /tmp/parallel.file*out /tmp/parallel.file/*out /tmp/parallel.file*done /tmp/parallel.file/*done
 cat /tmp/test17abs | $PAR -k --trc {.}.out --trc {}.done \
diff --git a/unittest/tests-to-run/test19.sh b/unittest/tests-to-run/test19.sh
new file mode 100644
index 00000000..a845e859
--- /dev/null
+++ b/unittest/tests-to-run/test19.sh
@@ -0,0 +1,132 @@
+#!/bin/bash
+
+# TODO return multiple
+
+PAR=parallel
+
+SERVER1=parallel-server1
+SERVER2=parallel-server2
+
+echo '### Test --transfer --return --cleanup - files with newline'
+
+rm -rf /tmp/parallel.file*
+stdout ssh $SERVER1 rm -rf 'tmp/parallel.file*'  '/tmp/parallel.file*' 
+stdout ssh parallel@$SERVER2 rm -rf 'tmp/parallel.file*' '/tmp/parallel.file*' 
+
+cd /
+echo '### --transfer - file with newline'
+echo newline > '/tmp/parallel.file.
+newline1'
+echo newline > '/tmp/parallel.file.
+newline2'
+find tmp/parallel*newline* -print0 | $PAR -0 -k --transfer --sshlogin $SERVER1,parallel@$SERVER2 cat {}";"rm {}
+# Should give: No such file or directory
+echo good if no file
+stdout ssh $SERVER1 ls 'tmp/parallel.file*' 
+# Should give: No such file or directory
+stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*'
+
+echo '### --transfer --cleanup - file with newline'
+echo newline > '/tmp/parallel.file.
+newline1'
+echo newline > '/tmp/parallel.file.
+newline2'
+find tmp/parallel*newline* -print0 | $PAR -0 -k --transfer --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {}
+# Should give: No such file or directory
+echo good if no file
+stdout ssh $SERVER1 ls 'tmp/parallel.file*' 
+# Should give: No such file or directory
+stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*'
+
+echo '### --return - file with newline'
+echo newline > '/tmp/parallel.file.
+newline1'
+echo newline > '/tmp/parallel.file.
+newline2'
+find tmp/parallel*newline* -print0 | $PAR -0 -k --return {}.out --sshlogin $SERVER1,parallel@$SERVER2 echo remote '>' {}.out
+ls tmp/parallel*newline*out
+rm tmp/parallel*newline*out
+# Cleanup remote
+stdout ssh $SERVER1 rm -rf 'tmp/parallel.file*' 
+stdout ssh parallel@$SERVER2 rm -rf 'tmp/parallel.file*'
+
+echo '### --return --cleanup - file with newline'
+echo newline > '/tmp/parallel.file.
+newline1'
+echo newline > '/tmp/parallel.file.
+newline2'
+find tmp/parallel*newline* -print0 | $PAR -0 -k --return {}.out --cleanup --sshlogin $SERVER1,parallel@$SERVER2 echo remote '>' {}.out
+ls tmp/parallel*newline*out
+rm tmp/parallel*newline*out
+echo good if no file
+stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
+# Should give: No such file or directory
+stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
+
+echo '### --transfer --return --cleanup - file with newline'
+echo newline > '/tmp/parallel.file.
+newline1'
+echo newline > '/tmp/parallel.file.
+newline2'
+find tmp/parallel*newline* -print0 | $PAR -0 -k --transfer --return {}.out --cleanup --sshlogin $SERVER1,parallel@$SERVER2 cat {} '>' {}.out
+ls tmp/parallel*newline*out
+rm tmp/parallel*newline*out
+echo good if no file
+stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
+# Should give: No such file or directory
+stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
+
+echo '### --trc - file with newline'
+echo newline > '/tmp/parallel.file.
+newline1'
+echo newline > '/tmp/parallel.file.
+newline2'
+find tmp/parallel*newline* -print0 | $PAR -0 -k --trc {}.out --sshlogin $SERVER1,parallel@$SERVER2 cat {} '>' {}.out
+ls tmp/parallel*newline*out
+rm tmp/parallel*newline*out
+echo good if no file
+stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
+# Should give: No such file or directory
+stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
+
+echo '### --trc - multiple file with newline'
+echo newline > '/tmp/parallel.file.
+newline1'
+echo newline > '/tmp/parallel.file.
+newline2'
+find tmp/parallel*newline* -print0 | $PAR -0 -k --trc {}.out --trc {}.out2 --sshlogin $SERVER1,parallel@$SERVER2 cat {} '>' {}.out';'cat {} '>' {}.out2
+ls tmp/parallel*newline*out*
+rm tmp/parallel*newline*out*
+echo good if no file
+stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
+# Should give: No such file or directory
+stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
+
+echo '### Test use special ssh'
+echo 'ssh "$@"; echo "$@" >>/tmp/myssh1-run' >/tmp/myssh1
+echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2
+chmod 755 /tmp/myssh1 /tmp/myssh2
+rm -rf /tmp/myssh1-run /tmp/myssh2-run
+echo newline > '/tmp/parallel.file.
+newline1'
+echo newline > '/tmp/parallel.file.
+newline2'
+find tmp/parallel*newline* -print0 | $PAR -0 -k -j1 --trc {}.out --trc {}.out2 \
+  --sshlogin "/tmp/myssh1 $SERVER1, /tmp/myssh2 parallel@$SERVER2" \
+  cat {} '>' {}.out';'cat {} '>' {}.out2
+ls tmp/parallel*newline*out*
+rm tmp/parallel*newline*out*
+echo good if no file
+stdout ssh $SERVER1 ls 'tmp/parallel.file*' || echo OK
+# Should give: No such file or directory
+stdout ssh parallel@$SERVER2 ls 'tmp/parallel.file*' || echo OK
+echo Input for ssh
+cat /tmp/myssh1-run /tmp/myssh2-run
+rm /tmp/myssh1-run /tmp/myssh2-run
+
+echo '### Test use special ssh with > 9 simultaneous'
+echo 'ssh "$@"; echo "$@" >>/tmp/myssh1-run' >/tmp/myssh1
+echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2
+chmod 755 /tmp/myssh1 /tmp/myssh2
+seq 1 100 | $PAR --sshlogin "/tmp/myssh1 $SERVER1, /tmp/myssh2 parallel@$SERVER2" \
+  -j10000% -k echo
diff --git a/unittest/tests-to-run/test20.sh b/unittest/tests-to-run/test20.sh
new file mode 100644
index 00000000..bdd7f8e8
--- /dev/null
+++ b/unittest/tests-to-run/test20.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+PAR=parallel
+
+echo '### Test --number-of-cpus'
+$PAR --number-of-cpus
+
+echo '### Test --number-of-cores'
+$PAR --number-of-cores
+
+echo '### Test --use-cpus-instead-of-cores'
+(seq 1 4 | $PAR --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done &
+(seq 1 4 | $PAR -j100% sleep) && echo cores done &
+echo 'Cores should complete first on machines with less than 4 physical CPUs'
+wait
+
+
diff --git a/unittest/wanted-results/test17 b/unittest/wanted-results/test17
index 1a34189c..955663cf 100644
--- a/unittest/wanted-results/test17
+++ b/unittest/wanted-results/test17
@@ -45,7 +45,6 @@ file19
 file20
 good if no file
 ls: cannot access tmp/parallel.file*: No such file or directory
-### --transfer - file with newline
 ### --transfer --cleanup - abspath
 file1
 file2
@@ -91,9 +90,8 @@ file18
 file19
 file20
 good if no file
-ls: cannot access /tmp/parallel.file*: No such file or directory
+ls: cannot access tmp/parallel.file*: No such file or directory
 OK
-### --transfer --cleanup - file with newline
 ### --return - abspath
 /tmp/parallel.file10.out
 /tmp/parallel.file13.out
@@ -116,68 +114,67 @@ OK
 /tmp/parallel.file : & ) \n*.jpg.out
 /tmp/parallel.file/sub dir.out
 ### --return - relpath
-/tmp/parallel.file10.out
-/tmp/parallel.file13.out
-/tmp/parallel.file14.out
-/tmp/parallel.file15.out
-/tmp/parallel.file16.out
-/tmp/parallel.file17.out
-/tmp/parallel.file18.out
-/tmp/parallel.file19.out
-/tmp/parallel.file1.out
-/tmp/parallel.file20.out
-/tmp/parallel.file2.out
-/tmp/parallel.file3.out
-/tmp/parallel.file5.out
-/tmp/parallel.file6.out
-/tmp/parallel.file7.out
-/tmp/parallel.file8.out
-/tmp/parallel.file9.out
-/tmp/parallel.file>fire.out
-/tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.out
-### --return - file with newline
+tmp/parallel.file10.out
+tmp/parallel.file13.out
+tmp/parallel.file14.out
+tmp/parallel.file15.out
+tmp/parallel.file16.out
+tmp/parallel.file17.out
+tmp/parallel.file18.out
+tmp/parallel.file19.out
+tmp/parallel.file1.out
+tmp/parallel.file20.out
+tmp/parallel.file2.out
+tmp/parallel.file3.out
+tmp/parallel.file5.out
+tmp/parallel.file6.out
+tmp/parallel.file7.out
+tmp/parallel.file8.out
+tmp/parallel.file9.out
+tmp/parallel.file>fire.out
+tmp/parallel.file : & ) \n*.jpg.out
+tmp/parallel.file/sub dir.out
 ### --return - multiple files
-/tmp/parallel.file10.file.done
-/tmp/parallel.file10.out
-/tmp/parallel.file13.file.done
-/tmp/parallel.file13.out
-/tmp/parallel.file14.file.done
-/tmp/parallel.file14.out
-/tmp/parallel.file15.file.done
-/tmp/parallel.file15.out
-/tmp/parallel.file16.file.done
-/tmp/parallel.file16.out
-/tmp/parallel.file17.file.done
-/tmp/parallel.file17.out
-/tmp/parallel.file18.file.done
-/tmp/parallel.file18.out
-/tmp/parallel.file19.file.done
-/tmp/parallel.file19.out
-/tmp/parallel.file1.file.done
-/tmp/parallel.file1.out
-/tmp/parallel.file20.file.done
-/tmp/parallel.file20.out
-/tmp/parallel.file2.file.done
-/tmp/parallel.file2.out
-/tmp/parallel.file3.file.done
-/tmp/parallel.file3.out
-/tmp/parallel.file5.file.done
-/tmp/parallel.file5.out
-/tmp/parallel.file6.file.done
-/tmp/parallel.file6.out
-/tmp/parallel.file7.file.done
-/tmp/parallel.file7.out
-/tmp/parallel.file8.file.done
-/tmp/parallel.file8.out
-/tmp/parallel.file9.file.done
-/tmp/parallel.file9.out
-/tmp/parallel.file>fire.file.done
-/tmp/parallel.file>fire.out
-/tmp/parallel.file : & ) \n*.jpg.file.done
-/tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.file.done
-/tmp/parallel.file/sub dir.out
+tmp/parallel.file10.file.done
+tmp/parallel.file10.out
+tmp/parallel.file13.file.done
+tmp/parallel.file13.out
+tmp/parallel.file14.file.done
+tmp/parallel.file14.out
+tmp/parallel.file15.file.done
+tmp/parallel.file15.out
+tmp/parallel.file16.file.done
+tmp/parallel.file16.out
+tmp/parallel.file17.file.done
+tmp/parallel.file17.out
+tmp/parallel.file18.file.done
+tmp/parallel.file18.out
+tmp/parallel.file19.file.done
+tmp/parallel.file19.out
+tmp/parallel.file1.file.done
+tmp/parallel.file1.out
+tmp/parallel.file20.file.done
+tmp/parallel.file20.out
+tmp/parallel.file2.file.done
+tmp/parallel.file2.out
+tmp/parallel.file3.file.done
+tmp/parallel.file3.out
+tmp/parallel.file5.file.done
+tmp/parallel.file5.out
+tmp/parallel.file6.file.done
+tmp/parallel.file6.out
+tmp/parallel.file7.file.done
+tmp/parallel.file7.out
+tmp/parallel.file8.file.done
+tmp/parallel.file8.out
+tmp/parallel.file9.file.done
+tmp/parallel.file9.out
+tmp/parallel.file>fire.file.done
+tmp/parallel.file>fire.out
+tmp/parallel.file : & ) \n*.jpg.file.done
+tmp/parallel.file : & ) \n*.jpg.out
+tmp/parallel.file/sub dir.file.done
+tmp/parallel.file/sub dir.out
 ### --return --cleanup - abspath
 /tmp/parallel.file10.file.done
 /tmp/parallel.file10.out
@@ -223,50 +220,49 @@ good if no file
 ls: cannot access /tmp/parallel.file*: No such file or directory
 OK
 ### --return --cleanup - relpath
-/tmp/parallel.file10.file.done
-/tmp/parallel.file10.out
-/tmp/parallel.file13.file.done
-/tmp/parallel.file13.out
-/tmp/parallel.file14.file.done
-/tmp/parallel.file14.out
-/tmp/parallel.file15.file.done
-/tmp/parallel.file15.out
-/tmp/parallel.file16.file.done
-/tmp/parallel.file16.out
-/tmp/parallel.file17.file.done
-/tmp/parallel.file17.out
-/tmp/parallel.file18.file.done
-/tmp/parallel.file18.out
-/tmp/parallel.file19.file.done
-/tmp/parallel.file19.out
-/tmp/parallel.file1.file.done
-/tmp/parallel.file1.out
-/tmp/parallel.file20.file.done
-/tmp/parallel.file20.out
-/tmp/parallel.file2.file.done
-/tmp/parallel.file2.out
-/tmp/parallel.file3.file.done
-/tmp/parallel.file3.out
-/tmp/parallel.file5.file.done
-/tmp/parallel.file5.out
-/tmp/parallel.file6.file.done
-/tmp/parallel.file6.out
-/tmp/parallel.file7.file.done
-/tmp/parallel.file7.out
-/tmp/parallel.file8.file.done
-/tmp/parallel.file8.out
-/tmp/parallel.file9.file.done
-/tmp/parallel.file9.out
-/tmp/parallel.file>fire.file.done
-/tmp/parallel.file>fire.out
-/tmp/parallel.file : & ) \n*.jpg.file.done
-/tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.file.done
-/tmp/parallel.file/sub dir.out
+tmp/parallel.file10.file.done
+tmp/parallel.file10.out
+tmp/parallel.file13.file.done
+tmp/parallel.file13.out
+tmp/parallel.file14.file.done
+tmp/parallel.file14.out
+tmp/parallel.file15.file.done
+tmp/parallel.file15.out
+tmp/parallel.file16.file.done
+tmp/parallel.file16.out
+tmp/parallel.file17.file.done
+tmp/parallel.file17.out
+tmp/parallel.file18.file.done
+tmp/parallel.file18.out
+tmp/parallel.file19.file.done
+tmp/parallel.file19.out
+tmp/parallel.file1.file.done
+tmp/parallel.file1.out
+tmp/parallel.file20.file.done
+tmp/parallel.file20.out
+tmp/parallel.file2.file.done
+tmp/parallel.file2.out
+tmp/parallel.file3.file.done
+tmp/parallel.file3.out
+tmp/parallel.file5.file.done
+tmp/parallel.file5.out
+tmp/parallel.file6.file.done
+tmp/parallel.file6.out
+tmp/parallel.file7.file.done
+tmp/parallel.file7.out
+tmp/parallel.file8.file.done
+tmp/parallel.file8.out
+tmp/parallel.file9.file.done
+tmp/parallel.file9.out
+tmp/parallel.file>fire.file.done
+tmp/parallel.file>fire.out
+tmp/parallel.file : & ) \n*.jpg.file.done
+tmp/parallel.file : & ) \n*.jpg.out
+tmp/parallel.file/sub dir.file.done
+tmp/parallel.file/sub dir.out
 good if no file
-ls: cannot access /tmp/parallel.file*: No such file or directory
+ls: cannot access tmp/parallel.file*: No such file or directory
 OK
-### --return --cleanup - file with newline
 ### --return --cleanup - multiple returns
 /tmp/parallel.file10.file.done
 /tmp/parallel.file10.out
@@ -309,7 +305,7 @@ OK
 /tmp/parallel.file/sub dir.file.done
 /tmp/parallel.file/sub dir.out
 good if no file
-ls: cannot access /tmp/parallel.file*: No such file or directory
+ls: cannot access tmp/parallel.file*: No such file or directory
 OK
 ### --transfer --return --cleanup - abspath
 /tmp/parallel.file10.file.done
@@ -397,9 +393,8 @@ OK
 /tmp/parallel.file/sub dir.file.done
 /tmp/parallel.file/sub dir.out
 good if no file
-ls: cannot access /tmp/parallel.file*: No such file or directory
+ls: cannot access tmp/parallel.file*: No such file or directory
 OK
-### --transfer --return --cleanup - file with newline
 ### --transfer --return --cleanup - multiple files
 /tmp/parallel.file10.file.done
 /tmp/parallel.file10.out
@@ -441,7 +436,7 @@ OK
 /tmp/parallel.file : & ) \n*.jpg.out
 /tmp/parallel.file/sub dir.file.done
 /tmp/parallel.file/sub dir.out
-ls: cannot access /tmp/parallel.file*: No such file or directory
+ls: cannot access tmp/parallel.file*: No such file or directory
 OK
 ### --trc - abspath
 /tmp/parallel.file10.file.done
@@ -488,50 +483,49 @@ good if no file
 ls: cannot access /tmp/parallel.file*: No such file or directory
 OK
 ### --trc - relpath
-/tmp/parallel.file10.file.done
-/tmp/parallel.file10.out
-/tmp/parallel.file13.file.done
-/tmp/parallel.file13.out
-/tmp/parallel.file14.file.done
-/tmp/parallel.file14.out
-/tmp/parallel.file15.file.done
-/tmp/parallel.file15.out
-/tmp/parallel.file16.file.done
-/tmp/parallel.file16.out
-/tmp/parallel.file17.file.done
-/tmp/parallel.file17.out
-/tmp/parallel.file18.file.done
-/tmp/parallel.file18.out
-/tmp/parallel.file19.file.done
-/tmp/parallel.file19.out
-/tmp/parallel.file1.file.done
-/tmp/parallel.file1.out
-/tmp/parallel.file20.file.done
-/tmp/parallel.file20.out
-/tmp/parallel.file2.file.done
-/tmp/parallel.file2.out
-/tmp/parallel.file3.file.done
-/tmp/parallel.file3.out
-/tmp/parallel.file5.file.done
-/tmp/parallel.file5.out
-/tmp/parallel.file6.file.done
-/tmp/parallel.file6.out
-/tmp/parallel.file7.file.done
-/tmp/parallel.file7.out
-/tmp/parallel.file8.file.done
-/tmp/parallel.file8.out
-/tmp/parallel.file9.file.done
-/tmp/parallel.file9.out
-/tmp/parallel.file>fire.file.done
-/tmp/parallel.file>fire.out
-/tmp/parallel.file : & ) \n*.jpg.file.done
-/tmp/parallel.file : & ) \n*.jpg.out
-/tmp/parallel.file/sub dir.file.done
-/tmp/parallel.file/sub dir.out
+tmp/parallel.file10.file.done
+tmp/parallel.file10.out
+tmp/parallel.file13.file.done
+tmp/parallel.file13.out
+tmp/parallel.file14.file.done
+tmp/parallel.file14.out
+tmp/parallel.file15.file.done
+tmp/parallel.file15.out
+tmp/parallel.file16.file.done
+tmp/parallel.file16.out
+tmp/parallel.file17.file.done
+tmp/parallel.file17.out
+tmp/parallel.file18.file.done
+tmp/parallel.file18.out
+tmp/parallel.file19.file.done
+tmp/parallel.file19.out
+tmp/parallel.file1.file.done
+tmp/parallel.file1.out
+tmp/parallel.file20.file.done
+tmp/parallel.file20.out
+tmp/parallel.file2.file.done
+tmp/parallel.file2.out
+tmp/parallel.file3.file.done
+tmp/parallel.file3.out
+tmp/parallel.file5.file.done
+tmp/parallel.file5.out
+tmp/parallel.file6.file.done
+tmp/parallel.file6.out
+tmp/parallel.file7.file.done
+tmp/parallel.file7.out
+tmp/parallel.file8.file.done
+tmp/parallel.file8.out
+tmp/parallel.file9.file.done
+tmp/parallel.file9.out
+tmp/parallel.file>fire.file.done
+tmp/parallel.file>fire.out
+tmp/parallel.file : & ) \n*.jpg.file.done
+tmp/parallel.file : & ) \n*.jpg.out
+tmp/parallel.file/sub dir.file.done
+tmp/parallel.file/sub dir.out
 good if no file
-ls: cannot access /tmp/parallel.file*: No such file or directory
+ls: cannot access tmp/parallel.file*: No such file or directory
 OK
-### --trc - file with newline
 ### --trc - multiple files
 /tmp/parallel.file10.file.done
 /tmp/parallel.file10.out
diff --git a/unittest/wanted-results/test19 b/unittest/wanted-results/test19
new file mode 100644
index 00000000..608bafed
--- /dev/null
+++ b/unittest/wanted-results/test19
@@ -0,0 +1,202 @@
+### Test --transfer --return --cleanup - files with newline
+### --transfer - file with newline
+newline
+newline
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+ls: cannot access tmp/parallel.file*: No such file or directory
+### --transfer --cleanup - file with newline
+newline
+newline
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+ls: cannot access tmp/parallel.file*: No such file or directory
+### --return - file with newline
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline2.out
+### --return --cleanup - file with newline
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline2.out
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+### --transfer --return --cleanup - file with newline
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline2.out
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+### --trc - file with newline
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline2.out
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+### --trc - multiple file with newline
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline1.out2
+tmp/parallel.file.
+newline2.out
+tmp/parallel.file.
+newline2.out2
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+### Test use special ssh
+tmp/parallel.file.
+newline1.out
+tmp/parallel.file.
+newline1.out2
+tmp/parallel.file.
+newline2.out
+tmp/parallel.file.
+newline2.out2
+good if no file
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+ls: cannot access tmp/parallel.file*: No such file or directory
+OK
+Input for ssh
+parallel-server1 rsync --server -lDrRze.iLsf . ./
+parallel-server1 cat tmp/parallel.file.'
+'newline2 > tmp/parallel.file.'
+'newline2.out;cat tmp/parallel.file.'
+'newline2 > tmp/parallel.file.'
+'newline2.out2
+parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
+'newline2.out
+parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
+'newline2.out2
+parallel-server1 rm -f tmp/parallel.file.'
+'newline2
+-l parallel parallel-server2 rsync --server -lDrRze.iLsf . ./
+parallel@parallel-server2 cat tmp/parallel.file.'
+'newline1 > tmp/parallel.file.'
+'newline1.out;cat tmp/parallel.file.'
+'newline1 > tmp/parallel.file.'
+'newline1.out2
+-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
+'newline1.out
+-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
+'newline1.out2
+parallel@parallel-server2 rm -f tmp/parallel.file.'
+'newline1
+### Test use special ssh with > 9 simultaneous
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
+100
diff --git a/unittest/wanted-results/test20 b/unittest/wanted-results/test20
new file mode 100644
index 00000000..e80c2ed8
--- /dev/null
+++ b/unittest/wanted-results/test20
@@ -0,0 +1,8 @@
+### Test --number-of-cpus
+1
+### Test --number-of-cores
+2
+### Test --use-cpus-instead-of-cores
+Cores should complete first on machines with less than 4 physical CPUs
+cores done
+CPUs done