Passes testsuite.

parallel: base64 encoding compatible with rc.

src/parallel

@@ -1183,7 +1183,7 @@ sub check_invalid_option_combinations {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20160323;
+    $Global::version = 20160414;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -3735,9 +3735,10 @@ sub __GENERIC_COMMON_FUNCTION__ {}
 sub mkdir_or_die {
     # If dir is not executable: die
     my $dir = shift;
-    File::Path::mkpath($dir);
+    # The eval is needed to catch exception from mkdir
+    eval { File::Path::mkpath($dir); };
     if(not -x $dir) {
-	::error("Cannot change into non-executable $dir: $!");
+	::error("Cannot change into non-executable dir $dir: $!");
 	::wait_and_exit(255);
     }
 }
@@ -6855,11 +6856,8 @@ sub wrapped {
 	    # $command = "perl -e '".base64_zip_eval()."' ".
 	    # join" ",string_zip_base64(
 	    #         'exec "'.::perl_quote_scalar($command).'"');
-	    $command = "perl -e ".
+	    $command = base64_wrap("exec \"$Global::shell\",'-c',\"".
-		    ::shell_quote_scalar(base64_eval())." ".
+				   ::perl_quote_scalar($command).'"');
-		    join(" ", ::shell_quote(
-			     string_base64("exec \"$Global::shell\",'-c',\"".
-					   ::perl_quote_scalar($command).'"')));
 	}
 	$self->{'wrapped'} = $command;
     }
@@ -6958,6 +6956,21 @@ sub base64_zip_eval {
     return $script;
 }
 
+sub base64_wrap {
+    # base64 encode Perl code
+    # Split it into chunks of < 1000 bytes
+    # Prepend it with a decoder that eval's it
+    # Input:
+    #   $eval_string = Perl code to run
+    # Returns:
+    #   $shell_command = shell command that runs $eval_string
+    my $eval_string = shift;
+    return
+	"perl -e ".
+	::shell_quote_scalar(base64_eval())." ".
+	join" ",::shell_quote(string_base64($eval_string));
+}
+
 sub base64_eval {
     # Script that:
     #   * reads base64 strings from @ARGV
@@ -7081,11 +7094,14 @@ sub sshlogin_wrap {
 	my @non_functions = grep { substr($ENV{$_},0,4) ne "() {" } @vars;
 	# eval of @envset will set %ENV
 	my $envset = join"", map {
-	    '$ENV{"'.::perl_quote_scalar($_).'"}="'. ::perl_quote_scalar($ENV{$_}).'";'; } @non_functions;
+	    '$ENV{"'.::perl_quote_scalar($_).'"}="'.
+		::perl_quote_scalar($ENV{$_}).'";'; } @non_functions;
 
 	# running @bashfunc on the command line, will set the functions
 	my @bashfunc = map {
-	    my $v=$_; s/BASH_FUNC_(.*)(\(\)|%%)/$1/; "$_$ENV{$v};export -f $_ >/dev/null;" } @bash_functions;
+	    my $v=$_;
+	    s/BASH_FUNC_(.*)(\(\)|%%)/$1/;
+	    "$_$ENV{$v};export -f $_ >/dev/null;" } @bash_functions;
 	# eval $bashfuncset will set $bashfunc
 	my $bashfuncset;
 	if(@bashfunc) {
@@ -7135,25 +7151,19 @@ sub sshlogin_wrap {
 	    if(@opt::env) {
 		# Prepend with environment setter, which sets functions in zsh
 		my ($csh_friendly,$envset,$bashfuncset) = env_as_eval();
-		my $env_command = $envset.$bashfuncset.
+		my $perl_code = $envset.$bashfuncset.
 		    '@ARGV="'.::perl_quote_scalar($command).'";'.
 		    "exec\"$Global::shell\",\"-c\",\(\$bashfunc.\"\@ARGV\"\)\;die\"exec:\$\!\\n\"\;";
-		if(length $env_command > 999
+		if(length $perl_code > 999
 		   or
 		   not $csh_friendly
 		   or
 		   $command =~ /\n/) {
 		    # csh does not deal well with > 1000 chars in one word
 		    # csh does not deal well with $ENV with \n
-		    # bzip2 breaks --sql mysql://...
+		    $self->{'sshlogin_wrap'} = base64_wrap($perl_code);
-		    # $env_command = "perl -e '".base64_zip_eval()."' ".
-		    # join" ",string_zip_base64($env_command);
-		    $env_command = "perl -e ".
-			::shell_quote_scalar(base64_eval())." ".
-			join" ",::shell_quote(string_base64($env_command));
-		    $self->{'sshlogin_wrap'} = $env_command;
 		} else {
-		    $self->{'sshlogin_wrap'} = "perl -e ".::shell_quote_scalar($env_command);
+		    $self->{'sshlogin_wrap'} = "perl -e ".::shell_quote_scalar($perl_code);
 		}
 	    } else {
 		$self->{'sshlogin_wrap'} = $command;
@@ -7181,9 +7191,10 @@ sub sshlogin_wrap {
 	       $command =~ /\n/) {
 		# csh does not deal well with > 1000 chars in one word
 		# csh does not deal well with $ENV with \n
-		$quoted_remote_command = "perl -e ".
+		$quoted_remote_command =
+		    "perl -e ".
 		    ::shell_quote_scalar(::shell_quote_scalar(base64_eval()))." ".
-		    join" ",::shell_quote(string_base64($remote_command));
+		    join" ",::shell_quote(::shell_quote(string_base64($remote_command)));
 	    } else {
 		$quoted_remote_command = $dq_remote_command;
 	    }
@@ -7668,6 +7679,7 @@ sub print_dryrun_and_verbose {
 	# The line to run contains a 118 chars extra code + the title 2x
 	my $l_tot = 2 * $l_tit + $l_act + $l_fifo;
 
+	my $quoted_space75 = ::shell_quote_scalar(" ")x75;
 	while($l_tit < 1000 and
 	      (
 	       (890 < $l_tot and $l_tot < 1350)
@@ -7681,7 +7693,7 @@ sub print_dryrun_and_verbose {
 	    # The measured lengths are:
 	    # 996 < (title + whole command) < 1127
 	    # 9331 < (title + whole command) < 9636
-	    $title = $title.('\ 'x75);
+	    $title .= $quoted_space75;
 	    $l_tit = length($title);
 	    $l_tot = 2 * $l_tit + $l_act + $l_fifo;
 	}

src/parallel_design.pod

@@ -671,7 +671,7 @@ special chars and ' is used for newline. Whether a char is special
 depends on the shell and the context. Luckily quoting a bit too many
 chars does not break things.
 
-It is fast, but had the distinct disadvantage that if a string needs
+It is fast, but has the distinct disadvantage that if a string needs
 to be quoted multiple times, the \'s double every time - increasing
 the string length exponentially.
 
@@ -748,11 +748,10 @@ are no visible files on it.
 
 GNU B<parallel> buffers on disk. If the disk is full data may be
 lost. To check if the disk is full GNU B<parallel> writes a 8193 byte
-file when a job finishes. If this file is written successfully, it is
+file every second. If this file is written successfully, it is removed
-removed immediately. If it is not written successfully, the disk is
+immediately. If it is not written successfully, the disk is full. The
-full. The size 8193 was chosen because 8192 gave wrong result on some
+size 8193 was chosen because 8192 gave wrong result on some file
-file systems, whereas 8193 did the correct thing on all tested
+systems, whereas 8193 did the correct thing on all tested filesystems.
-filesystems.
 
 
 =head2 Perl replacement strings, {= =}, and --rpl
@@ -899,12 +898,6 @@ Open3 is slow. Printing is slow. It would be good if they did not tie
 up ressources, but were run in separate threads.
 
 
-=head2 Transferring of variables and functions from zsh
-
-Transferring Bash functions to remote zsh works.
-Can parallel_bash_environment be used to import zsh functions?
-
-
 =head2 --rrs on remote using a perl wrapper
 
 ... | perl -pe '$/=$recend$recstart;BEGIN{ if(substr($_) eq $recstart) substr($_)="" } eof and substr($_) eq $recend) substr($_)="" 

testsuite/tests-to-run/parallel-local-sql.sh

@@ -31,44 +31,44 @@ echo '### --sqlandworker mysql'
   (sleep 2; parallel --sqlworker $MYSQLTBL sleep .3\;echo >$T1) &
   parallel --sqlandworker $MYSQLTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; 
   true sort -u $T1 $T2; 
-  sql $MYSQL 'select * from parsql order by seq;'
+  sleep 1; sql $MYSQL 'select * from parsql order by seq;'
 
 echo '### --sqlandworker postgresql'
   (sleep 2; parallel --sqlworker $PGTBL sleep .3\;echo >$T3) &
   parallel --sqlandworker $PGTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T4; 
   true sort -u $T3 $T4; 
-  sql $PG 'select * from parsql order by seq;'
+  sleep 1; sql $PG 'select * from parsql order by seq;'
 
 echo '### --sqlandworker sqlite'
   (sleep 2; parallel --sqlworker $SQLITETBL sleep .3\;echo >$T5) &
   parallel --sqlandworker $SQLITETBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T6; 
   true sort -u $T5 $T6; 
-  sql $SQLITE 'select * from parsql order by seq;'
+  sleep 1; sql $SQLITE 'select * from parsql order by seq;'
 
 echo '### --sqlandworker postgresql -S lo'
   (sleep 2; parallel -S lo --sqlworker $PGTBL2 sleep .3\;echo >$T7) &
   parallel -S lo --sqlandworker $PGTBL2 sleep .3\;echo ::: {1..5} ::: {a..e} >$T8; 
   true sort -u $T7 $T8; 
-  sql $PG 'select * from parsql2 order by seq;'
+  sleep 1; sql $PG 'select * from parsql2 order by seq;'
 
 echo '### --sqlandworker postgresql --results'
   mkdir -p /tmp/out--sql
   (sleep 2; parallel --results /tmp/out--sql --sqlworker $PGTBL3 sleep .3\;echo >$T9) &
   parallel --results /tmp/out--sql --sqlandworker $PGTBL3 sleep .3\;echo ::: {1..5} ::: {a..e} >$T10; 
   true sort -u $T9 $T10; 
-  sql $PG 'select * from parsql3 order by seq;'
+  sleep 1; sql $PG 'select * from parsql3 order by seq;'
 
 echo '### --sqlandworker postgresql --linebuffer'
   (sleep 2; parallel --linebuffer --sqlworker $PGTBL4 sleep .3\;echo >$T11) &
   parallel --linebuffer --sqlandworker $PGTBL4 sleep .3\;echo ::: {1..5} ::: {a..e} >$T12; 
   true sort -u $T11 $T12; 
-  sql $PG 'select * from parsql4 order by seq;'
+  sleep 1; sql $PG 'select * from parsql4 order by seq;'
 
 echo '### --sqlandworker postgresql -u'
   (sleep 2; parallel -u --sqlworker $PGTBL5 sleep .3\;echo >$T13) &
   parallel -u --sqlandworker $PGTBL5 sleep .3\;echo ::: {1..5} ::: {a..e} >$T14; 
   true sort -u $T13 $T14; 
-  sql $PG 'select * from parsql5 order by seq;'
+  sleep 1; sql $PG 'select * from parsql5 order by seq;'
 
 EOF
 

testsuite/tests-to-run/parallel-tutorial.sh

@@ -47,7 +47,7 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' ../../src/
             # /usr/bin/time -f %e
             s/^(\d+)\.\d+$/$1/;
             # Base 64 string
-            s:[+/a-z0-9=]{50,}:BASE64:ig;
+            s:[\\+/a-z0-9=]{50,}:BASE64:ig;
             # --workdir ...
             s:parallel/tmp/aspire-\d+-1:TMPWORKDIR:g;
             # + cat ... | (Bash outputs these in random order)

testsuite/wanted-results/parallel-local-0.3s

@@ -466,7 +466,7 @@ echo '**'
 echo 'bug #45993: --wd ... should also work when run locally'
 bug #45993: --wd ... should also work when run locally
 parallel --wd /bi 'pwd; echo $OLDPWD; echo' ::: fail
-parallel: Error: Cannot write to /bi: No such file or directory
+parallel: Error: Cannot change into non-executable dir /bi: No such file or directory
 parallel --wd /bin 'pwd; echo $OLDPWD; echo' ::: OK
 /bin
 /home/tange/privat/parallel/testsuite

testsuite/wanted-results/parallel-local-sql

@@ -1,7 +1,7 @@
 echo '### --sqlandworker mysql'
 ### --sqlandworker mysql
   (sleep 2; parallel --sqlworker $MYSQLTBL sleep .3\;echo >$T1) &
-  parallel --sqlandworker $MYSQLTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2;   true sort -u $T1 $T2;   sql $MYSQL 'select * from parsql order by seq;'
+  parallel --sqlandworker $MYSQLTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2;   true sort -u $T1 $T2;   sleep 1; sql $MYSQL 'select * from parsql order by seq;'
 Seq	Host	Starttime	JobRuntime	Send	Receive	Exitval	_Signal	Command	V1	V2	Stdout	Stderr
 1	:9999990	4	0	0	sleep .3;echo 1 a	1	a	1 a\n	
 2	:9999990	4	0	0	sleep .3;echo 1 b	1	b	1 b\n	
@@ -31,7 +31,7 @@ Seq	Host	Starttime	JobRuntime	Send	Receive	Exitval	_Signal	Command	V1	V2	Stdout
 echo '### --sqlandworker postgresql'
 ### --sqlandworker postgresql
   (sleep 2; parallel --sqlworker $PGTBL sleep .3\;echo >$T3) &
-  parallel --sqlandworker $PGTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T4;   true sort -u $T3 $T4;   sql $PG 'select * from parsql order by seq;'
+  parallel --sqlandworker $PGTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T4;   true sort -u $T3 $T4;   sleep 1; sql $PG 'select * from parsql order by seq;'
  seq | host |   starttime    | jobruntime | send | receive | exitval | _signal |      command      | v1 | v2 | stdout | stderr 
 -----+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+--------
    1 | :    |999|999|    0 |       4 |       0 |       0 | sleep .3;echo 1 a | 1  | a  | 1 a   +| 
@@ -89,7 +89,7 @@ echo '### --sqlandworker postgresql'
 echo '### --sqlandworker sqlite'
 ### --sqlandworker sqlite
   (sleep 2; parallel --sqlworker $SQLITETBL sleep .3\;echo >$T5) &
-  parallel --sqlandworker $SQLITETBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T6;   true sort -u $T5 $T6;   sql $SQLITE 'select * from parsql order by seq;'
+  parallel --sqlandworker $SQLITETBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T6;   true sort -u $T5 $T6;   sleep 1; sql $SQLITE 'select * from parsql order by seq;'
 Seq|Host|Starttime|JobRuntime|Send|Receive|Exitval|_Signal|Command|V1|V2|Stdout|Stderr
 1|:|999|999|0|4|0|0|sleep .3;echo 1 a|1|a|1 a
 |
@@ -144,7 +144,7 @@ Seq|Host|Starttime|JobRuntime|Send|Receive|Exitval|_Signal|Command|V1|V2|Stdout|
 echo '### --sqlandworker postgresql -S lo'
 ### --sqlandworker postgresql -S lo
   (sleep 2; parallel -S lo --sqlworker $PGTBL2 sleep .3\;echo >$T7) &
-  parallel -S lo --sqlandworker $PGTBL2 sleep .3\;echo ::: {1..5} ::: {a..e} >$T8;   true sort -u $T7 $T8;   sql $PG 'select * from parsql2 order by seq;'
+  parallel -S lo --sqlandworker $PGTBL2 sleep .3\;echo ::: {1..5} ::: {a..e} >$T8;   true sort -u $T7 $T8;   sleep 1; sql $PG 'select * from parsql2 order by seq;'
  seq | host |   starttime    | jobruntime | send | receive | exitval | _signal |      command      | v1 | v2 | stdout | stderr 
 -----+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+--------
    1 | lo   |999|999|    0 |       4 |       0 |       0 | sleep .3;echo 1 a | 1  | a  | 1 a   +| 
@@ -203,7 +203,7 @@ echo '### --sqlandworker postgresql --results'
 ### --sqlandworker postgresql --results
   mkdir -p /tmp/out--sql
   (sleep 2; parallel --results /tmp/out--sql --sqlworker $PGTBL3 sleep .3\;echo >$T9) &
-  parallel --results /tmp/out--sql --sqlandworker $PGTBL3 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999;   true sort -u $T9 $T999;   sql $PG 'select * from parsql3 order by seq;'
+  parallel --results /tmp/out--sql --sqlandworker $PGTBL3 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999;   true sort -u $T9 $T999;   sleep 1; sql $PG 'select * from parsql3 order by seq;'
  seq | host |   starttime    | jobruntime | send | receive | exitval | _signal |      command      | v1 | v2 |            stdout            |            stderr            
 -----+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+------------------------------+------------------------------
    1 | :    |999|999|    0 |       4 |       0 |       0 | sleep .3;echo 1 a | 1  | a  | /tmp/out--sql/1/1/2/a/stdout | /tmp/out--sql/1/1/2/a/stderr
@@ -236,7 +236,7 @@ echo '### --sqlandworker postgresql --results'
 echo '### --sqlandworker postgresql --linebuffer'
 ### --sqlandworker postgresql --linebuffer
   (sleep 2; parallel --linebuffer --sqlworker $PGTBL4 sleep .3\;echo >$T999) &
-  parallel --linebuffer --sqlandworker $PGTBL4 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999;   true sort -u $T999$T999;   sql $PG 'select * from parsql4 order by seq;'
+  parallel --linebuffer --sqlandworker $PGTBL4 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999;   true sort -u $T999$T999;   sleep 1; sql $PG 'select * from parsql4 order by seq;'
  seq | host |   starttime    | jobruntime | send | receive | exitval | _signal |      command      | v1 | v2 | stdout | stderr 
 -----+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+--------
    1 | :    |999|999|    0 |       4 |       0 |       0 | sleep .3;echo 1 a | 1  | a  |        | 
@@ -269,7 +269,7 @@ echo '### --sqlandworker postgresql --linebuffer'
 echo '### --sqlandworker postgresql -u'
 ### --sqlandworker postgresql -u
   (sleep 2; parallel -u --sqlworker $PGTBL5 sleep .3\;echo >$T999) &
-  parallel -u --sqlandworker $PGTBL5 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999;   true sort -u $T999$T999;   sql $PG 'select * from parsql5 order by seq;'
+  parallel -u --sqlandworker $PGTBL5 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999;   true sort -u $T999$T999;   sleep 1; sql $PG 'select * from parsql5 order by seq;'
  seq | host |   starttime    | jobruntime | send | receive | exitval | _signal |      command      | v1 | v2 | stdout | stderr 
 -----+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+--------
    1 | :    |999|999|    0 |       0 |       0 |       0 | sleep .3;echo 1 a | 1  | a  |        | 

testsuite/wanted-results/parallel-local150

@@ -242,4 +242,4 @@ echo '### Test bug #35820: sem breaks if $HOME is not writable'
   echo 'Workaround: use another writable dir';   rm -rf /tmp/.parallel;   HOME=/tmp sem echo OK;   HOME=/tmp sem --wait;   HOME=/usr/this/should/fail stdout sem echo should fail
 Workaround: use another writable dir
 OK
-parallel: Error: Cannot write to /usr/this/should/fail/.parallel: No such file or directory
+parallel: Error: Cannot change into non-executable dir /usr/this/should/fail/.parallel: No such file or directory

testsuite/wanted-results/parallel-tutorial

@@ -861,7 +861,7 @@ _
   }
   export -f my_func3
   parallel -vv --workdir ... --nice 17 --env _ --trc {}.out -S $SERVER1 my_func3 {} ::: abc-file
-( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; BASE64 BASE64\=;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status;
+( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; BASE64 BASE64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status;
   parallel --sqlandworker csv:////%2Ftmp%2Flog.csv seq ::: 10 ::: 12 13 14
   cat /tmp/log.csv
 10