From bf91e06e003245541233cd4ca126096a256b83c3 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 20 Mar 2016 23:57:25 +0100 Subject: [PATCH] Fixed bug #47462: --transfer with file containing /./ is not done relative. Added env_parallel to Makefile. --sql => --sqlmaster. ssh -tt code removed. Send/return column in --joblog set to 0 instead of null. parallel_tutorial.pod: Added CSV DBURL example. --- src/Makefile.am | 4 +- src/Makefile.in | 5 +- src/parallel | 96 +++-- src/parallel.pod | 31 +- src/parallel_tutorial.html | 86 +++-- src/parallel_tutorial.pod | 96 +++-- testsuite/tests-to-run/parallel-local-ssh1.sh | 7 + testsuite/tests-to-run/parallel-local-ssh4.sh | 11 +- testsuite/tests-to-run/parallel-tutorial.sh | 2 + testsuite/wanted-results/parallel-install | 2 +- testsuite/wanted-results/parallel-local-sql | 350 +++++++++--------- testsuite/wanted-results/parallel-local-ssh1 | 11 +- testsuite/wanted-results/parallel-local-ssh4 | 6 +- testsuite/wanted-results/parallel-local150 | 7 + testsuite/wanted-results/parallel-tutorial | 61 ++- 15 files changed, 452 insertions(+), 323 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 607424c1..0851cc06 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,4 +1,5 @@ -bin_SCRIPTS = parallel sql niceload +bin_SCRIPTS = parallel sql niceload \ + env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh install-exec-hook: rm $(DESTDIR)$(bindir)/sem || true @@ -147,5 +148,6 @@ DISTCLEANFILES = parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.7 parallel_ parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf EXTRA_DIST = parallel sem sql niceload \ + env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \ sem.pod parallel.pod niceload.pod parallel_tutorial.pod parallel_design.pod \ $(DISTCLEANFILES) diff --git a/src/Makefile.in b/src/Makefile.in index aa536945..41fc47da 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -217,7 +217,9 @@ target_alias = @target_alias@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ -bin_SCRIPTS = parallel sql niceload +bin_SCRIPTS = parallel sql niceload \ + env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh + @DOCUMENTATION_TRUE@man_MANS = parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.7 parallel_design.7 @DOCUMENTATION_TRUE@doc_DATA = parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html \ @DOCUMENTATION_TRUE@ parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi \ @@ -229,6 +231,7 @@ DISTCLEANFILES = parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.7 parallel_ parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf EXTRA_DIST = parallel sem sql niceload \ + env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \ sem.pod parallel.pod niceload.pod parallel_tutorial.pod parallel_design.pod \ $(DISTCLEANFILES) diff --git a/src/parallel b/src/parallel index dcdcbb3c..615dae39 100755 --- a/src/parallel +++ b/src/parallel @@ -59,7 +59,7 @@ if($opt::pipepart) { @input_source_fh = (*STDIN); } } -if($opt::sql) { +if($opt::sqlmaster) { # Create SQL table to hold joblog + output $Global::sql->create_table($#input_source_fh+1); if($opt::sqlworker) { @@ -760,7 +760,8 @@ sub options_hash { "m" => \$opt::m, "X" => \$opt::X, "v" => \@opt::v, - "sql=s" => \$opt::sql, + "sql=s" => \$opt::retired, + "sqlmaster=s" => \$opt::sqlmaster, "sqlworker=s" => \$opt::sqlworker, "sqlandworker=s" => \$opt::sqlandworker, "joblog=s" => \$opt::joblog, @@ -972,7 +973,7 @@ sub parse_options { if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; } $opt::nice ||= 0; if(defined $opt::help) { die_usage(); } - if(defined $opt::sqlandworker) { $opt::sql = $opt::sqlworker = $opt::sqlandworker; } + if(defined $opt::sqlandworker) { $opt::sqlmaster = $opt::sqlworker = $opt::sqlandworker; } if(defined $opt::colsep) { $Global::trim = 'lr'; } if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; } if(defined $opt::trim) { $Global::trim = $opt::trim; } @@ -1121,7 +1122,9 @@ sub parse_options { $opt::jobs = "100%"; } open_joblog(); - ($opt::sql or $opt::sqlworker) and $Global::sql = SQL->new($opt::sql || $opt::sqlworker); + if($opt::sqlmaster or $opt::sqlworker) { + $Global::sql = SQL->new($opt::sqlmaster || $opt::sqlworker); + } } sub check_invalid_option_combinations { @@ -1168,6 +1171,7 @@ sub check_invalid_option_combinations { "-W has been retired. Use --wd.", "-Y has been retired. Use --shebang.", "-H has been retired. Use --halt.", + "--sql has been retired. Use --sqlmaster.", "--ctrlc has been retired.", "--noctrlc has been retired."); ::wait_and_exit(255); @@ -2356,7 +2360,7 @@ sub drain_job_queue { ::warning("There are no job slots available. Increase --jobs."); } } - while($opt::sql and not $Global::sql->finished()) { + while($opt::sqlmaster and not $Global::sql->finished()) { # SQL master $sleep = ::reap_usleep($sleep); if($Global::sqlworker) { @@ -2376,7 +2380,7 @@ sub drain_job_queue { or not $Global::start_no_new_jobs and not $Global::JobQueue->empty() or - $opt::sql and not $Global::sql->finished()); + $opt::sqlmaster and not $Global::sql->finished()); if($opt::progress) { my %progress = progress(); ::status("\r", $progress{'status'}, "\n"); @@ -3344,8 +3348,8 @@ sub reaper { my $stiff; my @pids_reaped; debug("run", "Reaper "); - # For efficiency surround with BEGIN/COMMIT when using $opt::sql - $opt::sql and $Global::sql->run("BEGIN;"); + # For efficiency surround with BEGIN/COMMIT when using $opt::sqlmaster + $opt::sqlmaster and $Global::sql->run("BEGIN;"); while (($stiff = waitpid(-1, &WNOHANG)) > 0) { # $stiff = pid of dead process push(@pids_reaped,$stiff); @@ -3403,7 +3407,7 @@ sub reaper { ::status("\r",$progress{'status'}); } } - $opt::sql and $Global::sql->run("COMMIT;"); + $opt::sqlmaster and $Global::sql->run("COMMIT;"); debug("run", "done "); return @pids_reaped; } @@ -5821,11 +5825,12 @@ sub rsync_transfer_cmd { return "true"; } my $rsync_destdir; - if($file =~ m:^/:) { + my $relpath = ($file !~ m:^/:) || ($file =~ m:/\./:); # Is the path relative or /./? + if($relpath) { + $rsync_destdir = ::shell_quote_file($workdir); + } else { # rsync /foo/bar / $rsync_destdir = "/"; - } else { - $rsync_destdir = ::shell_quote_file($workdir); } $file = ::shell_quote_file($file); my $sshcmd = $self->sshcommand(); @@ -6897,14 +6902,17 @@ sub base64_eval { # * decodes them # * evals the result # Reverse of string_base64 + eval - # Will be wrapped in ' so single quote is forbidden + # Will be wrapped in ' so single quote is forbidden. + # Spaces are stripped so spaces cannot be significant. + # The funny 'use IPC::Open3'-syntax is to avoid spaces and + # to make it clear that this is a GNU Parallel command + # when looking at the process table. # Returns: # $script = 1-liner for perl -e my $script = ::spacefree(0,q{ @GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64"); eval "@GNU_Parallel"; - my $eval; - $eval = decode_base64(join"",@ARGV); + my $eval = decode_base64(join"",@ARGV); eval $eval; }); ::debug("base64",$script,"\n"); @@ -7145,16 +7153,17 @@ sub transfer { # Returns: # @transfer - File names of files to transfer my $self = shift; - $self->{'transfersize'} = 0; + my $transfersize = 0; my @transfer = $self->{'commandline'}-> replace_placeholders($self->{'commandline'}{'transfer_files'},0,0); for(@transfer) { # filesize if(-e $_) { - $self->{'transfersize'} += (stat($_))[7]; + $transfersize += (stat($_))[7]; } } + $self->add_transfersize($transfersize); return @transfer; } @@ -7228,7 +7237,7 @@ sub sshreturn { my $pre = ""; for my $file ($self->return()) { $file =~ s:^\./::g; # Remove ./ if any - my $relpath = ($file !~ m:^/:); # Is the path relative? + my $relpath = ($file !~ m:^/:) || ($file =~ m:/\./:); # Is the path relative or /./? my $cd = ""; my $wd = ""; if($relpath) { @@ -7454,7 +7463,7 @@ sub start { if($opt::ungroup or $opt::sqlworker) { print_dryrun_and_verbose($stdout_fh,$job,$command); } - if($opt::dryrun or $opt::sql) { $command = "true"; } + if($opt::dryrun or $opt::sqlmaster) { $command = "true"; } $ENV{'PARALLEL_SEQ'} = $job->seq(); $ENV{'PARALLEL_PID'} = $$; $ENV{'PARALLEL_TMP'} = ::tmpname("par"); @@ -7735,11 +7744,15 @@ sub print { if($opt::pipe and $self->virgin()) { # Skip --joblog, --dryrun, --verbose } else { - if($opt::ungroup and $Global::joblog and defined $self->{'exitstatus'}) { - # Add to joblog when finished - $self->print_joblog(); - # Printing is only relevant for grouped/--line-buffer output. - $opt::ungroup and return; + if($opt::ungroup) { + # NULL returnsize = 0 returnsize + $self->returnsize() or $self->add_returnsize(0); + if($Global::joblog and defined $self->{'exitstatus'}) { + # Add to joblog when finished + $self->print_joblog(); + # Printing is only relevant for grouped/--line-buffer output. + $opt::ungroup and return; + } } # Check for disk full @@ -7749,7 +7762,7 @@ sub print { and not $self->{'verbose_printed'} and - not $opt::sql + not $opt::sqlmaster and not $opt::sqlworker) { $self->{'verbose_printed'}++; @@ -7870,13 +7883,6 @@ sub linebuffer_print { my $i = ::rindex64($partial,"\n"); if($i != -1) { # One or more complete lines were found - if($fdno == 2 and not $self->{'printed_first_line',$fdno}++) { - # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt - # This is a crappy way of ignoring it. - $$partial =~ s/^(client_process_control: )?tcgetattr: Invalid argument\n//; - # Length of partial line has changed: Find the last \n again - $i = ::rindex64($partial,"\n"); - } $outputlength += $i+1; if($opt::tag or defined $opt::tagstring) { # Replace ^ with $tag within the full line @@ -7929,20 +7935,6 @@ sub tag_print { seek $in_fh, 0, 0; # $in_fh is now ready for reading at position 0 my $tag = $self->tag(); - if($fdno == 2) { - # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt - # This is a crappy way of ignoring it. - while(<$in_fh>) { - if(/^(client_process_control: )?tcgetattr: Invalid argument\n/) { - # Skip - } else { - $self->add_returnsize(length $_); - print $out_fd $tag,$_; - } - # At most run the loop once - last; - } - } my $outputlength = 0; while(<$in_fh>) { print $out_fd $tag,$_; @@ -8299,7 +8291,7 @@ sub populate { } } } - if($opt::sql) { + if($opt::sqlmaster) { # Insert the V1..Vn for this $seq in SQL table instead of generating one $Global::sql->insert_records($self->seq(),$self->{'arg_list_flat_orig'}); } @@ -9924,7 +9916,7 @@ sub run { } my @retval; my $dbh = $self->{'dbh'}; - ::debug("sql","$opt::sql$opt::sqlworker run $stmt\n"); + ::debug("sql","$opt::sqlmaster$opt::sqlworker run $stmt\n"); # Execute with the rest of the args - if any my $rv; my $sth; @@ -10034,10 +10026,10 @@ sub insert_records { my $record_ref = shift; my $table = $self->table(); my $v_cols = join ",", map { "V$_" } (1..$self->max_number_of_args()); - # Two extra value due to $seq, Exitval - my $v_vals = join ",", map { "?" } (1..$self->max_number_of_args()+2); - $self->run("INSERT INTO $table (Seq,Exitval,$v_cols) ". - "VALUES ($v_vals);", $seq, -1000, @$record_ref[1..$#$record_ref]); + # Two extra value due to $seq, Exitval, Send + my $v_vals = join ",", map { "?" } (1..$self->max_number_of_args()+3); + $self->run("INSERT INTO $table (Seq,Exitval,Send,$v_cols) ". + "VALUES ($v_vals);", $seq, -1000, 0, @$record_ref[1..$#$record_ref]); } sub get_record { diff --git a/src/parallel.pod b/src/parallel.pod index 399134e9..8b5399d0 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1791,7 +1791,12 @@ Do not use the first line of input (used by GNU B itself when called with B<--shebang>). -=item B<--sql> I +=item B<--sql> I (obsolete) + +Use B<--sqlmaster> instead. + + +=item B<--sqlmaster> I Submit jobs via SQL server. I must point to a table, which will contain the same information as B<--joblog>, the values from the input @@ -1801,9 +1806,9 @@ columns Stdout and Stderr). The table will be dropped and created with the correct amount of V-columns. -B<--sql> does not run any jobs, but it creates the values for the jobs -to be run and wait for them to complete. One or more B<--sqlworker> -must be run to actually execute the jobs. +B<--sqlmaster> does not run any jobs, but it creates the values for +the jobs to be run and wait for them to complete. One or more +B<--sqlworker> must be run to actually execute the jobs. The format of a DBURL is: @@ -1825,14 +1830,14 @@ It can also be an alias from ~/.sql/aliases: =item B<--sqlandworker> I -Shorthand for: B<--sql> I B<--sqlworker> I. +Shorthand for: B<--sqlmaster> I B<--sqlworker> I. =item B<--sqlworker> I Execute jobs via SQL server. Read the input sources variables from the table pointed to by I. The I on the command line -should be the same as given by B<--sql>. +should be the same as given by B<--sqlmaster>. =item B<--ssh> I @@ -2079,11 +2084,11 @@ I to the file I before running B on I. echo /tmp/./foo/bar.txt | parallel \ - --sshlogin server.example.com --transferfile {} wc + --sshlogin server.example.com --transferfile {} wc {= s:.*/./:./: =} This will transfer the file I to the computer I to the file I before running -B on I. +B on I. B<--transferfile> is often used with B<--return> and B<--cleanup>. A shorthand for B<--transferfile {}> is B<--transfer>. @@ -3857,17 +3862,13 @@ Killed by Ctrl-C, timeout, not enough memory or similar. $job->skip() was called in {= =}. -=item Z<>-1200 (In SQL table) +=item Z<>-1000 (In SQL table) -Job is ready to run (used with --sql). +Job is ready to run (set by --sqlmaster). =item Z<>-1220 (In SQL table) -Job is taken by worker to be started (used with --sql). - -=item Z<>-1250 (In SQL table) - -Job is running (used with --sql). +Job is taken by worker (set by --sqlworker). =back diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html index 304c1840..e574b8cd 100644 --- a/src/parallel_tutorial.html +++ b/src/parallel_tutorial.html @@ -83,6 +83,8 @@
  • Saving to an SQL base (advanced)
  • @@ -125,10 +127,10 @@
    -
    parallel >= version 20140622
    +
    parallel >= version 20160222
    -

    Install the newest version with:

    +

    Install the newest version using your package manager or with:

      (wget -O - pi.dk/3 || curl pi.dk/3/ || fetch -o - http://pi.dk/3) | bash
    @@ -289,7 +291,7 @@

    STDIN (standard input) can be one of the input sources using -:

    -
      cat abc-file | parallel -a - -a def-file echo 
    +
      cat abc-file | parallel -a - -a def-file echo

    Output: Same as above.

    @@ -1254,7 +1256,7 @@ 1:local / 2 / 2 Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete - ETA: 2s 0left 1.11avg local:0/9/100%/1.1s + ETA: 2s 0left 1.11avg local:0/9/100%/1.1s

    GNU parallel can give progress information with --progress:

    @@ -1278,7 +1280,7 @@

    A logfile of the jobs completed so far can be generated with --joblog:

    -
      parallel --joblog /tmp/log exit  ::: 1 2 3 0 
    +
      parallel --joblog /tmp/log exit  ::: 1 2 3 0
       cat /tmp/log

    Output:

    @@ -1293,7 +1295,7 @@

    With a joblog GNU parallel can be stopped and later pickup where it left off. It it important that the input of the completed jobs is unchanged.

    -
      parallel --joblog /tmp/log exit  ::: 1 2 3 0 
    +
      parallel --joblog /tmp/log exit  ::: 1 2 3 0
       cat /tmp/log
       parallel --resume --joblog /tmp/log exit  ::: 1 2 3 0 0 0
       cat /tmp/log
    @@ -1443,7 +1445,7 @@

    Using --termseq you can control which signals are sent when killing children. Normally children will be killed by sending them SIGTERM, waiting 200 ms, then another SIGTERM, waiting 100 ms, then another SIGTERM, waiting 50 ms, then a SIGKILL, finally waiting 25 ms before giving up. It looks like this:

      show_signals() {
    -    perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print \"Got $_\\n\"; }";} while(1){sleep 1}' 
    +    perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print \"Got $_\\n\"; }";} while(1){sleep 1}'
       }
       export -f show_signals
       echo | parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 -u --timeout 1 show_signals
    @@ -1475,7 +1477,7 @@

    To avoid overloading systems GNU parallel can look at the system load before starting another job:

    -
      parallel --load 100% echo load is less than {} job per cpu ::: 1 
    +
      parallel --load 100% echo load is less than {} job per cpu ::: 1

    Output:

    @@ -1619,7 +1621,7 @@

    GNU parallel can transfer the files to be processed to the remote host. It does that using rsync.

      echo This is input_file > input_file
    -  parallel -S $SERVER1 --transferfile {} cat ::: input_file 
    + parallel -S $SERVER1 --transferfile {} cat ::: input_file

    Output:

    @@ -1628,7 +1630,7 @@

    If the files are processed into another file, the resulting file can be transferred back:

      echo This is input_file > input_file
    -  parallel -S $SERVER1 --transferfile {} --return {}.out cat {} ">"{}.out ::: input_file 
    +  parallel -S $SERVER1 --transferfile {} --return {}.out cat {} ">"{}.out ::: input_file
       cat input_file.out

    Output: Same as above.

    @@ -1636,7 +1638,7 @@

    To remove the input and output file on the remote server use --cleanup:

      echo This is input_file > input_file
    -  parallel -S $SERVER1 --transferfile {} --return {}.out --cleanup cat {} ">"{}.out ::: input_file 
    +  parallel -S $SERVER1 --transferfile {} --return {}.out --cleanup cat {} ">"{}.out ::: input_file
       cat input_file.out

    Output: Same as above.

    @@ -1644,7 +1646,7 @@

    There is a shorthand for --transferfile {} --return --cleanup called --trc:

      echo This is input_file > input_file
    -  parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file 
    +  parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file
       cat input_file.out

    Output: Same as above.

    @@ -1703,7 +1705,7 @@

    In clusters with many hosts a few of them are often down. GNU parallel can ignore those hosts. In this case the host 173.194.32.46 is down:

    -
      parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar 
    +
      parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar

    Output:

    @@ -1802,7 +1804,7 @@

    Output:

      ssh lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
    -  eval"@GNU_Parallel";my$eval;$eval=decode_base64(join"",@ARGV);eval$eval;'\' 
    +  eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\'
       JEVOVnsiUEFSQUxMRUxfUElEIn09IjI3MzQiOyRFTlZ7IlBBUkFMTEVMX1NFUSJ9PSIx
       IjskYmFzaGZ1bmMgPSAiIjtAQVJHVj0iZWNobyBiYXIiOyRzaGVsbD0iJEVOVntTSEVM
       TH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTA7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
    @@ -1829,7 +1831,7 @@
     
      ( ssh lo -- mkdir -p ./.parallel/tmp/hk-3492-1;rsync --protocol 30
       -rlDzR -essh ./abc-file lo:./.parallel/tmp/hk-3492-1 );ssh lo --
       exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
    -  eval"@GNU_Parallel";my$eval;$eval=decode_base64(join"",@ARGV);eval$eval;'\'
    +  eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\'
       c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2hrLTM0OTItMSIp
       OyBjaGRpciAiLnBhcmFsbGVsL3RtcC9oay0zNDkyLTEiIHx8cHJpbnQoU1RERVJSICJw
       YXJhbGxlbDogQ2Fubm90IGNoZGlyIHRvIC5wYXJhbGxlbC90bXAvaGstMzQ5Mi0xXG4i
    @@ -1867,7 +1869,41 @@
     
     

    Saving to an SQL base (advanced)

    -

    GNU parallel can save into an SQL base. Point GNU parallel to a table and it will put the joblog there together with the variables and the outout each in their own column.

    +

    GNU parallel can save into an SQL base. Point GNU parallel to a table and it will put the joblog there together with the variables and the output each in their own column.

    + +

    CSV as SQL base

    + +

    The simplest is to use a CSV file as the storage table:

    + +
      parallel --sqlandworker csv:////%2Ftmp%2Flog.csv seq ::: 10 ::: 12 13 14
    +  cat /tmp/log.csv
    + +

    Note how '/' in the path must be written as %2F.

    + +

    Output will be similar to:

    + +
      Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
    +  1,:,1458254498.254,0.069,0,9,0,0,"seq 10 12",10,12,"10
    +  11
    +  12
    +  ",
    +  2,:,1458254498.278,0.080,0,12,0,0,"seq 10 13",10,13,"10
    +  11
    +  12
    +  13
    +  ",
    +  3,:,1458254498.301,0.083,0,15,0,0,"seq 10 14",10,14,"10
    +  11
    +  12
    +  13
    +  14
    +  ",
    + +

    A proper CSV reader (like LibreOffice or R's read.csv) will read this format correctly - even with fields containing newlines as above.

    + +

    DBURL as table

    + +

    The CSV file is an example of a DBURL.

    GNU parallel uses a DBURL to address the table. A DBURL has this format:

    @@ -1877,9 +1913,10 @@
      mysql://scott:tiger@my.example.com/mydatabase/mytable
       postgresql://scott:tiger@pg.example.com/mydatabase/mytable
    -  sqlite3:///%2Ftmp%2Fmydatabase/mytable
    + sqlite3:///%2Ftmp%2Fmydatabase/mytable + csv:////%2Ftmp%2Flog.csv
    -

    To refer to /tmp/mydatabase with sqlite you need to encode the / as %2F.

    +

    To refer to /tmp/mydatabase with sqlite or csv you need to encode the / as %2F.

    Run a job using sqlite on mytable in /tmp/mydatabase:

    @@ -1907,15 +1944,15 @@

    Using multiple workers

    -

    Using an SQL base as storage costs a lot of performance.

    +

    Using an SQL base as storage costs overhead in the order of 1 second per job.

    One of the situations where it makes sense is if you have multiple workers.

    You can then have a single master machine that submits jobs to the SQL base (but does not do any of the work):

    -
      parallel --sql $DBURLTABLE echo ::: foo bar ::: baz quuz
    +
      parallel --sqlmaster $DBURLTABLE echo ::: foo bar ::: baz quuz
    -

    On the worker machines you run exactly the same command except you replace --sql with --sqlworker.

    +

    On the worker machines you run exactly the same command except you replace --sqlmaster with --sqlworker.

      parallel --sqlworker $DBURLTABLE echo ::: foo bar ::: baz quuz
    @@ -2340,8 +2377,7 @@
      #!/usr/bin/parallel --shebang-wrap ARGV={} /usr/bin/csharp
       
       var argv = Environment.GetEnvironmentVariable("ARGV");
    -  print("Arguments "+argv);
    -  
    + print("Arguments "+argv);
    @@ -2498,7 +2534,7 @@ When using GNU Parallel for a publication please cite: - O. Tange (2011): GNU Parallel - The Command-Line Power Tool, + O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47.

    In scripts --minversion can be used to ensure the user has at least this version:

    @@ -2539,7 +2575,7 @@

    --number-of-cpus and --number-of-cores run system specific code to determine the number of CPUs and CPU cores on the system. On unsupported platforms they will return 1:

    -
      parallel --number-of-cpus 
    +
      parallel --number-of-cpus
       parallel --number-of-cores

    Output (may vary on different systems):

    diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index ddd26c60..5c20b902 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -164,7 +164,7 @@ Output: Same as above. STDIN (standard input) can be one of the input sources using B<->: - cat abc-file | parallel -a - -a def-file echo + cat abc-file | parallel -a - -a def-file echo Output: Same as above. @@ -1045,7 +1045,7 @@ The directories are named after the variables and their values. =head1 Controlling the execution -=head2 Number of simultaneous jobs +=head2 Number of simultaneous jobs The number of concurrent jobs is given with B<--jobs>/B<-j>: @@ -1080,7 +1080,7 @@ The first second only 50% of the CPU cores will run a job. Then B<0> is put into B and then the rest of the jobs will be started in parallel. -Instead of basing the percentage on the number of CPU cores +Instead of basing the percentage on the number of CPU cores GNU B can base it on the number of CPUs: parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8 @@ -1188,7 +1188,7 @@ Output: 1:local / 2 / 2 Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete - ETA: 2s 0left 1.11avg local:0/9/100%/1.1s + ETA: 2s 0left 1.11avg local:0/9/100%/1.1s GNU B can give progress information with B<--progress>: @@ -1212,7 +1212,7 @@ And a graphic bar can be shown with B<--bar> and B: A logfile of the jobs completed so far can be generated with B<--joblog>: - parallel --joblog /tmp/log exit ::: 1 2 3 0 + parallel --joblog /tmp/log exit ::: 1 2 3 0 cat /tmp/log Output: @@ -1232,7 +1232,7 @@ With a joblog GNU B can be stopped and later pickup where it left off. It it important that the input of the completed jobs is unchanged. - parallel --joblog /tmp/log exit ::: 1 2 3 0 + parallel --joblog /tmp/log exit ::: 1 2 3 0 cat /tmp/log parallel --resume --joblog /tmp/log exit ::: 1 2 3 0 0 0 cat /tmp/log @@ -1338,7 +1338,7 @@ before GNU B stops spawning more jobs: parallel -j2 --halt soon,fail=20% echo {}\; exit {} ::: 0 1 2 3 4 5 6 7 8 9 Output: - + 0 1 parallel: This job failed: @@ -1396,7 +1396,7 @@ B, waiting 50 ms, then a B, finally waiting 25 ms before giving up. It looks like this: show_signals() { - perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print \"Got $_\\n\"; }";} while(1){sleep 1}' + perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print \"Got $_\\n\"; }";} while(1){sleep 1}' } export -f show_signals echo | parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 -u --timeout 1 show_signals @@ -1417,7 +1417,7 @@ You can change this to B, B, B: echo | parallel --termseq INT,200,TERM,100,KILL,25 -u --timeout 1 show_signals -Output: +Output: Got INT Got TERM @@ -1430,14 +1430,14 @@ The B does not show because it cannot be caught, and thus the child die To avoid overloading systems GNU B can look at the system load before starting another job: - parallel --load 100% echo load is less than {} job per cpu ::: 1 + parallel --load 100% echo load is less than {} job per cpu ::: 1 Output: [when then load is less than the number of cpu cores] load is less than 1 job per cpu -GNU B can also check if the system is swapping. +GNU B can also check if the system is swapping. parallel --noswap echo the system is not swapping ::: now @@ -1466,7 +1466,7 @@ Output: =head1 Remote execution GNU B can run jobs on remote servers. It uses B to -communicate with the remote machines. +communicate with the remote machines. =head2 Sshlogin @@ -1587,7 +1587,7 @@ GNU B can transfer the files to be processed to the remote host. It does that using rsync. echo This is input_file > input_file - parallel -S $SERVER1 --transferfile {} cat ::: input_file + parallel -S $SERVER1 --transferfile {} cat ::: input_file Output: @@ -1597,7 +1597,7 @@ If the files are processed into another file, the resulting file can be transferred back: echo This is input_file > input_file - parallel -S $SERVER1 --transferfile {} --return {}.out cat {} ">"{}.out ::: input_file + parallel -S $SERVER1 --transferfile {} --return {}.out cat {} ">"{}.out ::: input_file cat input_file.out Output: Same as above. @@ -1605,7 +1605,7 @@ Output: Same as above. To remove the input and output file on the remote server use B<--cleanup>: echo This is input_file > input_file - parallel -S $SERVER1 --transferfile {} --return {}.out --cleanup cat {} ">"{}.out ::: input_file + parallel -S $SERVER1 --transferfile {} --return {}.out --cleanup cat {} ">"{}.out ::: input_file cat input_file.out Output: Same as above. @@ -1613,7 +1613,7 @@ Output: Same as above. There is a shorthand for B<--transferfile {} --return --cleanup> called B<--trc>: echo This is input_file > input_file - parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file + parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file cat input_file.out Output: Same as above. @@ -1689,7 +1689,7 @@ Output: Same as above. In clusters with many hosts a few of them are often down. GNU B can ignore those hosts. In this case the host 173.194.32.46 is down: - parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar + parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar Output: @@ -1794,9 +1794,9 @@ the workdir, and setting B<--nice> value. B<-vv> shows all of this. parallel -vv -S $SERVER1 echo ::: bar Output: - + ssh lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64"); - eval"@GNU_Parallel";my$eval;$eval=decode_base64(join"",@ARGV);eval$eval;'\' + eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\' JEVOVnsiUEFSQUxMRUxfUElEIn09IjI3MzQiOyRFTlZ7IlBBUkFMTEVMX1NFUSJ9PSIx IjskYmFzaGZ1bmMgPSAiIjtAQVJHVj0iZWNobyBiYXIiOyRzaGVsbD0iJEVOVntTSEVM TH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTA7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0 @@ -1810,7 +1810,8 @@ Output: JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==; bar -When the command gets more complex, the output is so hard to read, that it is only useful for debugging: +When the command gets more complex, the output is so hard to read, +that it is only useful for debugging: my_func3() { echo in my_func $1 > $1.out @@ -1823,7 +1824,7 @@ Output will be similar to: ( ssh lo -- mkdir -p ./.parallel/tmp/hk-3492-1;rsync --protocol 30 -rlDzR -essh ./abc-file lo:./.parallel/tmp/hk-3492-1 );ssh lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64"); - eval"@GNU_Parallel";my$eval;$eval=decode_base64(join"",@ARGV);eval$eval;'\' + eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\' c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2hrLTM0OTItMSIp OyBjaGRpciAiLnBhcmFsbGVsL3RtcC9oay0zNDkyLTEiIHx8cHJpbnQoU1RERVJSICJw YXJhbGxlbDogQ2Fubm90IGNoZGlyIHRvIC5wYXJhbGxlbC90bXAvaGstMzQ5Mi0xXG4i @@ -1863,7 +1864,42 @@ Output will be similar to: GNU B can save into an SQL base. Point GNU B to a table and it will put the joblog there together with the variables and -the outout each in their own column. +the output each in their own column. + +=head2 CSV as SQL base + +The simplest is to use a CSV file as the storage table: + + parallel --sqlandworker csv:////%2Ftmp%2Flog.csv seq ::: 10 ::: 12 13 14 + cat /tmp/log.csv + +Note how '/' in the path must be written as %2F. + +Output will be similar to: + + Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr + 1,:,1458254498.254,0.069,0,9,0,0,"seq 10 12",10,12,"10 + 11 + 12 + ", + 2,:,1458254498.278,0.080,0,12,0,0,"seq 10 13",10,13,"10 + 11 + 12 + 13 + ", + 3,:,1458254498.301,0.083,0,15,0,0,"seq 10 14",10,14,"10 + 11 + 12 + 13 + 14 + ", + +A proper CSV reader (like LibreOffice or R's read.csv) will read this +format correctly - even with fields containing newlines as above. + +=head2 DBURL as table + +The CSV file is an example of a DBURL. GNU B uses a DBURL to address the table. A DBURL has this format: @@ -1874,8 +1910,10 @@ Example: mysql://scott:tiger@my.example.com/mydatabase/mytable postgresql://scott:tiger@pg.example.com/mydatabase/mytable sqlite3:///%2Ftmp%2Fmydatabase/mytable + csv:////%2Ftmp%2Flog.csv -To refer to B with B you need to encode the B as B<%2F>. +To refer to B with B or B you need to +encode the B as B<%2F>. Run a job using B on B in B: @@ -1905,7 +1943,8 @@ output and standard error, respectively. =head2 Using multiple workers -Using an SQL base as storage costs a lot of performance. +Using an SQL base as storage costs overhead in the order of 1 second +per job. One of the situations where it makes sense is if you have multiple workers. @@ -1913,10 +1952,10 @@ workers. You can then have a single master machine that submits jobs to the SQL base (but does not do any of the work): - parallel --sql $DBURLTABLE echo ::: foo bar ::: baz quuz + parallel --sqlmaster $DBURLTABLE echo ::: foo bar ::: baz quuz On the worker machines you run exactly the same command except you -replace B<--sql> with B<--sqlworker>. +replace B<--sqlmaster> with B<--sqlworker>. parallel --sqlworker $DBURLTABLE echo ::: foo bar ::: baz quuz @@ -2356,7 +2395,6 @@ This technique can be used for: var argv = Environment.GetEnvironmentVariable("ARGV"); print("Arguments "+argv); - =back @@ -2531,7 +2569,7 @@ Output: When using GNU Parallel for a publication please cite: - O. Tange (2011): GNU Parallel - The Command-Line Power Tool, + O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. In scripts B<--minversion> can be used to ensure the user has at least @@ -2577,7 +2615,7 @@ B<--number-of-cpus> and B<--number-of-cores> run system specific code to determine the number of CPUs and CPU cores on the system. On unsupported platforms they will return 1: - parallel --number-of-cpus + parallel --number-of-cpus parallel --number-of-cores Output (may vary on different systems): diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index fee3d525..8507be32 100644 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -116,6 +116,13 @@ echo '### Test --trc "-- " "-- "' parallel --nonall -k -S csh@lo,sh@lo 'ls ./?--.a || echo OK'; parallel --nonall -k -S csh@lo,sh@lo 'ls ./--?.a || echo OK' +echo '### Test --trc "/tmp/./--- /A" "/tmp/./ ---/B"' + mkdir -p '/tmp/./--- ' '/tmp/ ---'; + touch -- '/tmp/./--- /A' '/tmp/ ---/B'; + rm -f ./---?/A.a ./?---/B.a; + parallel --trc {=s:.*/./::=}.a -S csh@lo,sh@lo touch ./{=s:.*/./::=}.a ::: '/tmp/./--- /A' '/tmp/./ ---/B'; + ls ./---?/A.a ./?---/B.a; + parallel --nonall -k -S csh@lo,sh@lo 'ls ./?--- ./---? || echo OK'; echo '### bug #46519: --onall ignores --transfer' touch bug46519.{a,b,c}; rm -f bug46519.?? bug46519.???; diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index 44f20479..34c34e0e 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -1,5 +1,7 @@ #!/bin/bash +# TODO ksh fish + unset run_test # SSH only allowed to localhost/lo @@ -18,11 +20,7 @@ echo 'bug #40137: SHELL not bash: Warning when exporting funcs' . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning echo 'env_parallel from man page - transfer non-exported var' - env_parallel() { - export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"; - `which parallel` "$@"; - unset parallel_bash_environment; - }; + source $(which env_parallel.bash); var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable echo 'compared to parallel - no transfer non-exported var' @@ -95,7 +93,8 @@ echo '### bug #43746: --transfer and --return of multiple inputs {1} and {2}' echo '### and:' echo '### bug #44371: --trc with csh complains' cd /tmp; echo 1 > file1; echo 2 > file2; - parallel -Scsh@lo --trc {1}.a --trc {2}.b 'echo A {1} > {1}.a; echo B {2} > {2}.b' ::: file1 ::: file2; + parallel -Scsh@lo --transferfile {1} --transferfile {2} --trc {1}.a --trc {2}.b + '(cat {1}; echo A {1}) > {1}.a; (cat {2};echo B {2}) > {2}.b' ::: file1 ::: file2; cat file1.a file2.b; rm /tmp/file1 /tmp/file2 /tmp/file1.a /tmp/file2.b diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index 3e852043..93b03601 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -42,6 +42,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ s/^[123] [abc] [ABC]$/123 abc ABC/g; # Remote script s/(PARALLEL_PID\D+)\d+/${1}000000/g; + # sql timing + s/,:,\d+.\d+,\d+.\d+/,:,000000000.000,0.000/g; # /usr/bin/time -f %e s/^(\d+)\.\d+$/$1/; # Base 64 string diff --git a/testsuite/wanted-results/parallel-install b/testsuite/wanted-results/parallel-install index 75aeb5ed..e8f4cc0a 100644 --- a/testsuite/wanted-results/parallel-install +++ b/testsuite/wanted-results/parallel-install @@ -155,7 +155,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P Warning: pod2pdf not found. Using old parallel_design.pdf make[0]: Entering directory `/tmp/parallel-00000000/src' /bin/mkdir -p '/usr/local/bin' - /usr/bin/install -c parallel sql niceload '/usr/local/bin' + /usr/bin/install -c parallel sql niceload env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh '/usr/local/bin' make install-exec-hook make[0]: Entering directory `/tmp/parallel-00000000/src' rm /usr/local/bin/sem || true diff --git a/testsuite/wanted-results/parallel-local-sql b/testsuite/wanted-results/parallel-local-sql index 17808bf5..73951e87 100644 --- a/testsuite/wanted-results/parallel-local-sql +++ b/testsuite/wanted-results/parallel-local-sql @@ -3,86 +3,86 @@ echo '### --sqlandworker mysql' (sleep 2; parallel --sqlworker $MYSQLTBL sleep .3\;echo >$T1) & parallel --sqlandworker $MYSQLTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; true sort -u $T1 $T2; sql $MYSQL 'select * from parsql order by seq;' Seq Host Starttime JobRuntime Send Receive Exitval _Signal Command V1 V2 Stdout Stderr -1 :999999NULL 4 0 0 sleep .3;echo 1 a 1 a 1 a\n -2 :999999NULL 4 0 0 sleep .3;echo 1 b 1 b 1 b\n -3 :999999NULL 4 0 0 sleep .3;echo 1 c 1 c 1 c\n -4 :999999NULL 4 0 0 sleep .3;echo 1 d 1 d 1 d\n -5 :999999NULL 4 0 0 sleep .3;echo 1 e 1 e 1 e\n -6 :999999NULL 4 0 0 sleep .3;echo 2 a 2 a 2 a\n -7 :999999NULL 4 0 0 sleep .3;echo 2 b 2 b 2 b\n -8 :999999NULL 4 0 0 sleep .3;echo 2 c 2 c 2 c\n -9 :999999NULL 4 0 0 sleep .3;echo 2 d 2 d 2 d\n -999:999999NULL 4 0 0 sleep .3;echo 2 e 2 e 2 e\n -999:999999NULL 4 0 0 sleep .3;echo 3 a 3 a 3 a\n -999:999999NULL 4 0 0 sleep .3;echo 3 b 3 b 3 b\n -999:999999NULL 4 0 0 sleep .3;echo 3 c 3 c 3 c\n -999:999999NULL 4 0 0 sleep .3;echo 3 d 3 d 3 d\n -999:999999NULL 4 0 0 sleep .3;echo 3 e 3 e 3 e\n -999:999999NULL 4 0 0 sleep .3;echo 4 a 4 a 4 a\n -999:999999NULL 4 0 0 sleep .3;echo 4 b 4 b 4 b\n -999:999999NULL 4 0 0 sleep .3;echo 4 c 4 c 4 c\n -999:999999NULL 4 0 0 sleep .3;echo 4 d 4 d 4 d\n -999:999999NULL 4 0 0 sleep .3;echo 4 e 4 e 4 e\n -999:999999NULL 4 0 0 sleep .3;echo 5 a 5 a 5 a\n -999:999999NULL 4 0 0 sleep .3;echo 5 b 5 b 5 b\n -999:999999NULL 4 0 0 sleep .3;echo 5 c 5 c 5 c\n -999:999999NULL 4 0 0 sleep .3;echo 5 d 5 d 5 d\n -999:999999NULL 4 0 0 sleep .3;echo 5 e 5 e 5 e\n +1 :9999990 4 0 0 sleep .3;echo 1 a 1 a 1 a\n +2 :9999990 4 0 0 sleep .3;echo 1 b 1 b 1 b\n +3 :9999990 4 0 0 sleep .3;echo 1 c 1 c 1 c\n +4 :9999990 4 0 0 sleep .3;echo 1 d 1 d 1 d\n +5 :9999990 4 0 0 sleep .3;echo 1 e 1 e 1 e\n +6 :9999990 4 0 0 sleep .3;echo 2 a 2 a 2 a\n +7 :9999990 4 0 0 sleep .3;echo 2 b 2 b 2 b\n +8 :9999990 4 0 0 sleep .3;echo 2 c 2 c 2 c\n +9 :9999990 4 0 0 sleep .3;echo 2 d 2 d 2 d\n +999:9999990 4 0 0 sleep .3;echo 2 e 2 e 2 e\n +999:9999990 4 0 0 sleep .3;echo 3 a 3 a 3 a\n +999:9999990 4 0 0 sleep .3;echo 3 b 3 b 3 b\n +999:9999990 4 0 0 sleep .3;echo 3 c 3 c 3 c\n +999:9999990 4 0 0 sleep .3;echo 3 d 3 d 3 d\n +999:9999990 4 0 0 sleep .3;echo 3 e 3 e 3 e\n +999:9999990 4 0 0 sleep .3;echo 4 a 4 a 4 a\n +999:9999990 4 0 0 sleep .3;echo 4 b 4 b 4 b\n +999:9999990 4 0 0 sleep .3;echo 4 c 4 c 4 c\n +999:9999990 4 0 0 sleep .3;echo 4 d 4 d 4 d\n +999:9999990 4 0 0 sleep .3;echo 4 e 4 e 4 e\n +999:9999990 4 0 0 sleep .3;echo 5 a 5 a 5 a\n +999:9999990 4 0 0 sleep .3;echo 5 b 5 b 5 b\n +999:9999990 4 0 0 sleep .3;echo 5 c 5 c 5 c\n +999:9999990 4 0 0 sleep .3;echo 5 d 5 d 5 d\n +999:9999990 4 0 0 sleep .3;echo 5 e 5 e 5 e\n echo '### --sqlandworker postgresql' ### --sqlandworker postgresql (sleep 2; parallel --sqlworker $PGTBL sleep .3\;echo >$T3) & parallel --sqlandworker $PGTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T4; true sort -u $T3 $T4; sql $PG 'select * from parsql order by seq;' seq | host | starttime | jobruntime | send | receive | exitval | _signal | command | v1 | v2 | stdout | stderr -----+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+-------- - 1 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | 1 a +| + 1 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | 1 a +| | | | | | | | | | | | | - 2 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | 1 b +| + 2 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | 1 b +| | | | | | | | | | | | | - 3 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | 1 c +| + 3 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | 1 c +| | | | | | | | | | | | | - 4 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | 1 d +| + 4 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | 1 d +| | | | | | | | | | | | | - 5 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | 1 e +| + 5 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | 1 e +| | | | | | | | | | | | | - 6 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | 2 a +| + 6 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | 2 a +| | | | | | | | | | | | | - 7 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | 2 b +| + 7 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | 2 b +| | | | | | | | | | | | | - 8 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | 2 c +| + 8 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | 2 c +| | | | | | | | | | | | | - 9 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | 2 d +| + 9 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | 2 d +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | 2 e +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | 2 e +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | 3 a +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | 3 a +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | 3 b +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | 3 b +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | 3 c +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | 3 c +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | 3 d +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | 3 d +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | 3 e +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | 3 e +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | 4 a +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | 4 a +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | 4 b +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | 4 b +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | 4 c +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | 4 c +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | 4 d +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | 4 d +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | 4 e +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | 4 e +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | 5 a +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | 5 a +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | 5 b +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | 5 b +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | 5 c +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | 5 c +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | 5 d +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | 5 d +| | | | | | | | | | | | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | 5 e +| +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | 5 e +| | | | | | | | | | | | | (999rows) @@ -91,55 +91,55 @@ echo '### --sqlandworker sqlite' (sleep 2; parallel --sqlworker $SQLITETBL sleep .3\;echo >$T5) & parallel --sqlandworker $SQLITETBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T6; true sort -u $T5 $T6; sql $SQLITE 'select * from parsql order by seq;' Seq|Host|Starttime|JobRuntime|Send|Receive|Exitval|_Signal|Command|V1|V2|Stdout|Stderr -1|:|999|999||4|0|0|sleep .3;echo 1 a|1|a|1 a +1|:|999|999|0|4|0|0|sleep .3;echo 1 a|1|a|1 a | -2|:|999|999||4|0|0|sleep .3;echo 1 b|1|b|1 b +2|:|999|999|0|4|0|0|sleep .3;echo 1 b|1|b|1 b | -3|:|999|999||4|0|0|sleep .3;echo 1 c|1|c|1 c +3|:|999|999|0|4|0|0|sleep .3;echo 1 c|1|c|1 c | -4|:|999|999||4|0|0|sleep .3;echo 1 d|1|d|1 d +4|:|999|999|0|4|0|0|sleep .3;echo 1 d|1|d|1 d | -5|:|999|999||4|0|0|sleep .3;echo 1 e|1|e|1 e +5|:|999|999|0|4|0|0|sleep .3;echo 1 e|1|e|1 e | -6|:|999|999||4|0|0|sleep .3;echo 2 a|2|a|2 a +6|:|999|999|0|4|0|0|sleep .3;echo 2 a|2|a|2 a | -7|:|999|999||4|0|0|sleep .3;echo 2 b|2|b|2 b +7|:|999|999|0|4|0|0|sleep .3;echo 2 b|2|b|2 b | -8|:|999|999||4|0|0|sleep .3;echo 2 c|2|c|2 c +8|:|999|999|0|4|0|0|sleep .3;echo 2 c|2|c|2 c | -9|:|999|999||4|0|0|sleep .3;echo 2 d|2|d|2 d +9|:|999|999|0|4|0|0|sleep .3;echo 2 d|2|d|2 d | -999|:|999|999||4|0|0|sleep .3;echo 2 e|2|e|2 e +999|:|999|999|0|4|0|0|sleep .3;echo 2 e|2|e|2 e | -999|:|999|999||4|0|0|sleep .3;echo 3 a|3|a|3 a +999|:|999|999|0|4|0|0|sleep .3;echo 3 a|3|a|3 a | -999|:|999|999||4|0|0|sleep .3;echo 3 b|3|b|3 b +999|:|999|999|0|4|0|0|sleep .3;echo 3 b|3|b|3 b | -999|:|999|999||4|0|0|sleep .3;echo 3 c|3|c|3 c +999|:|999|999|0|4|0|0|sleep .3;echo 3 c|3|c|3 c | -999|:|999|999||4|0|0|sleep .3;echo 3 d|3|d|3 d +999|:|999|999|0|4|0|0|sleep .3;echo 3 d|3|d|3 d | -999|:|999|999||4|0|0|sleep .3;echo 3 e|3|e|3 e +999|:|999|999|0|4|0|0|sleep .3;echo 3 e|3|e|3 e | -999|:|999|999||4|0|0|sleep .3;echo 4 a|4|a|4 a +999|:|999|999|0|4|0|0|sleep .3;echo 4 a|4|a|4 a | -999|:|999|999||4|0|0|sleep .3;echo 4 b|4|b|4 b +999|:|999|999|0|4|0|0|sleep .3;echo 4 b|4|b|4 b | -999|:|999|999||4|0|0|sleep .3;echo 4 c|4|c|4 c +999|:|999|999|0|4|0|0|sleep .3;echo 4 c|4|c|4 c | -999|:|999|999||4|0|0|sleep .3;echo 4 d|4|d|4 d +999|:|999|999|0|4|0|0|sleep .3;echo 4 d|4|d|4 d | -999|:|999|999||4|0|0|sleep .3;echo 4 e|4|e|4 e +999|:|999|999|0|4|0|0|sleep .3;echo 4 e|4|e|4 e | -999|:|999|999||4|0|0|sleep .3;echo 5 a|5|a|5 a +999|:|999|999|0|4|0|0|sleep .3;echo 5 a|5|a|5 a | -999|:|999|999||4|0|0|sleep .3;echo 5 b|5|b|5 b +999|:|999|999|0|4|0|0|sleep .3;echo 5 b|5|b|5 b | -999|:|999|999||4|0|0|sleep .3;echo 5 c|5|c|5 c +999|:|999|999|0|4|0|0|sleep .3;echo 5 c|5|c|5 c | -999|:|999|999||4|0|0|sleep .3;echo 5 d|5|d|5 d +999|:|999|999|0|4|0|0|sleep .3;echo 5 d|5|d|5 d | -999|:|999|999||4|0|0|sleep .3;echo 5 e|5|e|5 e +999|:|999|999|0|4|0|0|sleep .3;echo 5 e|5|e|5 e | echo '### --sqlandworker postgresql -S lo' ### --sqlandworker postgresql -S lo @@ -147,55 +147,55 @@ echo '### --sqlandworker postgresql -S lo' parallel -S lo --sqlandworker $PGTBL2 sleep .3\;echo ::: {1..5} ::: {a..e} >$T8; true sort -u $T7 $T8; sql $PG 'select * from parsql2 order by seq;' seq | host | starttime | jobruntime | send | receive | exitval | _signal | command | v1 | v2 | stdout | stderr -----+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+-------- - 1 | lo |999|999| | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | 1 a +| + 1 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | 1 a +| | | | | | | | | | | | | - 2 | lo |999|999| | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | 1 b +| + 2 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | 1 b +| | | | | | | | | | | | | - 3 | lo |999|999| | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | 1 c +| + 3 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | 1 c +| | | | | | | | | | | | | - 4 | lo |999|999| | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | 1 d +| + 4 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | 1 d +| | | | | | | | | | | | | - 5 | lo |999|999| | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | 1 e +| + 5 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | 1 e +| | | | | | | | | | | | | - 6 | lo |999|999| | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | 2 a +| + 6 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | 2 a +| | | | | | | | | | | | | - 7 | lo |999|999| | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | 2 b +| + 7 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | 2 b +| | | | | | | | | | | | | - 8 | lo |999|999| | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | 2 c +| + 8 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | 2 c +| | | | | | | | | | | | | - 9 | lo |999|999| | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | 2 d +| + 9 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | 2 d +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | 2 e +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | 2 e +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | 3 a +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | 3 a +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | 3 b +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | 3 b +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | 3 c +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | 3 c +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | 3 d +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | 3 d +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | 3 e +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | 3 e +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | 4 a +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | 4 a +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | 4 b +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | 4 b +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | 4 c +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | 4 c +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | 4 d +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | 4 d +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | 4 e +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | 4 e +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | 5 a +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | 5 a +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | 5 b +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | 5 b +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | 5 c +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | 5 c +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | 5 d +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | 5 d +| | | | | | | | | | | | | -999| lo |999|999| | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | 5 e +| +999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | 5 e +| | | | | | | | | | | | | (999rows) @@ -206,31 +206,31 @@ echo '### --sqlandworker postgresql --results' parallel --results /tmp/out--sql --sqlandworker $PGTBL3 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999; true sort -u $T9 $T999; sql $PG 'select * from parsql3 order by seq;' seq | host | starttime | jobruntime | send | receive | exitval | _signal | command | v1 | v2 | stdout | stderr -----+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+------------------------------+------------------------------ - 1 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | /tmp/out--sql/1/1/2/a/stdout | /tmp/out--sql/1/1/2/a/stderr - 2 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | /tmp/out--sql/1/1/2/b/stdout | /tmp/out--sql/1/1/2/b/stderr - 3 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | /tmp/out--sql/1/1/2/c/stdout | /tmp/out--sql/1/1/2/c/stderr - 4 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | /tmp/out--sql/1/1/2/d/stdout | /tmp/out--sql/1/1/2/d/stderr - 5 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | /tmp/out--sql/1/1/2/e/stdout | /tmp/out--sql/1/1/2/e/stderr - 6 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | /tmp/out--sql/1/2/2/a/stdout | /tmp/out--sql/1/2/2/a/stderr - 7 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | /tmp/out--sql/1/2/2/b/stdout | /tmp/out--sql/1/2/2/b/stderr - 8 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | /tmp/out--sql/1/2/2/c/stdout | /tmp/out--sql/1/2/2/c/stderr - 9 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | /tmp/out--sql/1/2/2/d/stdout | /tmp/out--sql/1/2/2/d/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | /tmp/out--sql/1/2/2/e/stdout | /tmp/out--sql/1/2/2/e/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | /tmp/out--sql/1/3/2/a/stdout | /tmp/out--sql/1/3/2/a/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | /tmp/out--sql/1/3/2/b/stdout | /tmp/out--sql/1/3/2/b/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | /tmp/out--sql/1/3/2/c/stdout | /tmp/out--sql/1/3/2/c/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | /tmp/out--sql/1/3/2/d/stdout | /tmp/out--sql/1/3/2/d/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | /tmp/out--sql/1/3/2/e/stdout | /tmp/out--sql/1/3/2/e/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | /tmp/out--sql/1/4/2/a/stdout | /tmp/out--sql/1/4/2/a/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | /tmp/out--sql/1/4/2/b/stdout | /tmp/out--sql/1/4/2/b/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | /tmp/out--sql/1/4/2/c/stdout | /tmp/out--sql/1/4/2/c/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | /tmp/out--sql/1/4/2/d/stdout | /tmp/out--sql/1/4/2/d/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | /tmp/out--sql/1/4/2/e/stdout | /tmp/out--sql/1/4/2/e/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | /tmp/out--sql/1/5/2/a/stdout | /tmp/out--sql/1/5/2/a/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | /tmp/out--sql/1/5/2/b/stdout | /tmp/out--sql/1/5/2/b/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | /tmp/out--sql/1/5/2/c/stdout | /tmp/out--sql/1/5/2/c/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | /tmp/out--sql/1/5/2/d/stdout | /tmp/out--sql/1/5/2/d/stderr -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | /tmp/out--sql/1/5/2/e/stdout | /tmp/out--sql/1/5/2/e/stderr + 1 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | /tmp/out--sql/1/1/2/a/stdout | /tmp/out--sql/1/1/2/a/stderr + 2 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | /tmp/out--sql/1/1/2/b/stdout | /tmp/out--sql/1/1/2/b/stderr + 3 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | /tmp/out--sql/1/1/2/c/stdout | /tmp/out--sql/1/1/2/c/stderr + 4 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | /tmp/out--sql/1/1/2/d/stdout | /tmp/out--sql/1/1/2/d/stderr + 5 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | /tmp/out--sql/1/1/2/e/stdout | /tmp/out--sql/1/1/2/e/stderr + 6 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | /tmp/out--sql/1/2/2/a/stdout | /tmp/out--sql/1/2/2/a/stderr + 7 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | /tmp/out--sql/1/2/2/b/stdout | /tmp/out--sql/1/2/2/b/stderr + 8 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | /tmp/out--sql/1/2/2/c/stdout | /tmp/out--sql/1/2/2/c/stderr + 9 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | /tmp/out--sql/1/2/2/d/stdout | /tmp/out--sql/1/2/2/d/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | /tmp/out--sql/1/2/2/e/stdout | /tmp/out--sql/1/2/2/e/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | /tmp/out--sql/1/3/2/a/stdout | /tmp/out--sql/1/3/2/a/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | /tmp/out--sql/1/3/2/b/stdout | /tmp/out--sql/1/3/2/b/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | /tmp/out--sql/1/3/2/c/stdout | /tmp/out--sql/1/3/2/c/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | /tmp/out--sql/1/3/2/d/stdout | /tmp/out--sql/1/3/2/d/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | /tmp/out--sql/1/3/2/e/stdout | /tmp/out--sql/1/3/2/e/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | /tmp/out--sql/1/4/2/a/stdout | /tmp/out--sql/1/4/2/a/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | /tmp/out--sql/1/4/2/b/stdout | /tmp/out--sql/1/4/2/b/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | /tmp/out--sql/1/4/2/c/stdout | /tmp/out--sql/1/4/2/c/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | /tmp/out--sql/1/4/2/d/stdout | /tmp/out--sql/1/4/2/d/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | /tmp/out--sql/1/4/2/e/stdout | /tmp/out--sql/1/4/2/e/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | /tmp/out--sql/1/5/2/a/stdout | /tmp/out--sql/1/5/2/a/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | /tmp/out--sql/1/5/2/b/stdout | /tmp/out--sql/1/5/2/b/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | /tmp/out--sql/1/5/2/c/stdout | /tmp/out--sql/1/5/2/c/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | /tmp/out--sql/1/5/2/d/stdout | /tmp/out--sql/1/5/2/d/stderr +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | /tmp/out--sql/1/5/2/e/stdout | /tmp/out--sql/1/5/2/e/stderr (999rows) echo '### --sqlandworker postgresql --linebuffer' @@ -239,31 +239,31 @@ echo '### --sqlandworker postgresql --linebuffer' parallel --linebuffer --sqlandworker $PGTBL4 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999; true sort -u $T999$T999; sql $PG 'select * from parsql4 order by seq;' seq | host | starttime | jobruntime | send | receive | exitval | _signal | command | v1 | v2 | stdout | stderr -----+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+-------- - 1 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | | - 2 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | | - 3 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | | - 4 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | | - 5 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | | - 6 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | | - 7 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | | - 8 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | | - 9 | : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | | -999| : |999|999| | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | | + 1 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | | + 2 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | | + 3 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | | + 4 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | | + 5 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | | + 6 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | | + 7 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | | + 8 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | | + 9 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | | +999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | | (999rows) echo '### --sqlandworker postgresql -u' @@ -272,30 +272,30 @@ echo '### --sqlandworker postgresql -u' parallel -u --sqlandworker $PGTBL5 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999; true sort -u $T999$T999; sql $PG 'select * from parsql5 order by seq;' seq | host | starttime | jobruntime | send | receive | exitval | _signal | command | v1 | v2 | stdout | stderr -----+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+-------- - 1 | : |999|999| | | 0 | 0 | sleep .3;echo 1 a | 1 | a | | - 2 | : |999|999| | | 0 | 0 | sleep .3;echo 1 b | 1 | b | | - 3 | : |999|999| | | 0 | 0 | sleep .3;echo 1 c | 1 | c | | - 4 | : |999|999| | | 0 | 0 | sleep .3;echo 1 d | 1 | d | | - 5 | : |999|999| | | 0 | 0 | sleep .3;echo 1 e | 1 | e | | - 6 | : |999|999| | | 0 | 0 | sleep .3;echo 2 a | 2 | a | | - 7 | : |999|999| | | 0 | 0 | sleep .3;echo 2 b | 2 | b | | - 8 | : |999|999| | | 0 | 0 | sleep .3;echo 2 c | 2 | c | | - 9 | : |999|999| | | 0 | 0 | sleep .3;echo 2 d | 2 | d | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 2 e | 2 | e | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 3 a | 3 | a | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 3 b | 3 | b | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 3 c | 3 | c | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 3 d | 3 | d | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 3 e | 3 | e | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 4 a | 4 | a | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 4 b | 4 | b | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 4 c | 4 | c | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 4 d | 4 | d | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 4 e | 4 | e | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 5 a | 5 | a | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 5 b | 5 | b | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 5 c | 5 | c | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 5 d | 5 | d | | -999| : |999|999| | | 0 | 0 | sleep .3;echo 5 e | 5 | e | | + 1 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 1 a | 1 | a | | + 2 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 1 b | 1 | b | | + 3 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 1 c | 1 | c | | + 4 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 1 d | 1 | d | | + 5 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 1 e | 1 | e | | + 6 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 2 a | 2 | a | | + 7 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 2 b | 2 | b | | + 8 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 2 c | 2 | c | | + 9 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 2 d | 2 | d | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 2 e | 2 | e | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 3 a | 3 | a | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 3 b | 3 | b | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 3 c | 3 | c | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 3 d | 3 | d | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 3 e | 3 | e | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 4 a | 4 | a | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 4 b | 4 | b | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 4 c | 4 | c | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 4 d | 4 | d | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 4 e | 4 | e | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 5 a | 5 | a | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 5 b | 5 | b | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 5 c | 5 | c | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 5 d | 5 | d | | +999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 5 e | 5 | e | | (999rows) diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 71245d90..956be59a 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -147,8 +147,17 @@ ls: No match. ls: cannot access ./?--.a: No such file or directory ls: No match. ls: cannot access ./--?.a: No such file or directory -echo '### bug #46519: --onall ignores --transfer' +echo '### Test --trc "/tmp/./--- /A" "/tmp/./ ---/B"' +### Test --trc "/tmp/./--- /A" "/tmp/./ ---/B" + mkdir -p '/tmp/./--- ' '/tmp/ ---'; touch -- '/tmp/./--- /A' '/tmp/ ---/B'; rm -f ./---?/A.a ./?---/B.a; parallel --trc {=s:.*/./::=}.a -S csh@lo,sh@lo touch ./{=s:.*/./::=}.a ::: '/tmp/./--- /A' '/tmp/./ ---/B'; ls ./---?/A.a ./?---/B.a; parallel --nonall -k -S csh@lo,sh@lo 'ls ./?--- ./---? || echo OK'; echo '### bug #46519: --onall ignores --transfer' +./ ---/B.a +./--- /A.a +OK +OK ### bug #46519: --onall ignores --transfer +ls: No match. +ls: cannot access ./?---: No such file or directory +ls: cannot access ./---?: No such file or directory touch bug46519.{a,b,c}; rm -f bug46519.?? bug46519.???; parallel --onall --tf bug46519.{} --trc bug46519.{}{} --trc bug46519.{}{}{} -S csh@lo,sh@lo 'ls bug46519.{}; touch bug46519.{}{} bug46519.{}{}{}' ::: a b c; ls bug46519.?? bug46519.???; parallel --onall -S csh@lo,sh@lo ls bug46519.{}{} bug46519.{}{}{} ::: a b c && echo Cleanup failed bug46519.a bug46519.b diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index 31c6e705..d432b242 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -17,7 +17,7 @@ no_warning warning echo 'env_parallel from man page - transfer non-exported var' env_parallel from man page - transfer non-exported var - env_parallel() { export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"; `which parallel` "$@"; unset parallel_bash_environment; }; var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable + source $(which env_parallel.bash); var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable nonexported variable echo 'compared to parallel - no transfer non-exported var' compared to parallel - no transfer non-exported var @@ -92,8 +92,10 @@ echo '### and:' ### and: echo '### bug #44371: --trc with csh complains' ### bug #44371: --trc with csh complains - cd /tmp; echo 1 > file1; echo 2 > file2; parallel -Scsh@lo --trc {1}.a --trc {2}.b 'echo A {1} > {1}.a; echo B {2} > {2}.b' ::: file1 ::: file2; cat file1.a file2.b; rm /tmp/file1 /tmp/file2 /tmp/file1.a /tmp/file2.b + cd /tmp; echo 1 > file1; echo 2 > file2; parallel -Scsh@lo --transferfile {1} --transferfile {2} --trc {1}.a --trc {2}.b '(cat {1}; echo A {1}) > {1}.a; (cat {2};echo B {2}) > {2}.b' ::: file1 ::: file2; cat file1.a file2.b; rm /tmp/file1 /tmp/file2 /tmp/file1.a /tmp/file2.b +1 A file1 +2 B file2 echo '### bug #44143: csh and nice' ### bug #44143: csh and nice diff --git a/testsuite/wanted-results/parallel-local150 b/testsuite/wanted-results/parallel-local150 index a91f6b36..4ead42ae 100644 --- a/testsuite/wanted-results/parallel-local150 +++ b/testsuite/wanted-results/parallel-local150 @@ -51,6 +51,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --sql has been retired. Use --sqlmaster. parallel: Error: --ctrlc has been retired. parallel: Error: --noctrlc has been retired. stdout parallel -g @@ -61,6 +62,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --sql has been retired. Use --sqlmaster. parallel: Error: --ctrlc has been retired. parallel: Error: --noctrlc has been retired. stdout parallel -H 1 @@ -71,6 +73,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --sql has been retired. Use --sqlmaster. parallel: Error: --ctrlc has been retired. parallel: Error: --noctrlc has been retired. stdout parallel -T @@ -81,6 +84,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --sql has been retired. Use --sqlmaster. parallel: Error: --ctrlc has been retired. parallel: Error: --noctrlc has been retired. stdout parallel -U foo @@ -91,6 +95,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --sql has been retired. Use --sqlmaster. parallel: Error: --ctrlc has been retired. parallel: Error: --noctrlc has been retired. stdout parallel -W foo @@ -101,6 +106,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --sql has been retired. Use --sqlmaster. parallel: Error: --ctrlc has been retired. parallel: Error: --noctrlc has been retired. stdout parallel -Y @@ -111,6 +117,7 @@ parallel: Error: -U has been retired. Use --er. parallel: Error: -W has been retired. Use --wd. parallel: Error: -Y has been retired. Use --shebang. parallel: Error: -H has been retired. Use --halt. +parallel: Error: --sql has been retired. Use --sqlmaster. parallel: Error: --ctrlc has been retired. parallel: Error: --noctrlc has been retired. echo '### Test --joblog followed by --resume --joblog' diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 8bfa0a39..1f4fd703 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -52,7 +52,7 @@ B F C D C E C F - cat abc-file | parallel -a - -a def-file echo + cat abc-file | parallel -a - -a def-file echo A D A E A F @@ -523,9 +523,9 @@ Computers / CPU cores / Max jobs to run Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --timeout=12 --progress --auto-kill) -BASE64 parallel --joblog /tmp/log exit ::: 1 2 3 0 +BASE64 parallel --joblog /tmp/log exit ::: 1 2 3 0 cat /tmp/log; - parallel --joblog /tmp/log exit ::: 1 2 3 0 + parallel --joblog /tmp/log exit ::: 1 2 3 0 cat /tmp/log; parallel --resume --joblog /tmp/log exit ::: 1 2 3 0 0 0 cat /tmp/log; parallel --resume-failed --joblog /tmp/log exit ::: 1 2 3 0 0 0 @@ -614,13 +614,13 @@ tried 0 tried 1 tried 1 show_signals() { - perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print \"Got $_\\n\"; }";} while(1){sleep 1}' + perl -e 'for(keys %SIG) { $SIG{$_} = eval "sub { print \"Got $_\\n\"; }";} while(1){sleep 1}' } export -f show_signals echo | parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 -u --timeout 1 show_signals echo | parallel --termseq INT,200,TERM,100,KILL,25 -u --timeout 1 show_signals /bin/bash: show_signals: command not found - parallel --load 100% echo load is less than {} job per cpu ::: 1 + parallel --load 100% echo load is less than {} job per cpu ::: 1 load is less than 1 job per cpu parallel --noswap echo the system is not swapping ::: now the system is not swapping now @@ -661,18 +661,18 @@ force 4 cpus on server run_on_grp1 run_on_grp2 echo This is input_file > input_file - parallel -S $SERVER1 --transferfile {} cat ::: input_file + parallel -S $SERVER1 --transferfile {} cat ::: input_file This is input_file echo This is input_file > input_file - parallel -S $SERVER1 --transferfile {} --return {}.out cat {} ">"{}.out ::: input_file + parallel -S $SERVER1 --transferfile {} --return {}.out cat {} ">"{}.out ::: input_file cat input_file.out This is input_file echo This is input_file > input_file - parallel -S $SERVER1 --transferfile {} --return {}.out --cleanup cat {} ">"{}.out ::: input_file + parallel -S $SERVER1 --transferfile {} --return {}.out --cleanup cat {} ">"{}.out ::: input_file cat input_file.out This is input_file echo This is input_file > input_file - parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file + parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file cat input_file.out This is input_file echo common data > common_file @@ -693,7 +693,7 @@ foo 1 2 3 - parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar + parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar bar parallel: Warning: Removed 173.194.32.46. parallel --onall -S $SERVER1,$SERVER2 echo ::: foo bar @@ -816,18 +816,50 @@ _ /bin/bash: my_func2: command not found parallel -vv -S $SERVER1 echo ::: bar -ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";my$eval;$eval=decode_base64(join"",@ARGV);eval$eval;'\' BASE64; +ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\' BASE64; bar my_func3() { echo in my_func $1 > $1.out } export -f my_func3 parallel -vv --workdir ... --nice 17 --env _ --trc {}.out -S $SERVER1 my_func3 {} ::: abc-file -( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";my$eval;$eval=decode_base64(join"",@ARGV);eval$eval;'\' BASE64 BASE64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status; +( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\' BASE64 BASE64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status; + parallel --sqlandworker csv:////%2Ftmp%2Flog.csv seq ::: 10 ::: 12 13 14 + cat /tmp/log.csv +10 +11 +12 +10 +11 +12 +13 +10 +11 +12 +13 +14 +Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr +1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,"10 +11 +12 +", +2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,"10 +11 +12 +13 +", +3,:,000000000.000,0.000,0,15,0,0,"seq 10 14",10,14,"10 +11 +12 +13 +14 +", vendor://[[user][:password]@][host][:port]/[database[/table] /bin/bash: vendor://[[user][:password]@][host][:port]/[database[/table]: No such file or directory sqlite3:///%2Ftmp%2Fmydatabase/mytable + csv:////%2Ftmp%2Flog.csv /bin/bash: sqlite3:///%2Ftmp%2Fmydatabase/mytable: No such file or directory +/bin/bash: line 1: csv:////%2Ftmp%2Flog.csv: No such file or directory DBURL=sqlite3:///%2Ftmp%2Fmydatabase DBURLTABLE=$DBURL/mytable parallel --sqlandworker $DBURLTABLE echo ::: foo bar ::: baz quuz @@ -840,7 +872,7 @@ Error: SELECT * FROM mytable ORDER BY Seq; is not a valid DBURL sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command] - parallel --sql $DBURLTABLE echo ::: foo bar ::: baz quuz + parallel --sqlmaster $DBURLTABLE echo ::: foo bar ::: baz quuz parallel: Error: echo is not a valid DBURL parallel --sqlworker $DBURLTABLE echo ::: foo bar ::: baz quuz parallel: Error: echo is not a valid DBURL @@ -1093,7 +1125,6 @@ Arguments var argv = Environment.GetEnvironmentVariable("ARGV"); print("Arguments "+argv); - /bin/bash: -c: line 2: syntax error near unexpected token `(' /bin/bash: -c: line 2: ` var argv = Environment.GetEnvironmentVariable("ARGV");' sem 'sleep 1; echo The first finished' && @@ -1206,7 +1237,7 @@ mentioned in the release notes of next version of GNU Parallel. parallel --max-line-length-allowed 131071 - parallel --number-of-cpus + parallel --number-of-cpus parallel --number-of-cores 1 8