mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 22:17:54 +00:00
parallel: --record_env and --env _ implemented. Passes test suite.
This commit is contained in:
parent
df204a69fa
commit
ee529c2d2a
|
@ -207,11 +207,14 @@ New in this release:
|
||||||
|
|
||||||
* New signing key. Due to recommendations from NIST
|
* New signing key. Due to recommendations from NIST
|
||||||
http://www.keylength.com/en/4/ the signing key was changed from
|
http://www.keylength.com/en/4/ the signing key was changed from
|
||||||
1024D/4000g/ID:FFFFFFF1 to 9888R/ID:88888888.
|
1024D/ID:FFFFFFF1 to 9888R/ID:88888888.
|
||||||
|
|
||||||
* Agalma: an automated phylogenomics workflow
|
* Agalma: an automated phylogenomics workflow
|
||||||
http://arxiv.org/pdf/1307.6432
|
http://arxiv.org/pdf/1307.6432
|
||||||
|
|
||||||
|
* Transient Beowulf Clustering with GNU Parallel and SSHfs
|
||||||
|
http://www.reddit.com/r/linux/comments/1ka8mn/transient_beowulf_clustering_with_gnu_parallel/
|
||||||
|
|
||||||
* Aligning to unique regions
|
* Aligning to unique regions
|
||||||
http://davetang.org/muse/2013/07/22/aligning-to-unique-regions/
|
http://davetang.org/muse/2013/07/22/aligning-to-unique-regions/
|
||||||
|
|
||||||
|
@ -224,6 +227,12 @@ New in this release:
|
||||||
* Compression of files in parallel using GNU parallel
|
* Compression of files in parallel using GNU parallel
|
||||||
http://codextechnicanum.blogspot.dk/2013/07/compression-of-files-in-parallel-using.html
|
http://codextechnicanum.blogspot.dk/2013/07/compression-of-files-in-parallel-using.html
|
||||||
|
|
||||||
|
* TimeMachineっぽいバックアップスクリプト
|
||||||
|
http://rio.tc/2013/07/timemachine-1.html
|
||||||
|
|
||||||
|
* Some useful comments on GNU Parallel
|
||||||
|
https://news.ycombinator.com/item?id=6209767
|
||||||
|
|
||||||
* Bug fixes and man page updates.
|
* Bug fixes and man page updates.
|
||||||
|
|
||||||
|
|
||||||
|
|
63
src/parallel
63
src/parallel
|
@ -432,7 +432,7 @@ sub spreadstdin {
|
||||||
$Global::start_no_new_jobs ||= 1;
|
$Global::start_no_new_jobs ||= 1;
|
||||||
if($opt::roundrobin) {
|
if($opt::roundrobin) {
|
||||||
for my $job (values %Global::running) {
|
for my $job (values %Global::running) {
|
||||||
my $fh = $job->stdin();
|
my $fh = $job->fd(0);
|
||||||
close $fh;
|
close $fh;
|
||||||
}
|
}
|
||||||
my %incomplete_jobs = %Global::running;
|
my %incomplete_jobs = %Global::running;
|
||||||
|
@ -608,7 +608,7 @@ sub options_hash {
|
||||||
"nice=i" => \$opt::nice,
|
"nice=i" => \$opt::nice,
|
||||||
"timeout=s" => \$opt::timeout,
|
"timeout=s" => \$opt::timeout,
|
||||||
"tag" => \$opt::tag,
|
"tag" => \$opt::tag,
|
||||||
"tagstring=s" => \$opt::tagstring,
|
"tagstring|tag-string=s" => \$opt::tagstring,
|
||||||
"onall" => \$opt::onall,
|
"onall" => \$opt::onall,
|
||||||
"nonall" => \$opt::nonall,
|
"nonall" => \$opt::nonall,
|
||||||
"filter-hosts|filterhosts|filter-host" => \$opt::filter_hosts,
|
"filter-hosts|filterhosts|filter-host" => \$opt::filter_hosts,
|
||||||
|
@ -639,6 +639,7 @@ sub options_hash {
|
||||||
"arg-file-sep|argfilesep=s" => \$opt::arg_file_sep,
|
"arg-file-sep|argfilesep=s" => \$opt::arg_file_sep,
|
||||||
"trim=s" => \$opt::trim,
|
"trim=s" => \$opt::trim,
|
||||||
"env=s" => \@opt::env,
|
"env=s" => \@opt::env,
|
||||||
|
"recordenv|record-env" => \$opt::record_env,
|
||||||
"plain" => \$opt::plain,
|
"plain" => \$opt::plain,
|
||||||
"profile|J=s" => \@opt::profile,
|
"profile|J=s" => \@opt::profile,
|
||||||
"pipe|spreadstdin" => \$opt::pipe,
|
"pipe|spreadstdin" => \$opt::pipe,
|
||||||
|
@ -727,7 +728,7 @@ sub get_options_from_array {
|
||||||
sub parse_options {
|
sub parse_options {
|
||||||
# Returns: N/A
|
# Returns: N/A
|
||||||
# Defaults:
|
# Defaults:
|
||||||
$Global::version = 20130730;
|
$Global::version = 20130814;
|
||||||
$Global::progname = 'parallel';
|
$Global::progname = 'parallel';
|
||||||
$Global::infinity = 2**31;
|
$Global::infinity = 2**31;
|
||||||
$Global::debug = 0;
|
$Global::debug = 0;
|
||||||
|
@ -756,16 +757,6 @@ sub parse_options {
|
||||||
|
|
||||||
@ARGV=read_options();
|
@ARGV=read_options();
|
||||||
|
|
||||||
if(defined $opt::retired) {
|
|
||||||
::error("-g has been retired. Use --group.\n");
|
|
||||||
::error("-B has been retired. Use --bf.\n");
|
|
||||||
::error("-T has been retired. Use --tty.\n");
|
|
||||||
::error("-U has been retired. Use --er.\n");
|
|
||||||
::error("-W has been retired. Use --wd.\n");
|
|
||||||
::error("-Y has been retired. Use --shebang.\n");
|
|
||||||
::error("-H has been retired. Use --halt.\n");
|
|
||||||
::wait_and_exit(255);
|
|
||||||
}
|
|
||||||
if(@opt::v) { $Global::verbose = $#opt::v+1; } # Convert -v -v to v=2
|
if(@opt::v) { $Global::verbose = $#opt::v+1; } # Convert -v -v to v=2
|
||||||
$Global::debug = (defined $opt::D);
|
$Global::debug = (defined $opt::D);
|
||||||
if(defined $opt::X) { $Global::ContextReplace = 1; }
|
if(defined $opt::X) { $Global::ContextReplace = 1; }
|
||||||
|
@ -811,6 +802,7 @@ sub parse_options {
|
||||||
}
|
}
|
||||||
if(defined $opt::version) { version(); wait_and_exit(0); }
|
if(defined $opt::version) { version(); wait_and_exit(0); }
|
||||||
if(defined $opt::bibtex) { bibtex(); wait_and_exit(0); }
|
if(defined $opt::bibtex) { bibtex(); wait_and_exit(0); }
|
||||||
|
if(defined $opt::record_env) { record_env(); wait_and_exit(0); }
|
||||||
if(defined $opt::show_limits) { show_limits(); }
|
if(defined $opt::show_limits) { show_limits(); }
|
||||||
if(@opt::sshlogin) { @Global::sshlogin = @opt::sshlogin; }
|
if(@opt::sshlogin) { @Global::sshlogin = @opt::sshlogin; }
|
||||||
if(@opt::sshloginfile) { read_sshloginfiles(@opt::sshloginfile); }
|
if(@opt::sshloginfile) { read_sshloginfiles(@opt::sshloginfile); }
|
||||||
|
@ -962,6 +954,16 @@ sub parse_options {
|
||||||
if(defined $opt::eta) {
|
if(defined $opt::eta) {
|
||||||
$opt::progress = $opt::eta;
|
$opt::progress = $opt::eta;
|
||||||
}
|
}
|
||||||
|
if(defined $opt::retired) {
|
||||||
|
::error("-g has been retired. Use --group.\n");
|
||||||
|
::error("-B has been retired. Use --bf.\n");
|
||||||
|
::error("-T has been retired. Use --tty.\n");
|
||||||
|
::error("-U has been retired. Use --er.\n");
|
||||||
|
::error("-W has been retired. Use --wd.\n");
|
||||||
|
::error("-Y has been retired. Use --shebang.\n");
|
||||||
|
::error("-H has been retired. Use --halt.\n");
|
||||||
|
::wait_and_exit(255);
|
||||||
|
}
|
||||||
|
|
||||||
parse_sshlogin();
|
parse_sshlogin();
|
||||||
parse_env_var();
|
parse_env_var();
|
||||||
|
@ -987,6 +989,17 @@ sub env_quote {
|
||||||
return $v;
|
return $v;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sub record_env {
|
||||||
|
# Record current %ENV-keys in ~/.parallel/recorded_env
|
||||||
|
# Returns: N/A
|
||||||
|
if(open(my $vars_fh, ">", $ENV{'HOME'} . "/.parallel/recorded_env")) {
|
||||||
|
print $vars_fh map { $_,"\n" } keys %ENV;
|
||||||
|
} else {
|
||||||
|
::error("Cannot write to ".$ENV{'HOME'} . "/.parallel/recorded_env\n");
|
||||||
|
::wait_and_exit(255);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
sub parse_env_var {
|
sub parse_env_var {
|
||||||
# Parse --env and set $Global::envvar
|
# Parse --env and set $Global::envvar
|
||||||
# Returns: N/A
|
# Returns: N/A
|
||||||
|
@ -996,6 +1009,21 @@ sub parse_env_var {
|
||||||
# Split up --env VAR1,VAR2
|
# Split up --env VAR1,VAR2
|
||||||
push @vars, split /,/, $varstring;
|
push @vars, split /,/, $varstring;
|
||||||
}
|
}
|
||||||
|
if(grep { /^_$/ } @vars) {
|
||||||
|
# Include all vars that are not in a clean environment
|
||||||
|
if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/recorded_env")) {
|
||||||
|
my @ignore = <$vars_fh>;
|
||||||
|
chomp @ignore;
|
||||||
|
my %ignore;
|
||||||
|
@ignore{@ignore} = @ignore;
|
||||||
|
close $vars_fh;
|
||||||
|
push @vars, grep { not defined $ignore{$_} } keys %ENV;
|
||||||
|
@vars = grep { not /^_$/ } @vars;
|
||||||
|
} else {
|
||||||
|
::error("Run '$Global::progname --record-env' in a clean environment first.\n");
|
||||||
|
::wait_and_exit(255);
|
||||||
|
}
|
||||||
|
}
|
||||||
# Keep only defined variables
|
# Keep only defined variables
|
||||||
@vars = grep { defined($ENV{$_}) } @vars;
|
@vars = grep { defined($ENV{$_}) } @vars;
|
||||||
my @qcsh = map { my $a=$_; "setenv $a " . env_quote($ENV{$a}) } @vars;
|
my @qcsh = map { my $a=$_; "setenv $a " . env_quote($ENV{$a}) } @vars;
|
||||||
|
@ -3729,13 +3757,10 @@ sub non_block_write {
|
||||||
my $self = shift;
|
my $self = shift;
|
||||||
my $something_written = 0;
|
my $something_written = 0;
|
||||||
use POSIX qw(:errno_h);
|
use POSIX qw(:errno_h);
|
||||||
use Fcntl;
|
# use Fcntl;
|
||||||
|
# my $flags = '';
|
||||||
my $flags = '';
|
|
||||||
|
|
||||||
|
|
||||||
for my $buf (substr($self->{'stdin_buffer'},$self->{'stdin_buffer_pos'})) {
|
for my $buf (substr($self->{'stdin_buffer'},$self->{'stdin_buffer_pos'})) {
|
||||||
my $in = $self->{'stdin'};
|
my $in = $self->fd(0);
|
||||||
# fcntl($in, F_GETFL, $flags)
|
# fcntl($in, F_GETFL, $flags)
|
||||||
# or die "Couldn't get flags for HANDLE : $!\n";
|
# or die "Couldn't get flags for HANDLE : $!\n";
|
||||||
# $flags |= O_NONBLOCK;
|
# $flags |= O_NONBLOCK;
|
||||||
|
|
|
@ -335,7 +335,7 @@ Run command in background thus GNU B<parallel> will not wait for
|
||||||
completion of the command before exiting. This is the default if
|
completion of the command before exiting. This is the default if
|
||||||
B<--semaphore> is set.
|
B<--semaphore> is set.
|
||||||
|
|
||||||
See also: B<--fg>, B<man sem>
|
See also: B<--fg>, B<man sem>.
|
||||||
|
|
||||||
Implies B<--semaphore>.
|
Implies B<--semaphore>.
|
||||||
|
|
||||||
|
@ -467,6 +467,11 @@ remote execution.
|
||||||
In Bash I<var> can also be a Bash function - just remember to B<export
|
In Bash I<var> can also be a Bash function - just remember to B<export
|
||||||
-f> the function.
|
-f> the function.
|
||||||
|
|
||||||
|
The variable '_' is special. It will copy all enviroment variables
|
||||||
|
except for the ones mentioned in ~/.parallel/recorded_env.
|
||||||
|
|
||||||
|
See also: B<--record-env>.
|
||||||
|
|
||||||
|
|
||||||
=item B<--eta>
|
=item B<--eta>
|
||||||
|
|
||||||
|
@ -481,7 +486,7 @@ Implies B<--progress>.
|
||||||
Run command in foreground thus GNU B<parallel> will wait for
|
Run command in foreground thus GNU B<parallel> will wait for
|
||||||
completion of the command before exiting.
|
completion of the command before exiting.
|
||||||
|
|
||||||
See also: B<--bg>, B<man sem>
|
See also B<--bg>, B<man sem>.
|
||||||
|
|
||||||
Implies B<--semaphore>.
|
Implies B<--semaphore>.
|
||||||
|
|
||||||
|
@ -838,7 +843,7 @@ reading everything at startup.
|
||||||
By sending GNU B<parallel> SIGUSR2 you can toggle turning on/off
|
By sending GNU B<parallel> SIGUSR2 you can toggle turning on/off
|
||||||
B<--progress> on a running GNU B<parallel> process.
|
B<--progress> on a running GNU B<parallel> process.
|
||||||
|
|
||||||
See also: B<--eta>
|
See also B<--eta>.
|
||||||
|
|
||||||
|
|
||||||
=item B<--max-args>=I<max-args>
|
=item B<--max-args>=I<max-args>
|
||||||
|
@ -950,6 +955,14 @@ If the stdin (standard input) only contains whitespace, do not run the command.
|
||||||
If used with B<--pipe> this is slow.
|
If used with B<--pipe> this is slow.
|
||||||
|
|
||||||
|
|
||||||
|
=item B<--record-env> (alpha testing)
|
||||||
|
|
||||||
|
Record current environment variables in ~/.parallel/recorded_env. This
|
||||||
|
is useful before using B<--env _>.
|
||||||
|
|
||||||
|
See also B<--env>.
|
||||||
|
|
||||||
|
|
||||||
=item B<--recstart> I<startstring>
|
=item B<--recstart> I<startstring>
|
||||||
|
|
||||||
=item B<--recend> I<endstring>
|
=item B<--recend> I<endstring>
|
||||||
|
@ -1042,7 +1055,7 @@ there. As GNU B<parallel> only looks at the sequence numbers in
|
||||||
B<--joblog> then the input, the command, and B<--joblog> all have to
|
B<--joblog> then the input, the command, and B<--joblog> all have to
|
||||||
remain unchanged; otherwise GNU B<parallel> may run wrong commands.
|
remain unchanged; otherwise GNU B<parallel> may run wrong commands.
|
||||||
|
|
||||||
See also: B<--joblog>, B<--resume-failed>.
|
See also B<--joblog>, B<--resume-failed>.
|
||||||
|
|
||||||
|
|
||||||
=item B<--resume-failed>
|
=item B<--resume-failed>
|
||||||
|
@ -1055,7 +1068,7 @@ numbers in B<--joblog> then the input, the command, and B<--joblog>
|
||||||
all have to remain unchanged; otherwise GNU B<parallel> may run wrong
|
all have to remain unchanged; otherwise GNU B<parallel> may run wrong
|
||||||
commands.
|
commands.
|
||||||
|
|
||||||
See also: B<--joblog>, B<--resume>.
|
See also B<--joblog>, B<--resume>.
|
||||||
|
|
||||||
|
|
||||||
=item B<--retries> I<n>
|
=item B<--retries> I<n>
|
||||||
|
@ -1151,7 +1164,7 @@ Used with B<--fg>, B<--wait>, and B<--semaphorename>.
|
||||||
|
|
||||||
The command B<sem> is an alias for B<parallel --semaphore>.
|
The command B<sem> is an alias for B<parallel --semaphore>.
|
||||||
|
|
||||||
See also: B<man sem>
|
See also B<man sem>.
|
||||||
|
|
||||||
|
|
||||||
=item B<--semaphorename> I<name>
|
=item B<--semaphorename> I<name>
|
||||||
|
@ -1169,7 +1182,7 @@ The semaphore is stored in ~/.parallel/semaphores/
|
||||||
|
|
||||||
Implies B<--semaphore>.
|
Implies B<--semaphore>.
|
||||||
|
|
||||||
See also: B<man sem>
|
See also B<man sem>.
|
||||||
|
|
||||||
|
|
||||||
=item B<--semaphoretimeout> I<secs> (not implemented)
|
=item B<--semaphoretimeout> I<secs> (not implemented)
|
||||||
|
@ -1178,7 +1191,7 @@ If the semaphore is not released within secs seconds, take it anyway.
|
||||||
|
|
||||||
Implies B<--semaphore>.
|
Implies B<--semaphore>.
|
||||||
|
|
||||||
See also: B<man sem>
|
See also B<man sem>.
|
||||||
|
|
||||||
|
|
||||||
=item B<--seqreplace> I<replace-str>
|
=item B<--seqreplace> I<replace-str>
|
||||||
|
@ -1414,7 +1427,7 @@ To override use B<--gnu>.
|
||||||
|
|
||||||
Print the job to be run on stderr (standard error).
|
Print the job to be run on stderr (standard error).
|
||||||
|
|
||||||
See also B<-v> and B<-p>.
|
See also B<-v>, B<-p>.
|
||||||
|
|
||||||
|
|
||||||
=item B<--transfer>
|
=item B<--transfer>
|
||||||
|
@ -1563,7 +1576,7 @@ Wait for all commands to complete.
|
||||||
|
|
||||||
Implies B<--semaphore>.
|
Implies B<--semaphore>.
|
||||||
|
|
||||||
See also: B<man sem>
|
See also B<man sem>.
|
||||||
|
|
||||||
|
|
||||||
=item B<-X>
|
=item B<-X>
|
||||||
|
@ -2286,7 +2299,7 @@ B<rsync -Havessh src-dir/ fooserver:/dest-dir/>
|
||||||
If you are unable to push data, but need to pull them and the files
|
If you are unable to push data, but need to pull them and the files
|
||||||
are called digits.png (e.g. 000000.png) you might be able to do:
|
are called digits.png (e.g. 000000.png) you might be able to do:
|
||||||
|
|
||||||
<seq -w 0 99 | parallel rsync -Havessh fooserver:src-path/*{}.png destdir/>
|
B<seq -w 0 99 | parallel rsync -Havessh fooserver:src-path/*{}.png destdir/>
|
||||||
|
|
||||||
|
|
||||||
=head1 EXAMPLE: Use multiple inputs in one command
|
=head1 EXAMPLE: Use multiple inputs in one command
|
||||||
|
@ -3672,8 +3685,8 @@ it also uses rsync with ssh.
|
||||||
|
|
||||||
=head1 SEE ALSO
|
=head1 SEE ALSO
|
||||||
|
|
||||||
B<ssh>(1), B<rsync>(1), B<find>(1), B<xargs>(1), B<dirname>,
|
B<ssh>(1), B<rsync>(1), B<find>(1), B<xargs>(1), B<dirname>(1),
|
||||||
B<make>(1), B<pexec>(1), B<ppss>(1), B<xjobs>(1), B<prll>(1),
|
B<make>(1), B<pexec>(1), B<ppss>(1), B<xjobs>(1), B<prll>(1),
|
||||||
B<dxargs>(1), B<mdm>(1),
|
B<dxargs>(1), B<mdm>(1)
|
||||||
|
|
||||||
=cut
|
=cut
|
||||||
|
|
|
@ -355,7 +355,7 @@ Run command in background thus GNU @strong{parallel} will not wait for
|
||||||
completion of the command before exiting. This is the default if
|
completion of the command before exiting. This is the default if
|
||||||
@strong{--semaphore} is set.
|
@strong{--semaphore} is set.
|
||||||
|
|
||||||
See also: @strong{--fg}, @strong{man sem}
|
See also: @strong{--fg}, @strong{man sem}.
|
||||||
|
|
||||||
Implies @strong{--semaphore}.
|
Implies @strong{--semaphore}.
|
||||||
|
|
||||||
|
@ -495,6 +495,11 @@ remote execution.
|
||||||
In Bash @emph{var} can also be a Bash function - just remember to @strong{export
|
In Bash @emph{var} can also be a Bash function - just remember to @strong{export
|
||||||
-f} the function.
|
-f} the function.
|
||||||
|
|
||||||
|
The variable '_' is special. It will copy all enviroment variables
|
||||||
|
except for the ones mentioned in ~/.parallel/recorded_env.
|
||||||
|
|
||||||
|
See also: @strong{--record-env}.
|
||||||
|
|
||||||
@item @strong{--eta}
|
@item @strong{--eta}
|
||||||
@anchor{@strong{--eta}}
|
@anchor{@strong{--eta}}
|
||||||
|
|
||||||
|
@ -509,7 +514,7 @@ Implies @strong{--progress}.
|
||||||
Run command in foreground thus GNU @strong{parallel} will wait for
|
Run command in foreground thus GNU @strong{parallel} will wait for
|
||||||
completion of the command before exiting.
|
completion of the command before exiting.
|
||||||
|
|
||||||
See also: @strong{--bg}, @strong{man sem}
|
See also @strong{--bg}, @strong{man sem}.
|
||||||
|
|
||||||
Implies @strong{--semaphore}.
|
Implies @strong{--semaphore}.
|
||||||
|
|
||||||
|
@ -895,7 +900,7 @@ reading everything at startup.
|
||||||
By sending GNU @strong{parallel} SIGUSR2 you can toggle turning on/off
|
By sending GNU @strong{parallel} SIGUSR2 you can toggle turning on/off
|
||||||
@strong{--progress} on a running GNU @strong{parallel} process.
|
@strong{--progress} on a running GNU @strong{parallel} process.
|
||||||
|
|
||||||
See also: @strong{--eta}
|
See also @strong{--eta}.
|
||||||
|
|
||||||
@item @strong{--max-args}=@emph{max-args}
|
@item @strong{--max-args}=@emph{max-args}
|
||||||
@anchor{@strong{--max-args}=@emph{max-args}}
|
@anchor{@strong{--max-args}=@emph{max-args}}
|
||||||
|
@ -1012,6 +1017,14 @@ If the stdin (standard input) only contains whitespace, do not run the command.
|
||||||
|
|
||||||
If used with @strong{--pipe} this is slow.
|
If used with @strong{--pipe} this is slow.
|
||||||
|
|
||||||
|
@item @strong{--record-env} (alpha testing)
|
||||||
|
@anchor{@strong{--record-env} (alpha testing)}
|
||||||
|
|
||||||
|
Record current environment variables in ~/.parallel/recorded_env. This
|
||||||
|
is useful before using @strong{--env _}.
|
||||||
|
|
||||||
|
See also @strong{--env}.
|
||||||
|
|
||||||
@item @strong{--recstart} @emph{startstring}
|
@item @strong{--recstart} @emph{startstring}
|
||||||
@anchor{@strong{--recstart} @emph{startstring}}
|
@anchor{@strong{--recstart} @emph{startstring}}
|
||||||
|
|
||||||
|
@ -1117,7 +1130,7 @@ there. As GNU @strong{parallel} only looks at the sequence numbers in
|
||||||
@strong{--joblog} then the input, the command, and @strong{--joblog} all have to
|
@strong{--joblog} then the input, the command, and @strong{--joblog} all have to
|
||||||
remain unchanged; otherwise GNU @strong{parallel} may run wrong commands.
|
remain unchanged; otherwise GNU @strong{parallel} may run wrong commands.
|
||||||
|
|
||||||
See also: @strong{--joblog}, @strong{--resume-failed}.
|
See also @strong{--joblog}, @strong{--resume-failed}.
|
||||||
|
|
||||||
@item @strong{--resume-failed}
|
@item @strong{--resume-failed}
|
||||||
@anchor{@strong{--resume-failed}}
|
@anchor{@strong{--resume-failed}}
|
||||||
|
@ -1130,7 +1143,7 @@ numbers in @strong{--joblog} then the input, the command, and @strong{--joblog}
|
||||||
all have to remain unchanged; otherwise GNU @strong{parallel} may run wrong
|
all have to remain unchanged; otherwise GNU @strong{parallel} may run wrong
|
||||||
commands.
|
commands.
|
||||||
|
|
||||||
See also: @strong{--joblog}, @strong{--resume}.
|
See also @strong{--joblog}, @strong{--resume}.
|
||||||
|
|
||||||
@item @strong{--retries} @emph{n}
|
@item @strong{--retries} @emph{n}
|
||||||
@anchor{@strong{--retries} @emph{n}}
|
@anchor{@strong{--retries} @emph{n}}
|
||||||
|
@ -1234,7 +1247,7 @@ Used with @strong{--fg}, @strong{--wait}, and @strong{--semaphorename}.
|
||||||
|
|
||||||
The command @strong{sem} is an alias for @strong{parallel --semaphore}.
|
The command @strong{sem} is an alias for @strong{parallel --semaphore}.
|
||||||
|
|
||||||
See also: @strong{man sem}
|
See also @strong{man sem}.
|
||||||
|
|
||||||
@item @strong{--semaphorename} @emph{name}
|
@item @strong{--semaphorename} @emph{name}
|
||||||
@anchor{@strong{--semaphorename} @emph{name}}
|
@anchor{@strong{--semaphorename} @emph{name}}
|
||||||
|
@ -1253,7 +1266,7 @@ The semaphore is stored in ~/.parallel/semaphores/
|
||||||
|
|
||||||
Implies @strong{--semaphore}.
|
Implies @strong{--semaphore}.
|
||||||
|
|
||||||
See also: @strong{man sem}
|
See also @strong{man sem}.
|
||||||
|
|
||||||
@item @strong{--semaphoretimeout} @emph{secs} (not implemented)
|
@item @strong{--semaphoretimeout} @emph{secs} (not implemented)
|
||||||
@anchor{@strong{--semaphoretimeout} @emph{secs} (not implemented)}
|
@anchor{@strong{--semaphoretimeout} @emph{secs} (not implemented)}
|
||||||
|
@ -1262,7 +1275,7 @@ If the semaphore is not released within secs seconds, take it anyway.
|
||||||
|
|
||||||
Implies @strong{--semaphore}.
|
Implies @strong{--semaphore}.
|
||||||
|
|
||||||
See also: @strong{man sem}
|
See also @strong{man sem}.
|
||||||
|
|
||||||
@item @strong{--seqreplace} @emph{replace-str}
|
@item @strong{--seqreplace} @emph{replace-str}
|
||||||
@anchor{@strong{--seqreplace} @emph{replace-str}}
|
@anchor{@strong{--seqreplace} @emph{replace-str}}
|
||||||
|
@ -1520,7 +1533,7 @@ To override use @strong{--gnu}.
|
||||||
|
|
||||||
Print the job to be run on stderr (standard error).
|
Print the job to be run on stderr (standard error).
|
||||||
|
|
||||||
See also @strong{-v} and @strong{-p}.
|
See also @strong{-v}, @strong{-p}.
|
||||||
|
|
||||||
@item @strong{--transfer}
|
@item @strong{--transfer}
|
||||||
@anchor{@strong{--transfer}}
|
@anchor{@strong{--transfer}}
|
||||||
|
@ -1680,7 +1693,7 @@ Wait for all commands to complete.
|
||||||
|
|
||||||
Implies @strong{--semaphore}.
|
Implies @strong{--semaphore}.
|
||||||
|
|
||||||
See also: @strong{man sem}
|
See also @strong{man sem}.
|
||||||
|
|
||||||
@item @strong{-X}
|
@item @strong{-X}
|
||||||
@anchor{@strong{-X}}
|
@anchor{@strong{-X}}
|
||||||
|
@ -2474,7 +2487,7 @@ are not being transferred. To fix those run @strong{rsync} a final time:
|
||||||
If you are unable to push data, but need to pull them and the files
|
If you are unable to push data, but need to pull them and the files
|
||||||
are called digits.png (e.g. 000000.png) you might be able to do:
|
are called digits.png (e.g. 000000.png) you might be able to do:
|
||||||
|
|
||||||
<seq -w 0 99 | parallel rsync -Havessh fooserver:src-path/*@{@}.png destdir/>
|
@strong{seq -w 0 99 | parallel rsync -Havessh fooserver:src-path/*@{@}.png destdir/}
|
||||||
|
|
||||||
@chapter EXAMPLE: Use multiple inputs in one command
|
@chapter EXAMPLE: Use multiple inputs in one command
|
||||||
@anchor{EXAMPLE: Use multiple inputs in one command}
|
@anchor{EXAMPLE: Use multiple inputs in one command}
|
||||||
|
@ -3911,8 +3924,8 @@ it also uses rsync with ssh.
|
||||||
@chapter SEE ALSO
|
@chapter SEE ALSO
|
||||||
@anchor{SEE ALSO}
|
@anchor{SEE ALSO}
|
||||||
|
|
||||||
@strong{ssh}(1), @strong{rsync}(1), @strong{find}(1), @strong{xargs}(1), @strong{dirname},
|
@strong{ssh}(1), @strong{rsync}(1), @strong{find}(1), @strong{xargs}(1), @strong{dirname}(1),
|
||||||
@strong{make}(1), @strong{pexec}(1), @strong{ppss}(1), @strong{xjobs}(1), @strong{prll}(1),
|
@strong{make}(1), @strong{pexec}(1), @strong{ppss}(1), @strong{xjobs}(1), @strong{prll}(1),
|
||||||
@strong{dxargs}(1), @strong{mdm}(1),
|
@strong{dxargs}(1), @strong{mdm}(1)
|
||||||
|
|
||||||
@bye
|
@bye
|
||||||
|
|
|
@ -3,4 +3,10 @@
|
||||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j10 -k -L1
|
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j10 -k -L1
|
||||||
echo '### Test bug #34241: --pipe should not spawn unneeded processes'
|
echo '### Test bug #34241: --pipe should not spawn unneeded processes'
|
||||||
seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end
|
seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end
|
||||||
|
|
||||||
|
echo '### --env _'
|
||||||
|
fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $PATH' ::: test
|
||||||
|
echo '### --env _ with explicit mentioning of normally ignored var $PATH'
|
||||||
|
fUbAr="OK FUBAR" parallel -S csh@lo --env PATH,_ echo '$fUbAr $PATH' ::: test
|
||||||
|
|
||||||
EOF
|
EOF
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
rsync -Ha --delete input-files/testdir/ tmp/
|
rsync -Ha --delete input-files/testdir/ /tmp/parallel_$$
|
||||||
cd tmp
|
cd /tmp/parallel_$$
|
||||||
|
|
||||||
echo echo test of cat pipe sh | parallel -j 50 2>&1
|
echo echo test of cat pipe sh | parallel -j 50 2>&1
|
||||||
find . -name '*.jpg' | parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}
|
find . -name '*.jpg' | parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}
|
||||||
|
@ -11,15 +11,18 @@ ls | parallel echo ls | sort
|
||||||
ls | parallel -j 1 echo ls | sort
|
ls | parallel -j 1 echo ls | sort
|
||||||
find -type f | parallel diff {} a/foo ">"{}.diff | sort
|
find -type f | parallel diff {} a/foo ">"{}.diff | sort
|
||||||
ls | parallel -v --group "ls {}|wc;echo {}" | sort
|
ls | parallel -v --group "ls {}|wc;echo {}" | sort
|
||||||
echo '### Check that we can have more input than max procs (-j 0)'
|
echo '### Check that we can have more input than max procs (-j 0) - touch'
|
||||||
perl -e 'print map {"more_than_5000-$_\n" } (4000..9999)' | parallel -vj 0 touch | sort | tail
|
perl -e 'print map {"more_than_5000-$_\n" } (4000..9999)' | parallel -vj 0 touch | sort | tail
|
||||||
|
echo '### rm'
|
||||||
perl -e 'print map {"more_than_5000-$_\n" } (4000..9900)' | parallel -j 0 rm | sort
|
perl -e 'print map {"more_than_5000-$_\n" } (4000..9900)' | parallel -j 0 rm | sort
|
||||||
|
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout nice parallel -k -L1
|
||||||
ls | parallel -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" {}\n\"}"' | sort
|
ls | parallel -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" {}\n\"}"' | sort
|
||||||
ls | parallel --group -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" {}\n\"}"' | sort
|
ls | parallel --group -j500 'sleep 1; find {} -type f | perl -ne "END{print $..\" {}\n\"}"' | sort
|
||||||
find . -type f | parallel --group "perl -ne '/^\\S+\\s+\\S+$/ and print \$ARGV,\"\\n\"'" | sort
|
find . -type f | parallel --group "perl -ne '/^\\S+\\s+\\S+$/ and print \$ARGV,\"\\n\"'" | sort
|
||||||
find . -type f | parallel -v --group "perl -ne '/^\\S+\\s+\\S+$/ and print \$ARGV,\"\\n\"'" | sort
|
find . -type f | parallel -v --group "perl -ne '/^\\S+\\s+\\S+$/ and print \$ARGV,\"\\n\"'" | sort
|
||||||
find . -type f | parallel -q --group perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' | sort
|
find . -type f | parallel -q --group perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' | sort
|
||||||
find . -type f | parallel -qv --group perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' | sort
|
find . -type f | parallel -qv --group perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' | sort
|
||||||
|
EOF
|
||||||
|
|
||||||
cd ..
|
cd -
|
||||||
rm -rf tmp
|
rm -rf /tmp/parallel_$$
|
||||||
|
|
|
@ -17,6 +17,12 @@ echo '### bug #39554: Feature request: line buffered output --tag';
|
||||||
parallel --tag -j0 --linebuffer 'echo -n start {};sleep 0.{#};echo middle -n {};sleep 1.{#}5;echo next to last {};sleep 1.{#};echo -n last {}' ::: A B C
|
parallel --tag -j0 --linebuffer 'echo -n start {};sleep 0.{#};echo middle -n {};sleep 1.{#}5;echo next to last {};sleep 1.{#};echo -n last {}' ::: A B C
|
||||||
echo
|
echo
|
||||||
|
|
||||||
|
echo '### test round-robin';
|
||||||
|
seq 1000 | parallel --block 1k --pipe --round-robin wc | sort
|
||||||
|
|
||||||
|
echo '### --version must have higher priority than retired options'
|
||||||
|
parallel --version -g -Y -U -W -T | tail
|
||||||
|
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
rm -rf tmp
|
rm -rf tmp
|
||||||
|
|
|
@ -5,7 +5,7 @@ parallel --joblog /tmp/parallel_joblog_exitval 'sleep {} && echo sleep was not k
|
||||||
parallel --joblog /tmp/parallel_joblog_signal 'sleep {}' ::: 100 2>/dev/null &
|
parallel --joblog /tmp/parallel_joblog_signal 'sleep {}' ::: 100 2>/dev/null &
|
||||||
sleep 1
|
sleep 1
|
||||||
killall -6 sleep
|
killall -6 sleep
|
||||||
sleep 0.1
|
sleep 1
|
||||||
grep -q 134 /tmp/parallel_joblog_exitval && echo exitval OK
|
grep -q 134 /tmp/parallel_joblog_exitval && echo exitval OK
|
||||||
grep -q '[^0-9]6[^0-9]' /tmp/parallel_joblog_signal && echo signal OK
|
grep -q '[^0-9]6[^0-9]' /tmp/parallel_joblog_signal && echo signal OK
|
||||||
|
|
||||||
|
|
|
@ -6,3 +6,7 @@ Block_end
|
||||||
4
|
4
|
||||||
5
|
5
|
||||||
Block_end
|
Block_end
|
||||||
|
### --env _
|
||||||
|
OK FUBAR /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games test
|
||||||
|
### --env _ with explicit mentioning of normally ignored var $PATH
|
||||||
|
OK FUBAR /home/tange/bin:/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin test
|
||||||
|
|
|
@ -50,7 +50,7 @@ ls \ ab|wc;echo \ ab
|
||||||
ls a\ b|wc;echo a\ b
|
ls a\ b|wc;echo a\ b
|
||||||
ls a|wc;echo a
|
ls a|wc;echo a
|
||||||
ls b|wc;echo b
|
ls b|wc;echo b
|
||||||
### Check that we can have more input than max procs (-j 0)
|
### Check that we can have more input than max procs (-j 0) - touch
|
||||||
touch more_than_5000-9990
|
touch more_than_5000-9990
|
||||||
touch more_than_5000-9991
|
touch more_than_5000-9991
|
||||||
touch more_than_5000-9992
|
touch more_than_5000-9992
|
||||||
|
@ -61,6 +61,7 @@ touch more_than_5000-9996
|
||||||
touch more_than_5000-9997
|
touch more_than_5000-9997
|
||||||
touch more_than_5000-9998
|
touch more_than_5000-9998
|
||||||
touch more_than_5000-9999
|
touch more_than_5000-9999
|
||||||
|
### rm
|
||||||
1 1-col.txt
|
1 1-col.txt
|
||||||
1 1-col.txt.diff
|
1 1-col.txt.diff
|
||||||
1 2-col.txt
|
1 2-col.txt
|
||||||
|
@ -591,3 +592,4 @@ perl -ne /^\\S+\\s+\\S+\$/\ and\ print\ \$ARGV,\"\\n\" ./more_than_5000-9996
|
||||||
perl -ne /^\\S+\\s+\\S+\$/\ and\ print\ \$ARGV,\"\\n\" ./more_than_5000-9997
|
perl -ne /^\\S+\\s+\\S+\$/\ and\ print\ \$ARGV,\"\\n\" ./more_than_5000-9997
|
||||||
perl -ne /^\\S+\\s+\\S+\$/\ and\ print\ \$ARGV,\"\\n\" ./more_than_5000-9998
|
perl -ne /^\\S+\\s+\\S+\$/\ and\ print\ \$ARGV,\"\\n\" ./more_than_5000-9998
|
||||||
perl -ne /^\\S+\\s+\\S+\$/\ and\ print\ \$ARGV,\"\\n\" ./more_than_5000-9999
|
perl -ne /^\\S+\\s+\\S+\$/\ and\ print\ \$ARGV,\"\\n\" ./more_than_5000-9999
|
||||||
|
/home/tange/privat/parallel/testsuite
|
||||||
|
|
|
@ -28,3 +28,17 @@ A next to last A
|
||||||
B next to last B
|
B next to last B
|
||||||
C next to last C
|
C next to last C
|
||||||
A last AB last BC last C
|
A last AB last BC last C
|
||||||
|
### test round-robin
|
||||||
|
473 473 1893
|
||||||
|
527 527 2000
|
||||||
|
### --version must have higher priority than retired options
|
||||||
|
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
|
||||||
|
This is free software: you are free to change and redistribute it.
|
||||||
|
GNU parallel comes with no warranty.
|
||||||
|
|
||||||
|
Web site: http://www.gnu.org/software/parallel
|
||||||
|
|
||||||
|
When using GNU Parallel for a publication please cite:
|
||||||
|
|
||||||
|
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
|
||||||
|
;login: The USENIX Magazine, February 2011:42-47.
|
||||||
|
|
|
@ -33,7 +33,7 @@ redhat.polarhome.com Works on redhat.polarhome.com
|
||||||
hpux.polarhome.com Works on hpux.polarhome.com
|
hpux.polarhome.com Works on hpux.polarhome.com
|
||||||
qnx.polarhome.com Works on qnx.polarhome.com
|
qnx.polarhome.com Works on qnx.polarhome.com
|
||||||
qnx.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1.
|
qnx.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1.
|
||||||
irix.polarhome.com Unknown open() mode '>&=' at bin/parallel line 1318.
|
irix.polarhome.com Unknown open() mode '>&=' at bin/parallel line 1346.
|
||||||
openindiana.polarhome.com Works on openindiana.polarhome.com
|
openindiana.polarhome.com Works on openindiana.polarhome.com
|
||||||
openindiana.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1.
|
openindiana.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1.
|
||||||
suse.polarhome.com Works on suse.polarhome.com
|
suse.polarhome.com Works on suse.polarhome.com
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
### Test -k
|
### Test -k
|
||||||
parallel: Warning: Only enough file handles to run 19 jobs in parallel.
|
parallel: Warning: Only enough file handles to run 18 jobs in parallel.
|
||||||
Raising ulimit -n or /etc/security/limits.conf may help.
|
Raising ulimit -n or /etc/security/limits.conf may help.
|
||||||
begin
|
begin
|
||||||
1
|
1
|
||||||
|
@ -17,8 +17,8 @@ begin
|
||||||
13
|
13
|
||||||
14
|
14
|
||||||
15
|
15
|
||||||
16
|
|
||||||
parallel: Warning: No more file handles. Raising ulimit -n or /etc/security/limits.conf may help.
|
parallel: Warning: No more file handles. Raising ulimit -n or /etc/security/limits.conf may help.
|
||||||
|
16
|
||||||
17
|
17
|
||||||
18
|
18
|
||||||
19
|
19
|
||||||
|
|
Loading…
Reference in a new issue