mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-12-22 20:57:53 +00:00
parallel: Fixed bug #42842: --bar with weird chars. Fixed bug #42845: rsync 3.1.x fails against 2.5.7.
Give warning if reading arguments (for --eta/--bar) takes > 30 sec. A few more characters annoy tmux. parallel.pod: Bash array copy function.
This commit is contained in:
Ole Tange
parent
9a25177a19
commit
af23562d99
2
CREDITS
Normal file
2
CREDITS
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
rici@stackoverflow.com: Documentation on exporting arrays using --env.
|
||||
Malcolm Cook: The idea to use a general perl expression as replacement strings.
|
||||
|
|
@ -1,459 +0,0 @@
|
|||
\input texinfo
|
||||
@setfilename niceload.info
|
||||
|
||||
@documentencoding utf-8
|
||||
|
||||
@settitle niceload - slow down a program when the load average is above a certain limit
|
||||
|
||||
@node Top
|
||||
@top niceload
|
||||
|
||||
@menu
|
||||
* NAME::
|
||||
* SYNOPSIS::
|
||||
* DESCRIPTION::
|
||||
* OPTIONS::
|
||||
* EXAMPLE@asis{:} See niceload in action::
|
||||
* EXAMPLE@asis{:} Run updatedb::
|
||||
* EXAMPLE@asis{:} Run rsync::
|
||||
* EXAMPLE@asis{:} Ensure enough disk cache::
|
||||
* ENVIRONMENT VARIABLES::
|
||||
* EXIT STATUS::
|
||||
* REPORTING BUGS::
|
||||
* AUTHOR::
|
||||
* LICENSE::
|
||||
* DEPENDENCIES::
|
||||
* SEE ALSO::
|
||||
@end menu
|
||||
|
||||
@node NAME
|
||||
@chapter NAME
|
||||
|
||||
niceload - slow down a program when the load average is above a certain limit
|
||||
|
||||
@node SYNOPSIS
|
||||
@chapter SYNOPSIS
|
||||
|
||||
@strong{niceload} [-v] [-h] [-n nice] [-I io] [-L load] [-M mem] [-N]
|
||||
[--sensor program] [-t time] [-s time|-f factor] ( command | -p PID [-p PID ...] )
|
||||
|
||||
@node DESCRIPTION
|
||||
@chapter DESCRIPTION
|
||||
|
||||
GNU @strong{niceload} will slow down a program when the load average (or
|
||||
other system activity) is above a certain limit. When the limit is
|
||||
reached the program will be suspended for some time. Then resumed
|
||||
again for some time. Then the load average is checked again and we
|
||||
start over.
|
||||
|
||||
Instead of load average @strong{niceload} can also look at disk I/O, amount
|
||||
of free memory, or swapping activity.
|
||||
|
||||
If the load is 3.00 then the default settings will run a program
|
||||
like this:
|
||||
|
||||
run 1 second, suspend (3.00-1.00) seconds, run 1 second, suspend
|
||||
(3.00-1.00) seconds, run 1 second, ...
|
||||
|
||||
@node OPTIONS
|
||||
@chapter OPTIONS
|
||||
|
||||
@table @asis
|
||||
@item @strong{-f} @emph{FACTOR}
|
||||
@anchor{@strong{-f} @emph{FACTOR}}
|
||||
|
||||
@item @strong{--factor} @emph{FACTOR}
|
||||
@anchor{@strong{--factor} @emph{FACTOR}}
|
||||
|
||||
Suspend time factor. Dynamically set @strong{-s} as amount over limit *
|
||||
factor. Default is 1.
|
||||
|
||||
@item @strong{-H}
|
||||
@anchor{@strong{-H}}
|
||||
|
||||
@item @strong{--hard}
|
||||
@anchor{@strong{--hard}}
|
||||
|
||||
Hard limit. @strong{--hard} will suspend the process until the system is
|
||||
under the limits. The default is @strong{--soft}.
|
||||
|
||||
@item @strong{--io} @emph{iolimit}
|
||||
@anchor{@strong{--io} @emph{iolimit}}
|
||||
|
||||
@item @strong{-I} @emph{iolimit}
|
||||
@anchor{@strong{-I} @emph{iolimit}}
|
||||
|
||||
Limit for I/O. The amount of disk I/O will be computed as a value 0 -
|
||||
10, where 0 is no I/O and 10 is at least one disk is 100% saturated.
|
||||
|
||||
@strong{--io} will set both @strong{--start-io} and @strong{run-io}.
|
||||
|
||||
@item @strong{--load} @emph{loadlimit}
|
||||
@anchor{@strong{--load} @emph{loadlimit}}
|
||||
|
||||
@item @strong{-L} @emph{loadlimit}
|
||||
@anchor{@strong{-L} @emph{loadlimit}}
|
||||
|
||||
Limit for load average.
|
||||
|
||||
@strong{--load} will set both @strong{--start-load} and @strong{run-load}.
|
||||
|
||||
@item @strong{--mem} @emph{memlimit}
|
||||
@anchor{@strong{--mem} @emph{memlimit}}
|
||||
|
||||
@item @strong{-M} @emph{memlimit}
|
||||
@anchor{@strong{-M} @emph{memlimit}}
|
||||
|
||||
Limit for free memory. This is the amount of bytes available as free
|
||||
+ cache. This limit is treated opposite other limits: If the system
|
||||
is above the limit the program will run, if it is below the limit the
|
||||
program will stop
|
||||
|
||||
@emph{memlimit} can be postfixed with K, M, G, T, or P which would
|
||||
multiply the size with 1024, 1048576, 1073741824, or 1099511627776
|
||||
respectively.
|
||||
|
||||
@strong{--mem} will set both @strong{--start-mem} and @strong{run-mem}.
|
||||
|
||||
@item @strong{--noswap}
|
||||
@anchor{@strong{--noswap}}
|
||||
|
||||
@item @strong{-N}
|
||||
@anchor{@strong{-N}}
|
||||
|
||||
No swapping. If the system is swapping both in and out it is a good
|
||||
indication that the system is memory stressed.
|
||||
|
||||
@strong{--noswap} is over limit if the system is swapping both in and out.
|
||||
|
||||
@strong{--noswap} will set both @strong{--start-noswap} and @strong{run-noswap}.
|
||||
|
||||
@item @strong{-n} @emph{niceness}
|
||||
@anchor{@strong{-n} @emph{niceness}}
|
||||
|
||||
@item @strong{--nice} @emph{niceness}
|
||||
@anchor{@strong{--nice} @emph{niceness}}
|
||||
|
||||
Sets niceness. See @strong{nice}(1).
|
||||
|
||||
@item @strong{-p} @emph{PID} (alpha testing)
|
||||
@anchor{@strong{-p} @emph{PID} (alpha testing)}
|
||||
|
||||
@item @strong{--pid} @emph{PID} (alpha testing)
|
||||
@anchor{@strong{--pid} @emph{PID} (alpha testing)}
|
||||
|
||||
Process ID of process to suspend. You can specify multiple process IDs
|
||||
with multiple @strong{-p} @emph{PID}.
|
||||
|
||||
@item @strong{--prg} @emph{program} (alpha testing)
|
||||
@anchor{@strong{--prg} @emph{program} (alpha testing)}
|
||||
|
||||
@item @strong{--program} @emph{program} (alpha testing)
|
||||
@anchor{@strong{--program} @emph{program} (alpha testing)}
|
||||
|
||||
Name of running program to suspend. You can specify multiple programs
|
||||
with multiple @strong{--prg} @emph{program}.
|
||||
|
||||
@item @strong{--quote}
|
||||
@anchor{@strong{--quote}}
|
||||
|
||||
@item @strong{-q}
|
||||
@anchor{@strong{-q}}
|
||||
|
||||
Quote the command line. Useful if the command contains chars like *,
|
||||
$, >, and " that should not be interpreted by the shell.
|
||||
|
||||
@item @strong{--run-io} @emph{iolimit}
|
||||
@anchor{@strong{--run-io} @emph{iolimit}}
|
||||
|
||||
@item @strong{--ri} @emph{iolimit}
|
||||
@anchor{@strong{--ri} @emph{iolimit}}
|
||||
|
||||
@item @strong{--run-load} @emph{loadlimit}
|
||||
@anchor{@strong{--run-load} @emph{loadlimit}}
|
||||
|
||||
@item @strong{--rl} @emph{loadlimit}
|
||||
@anchor{@strong{--rl} @emph{loadlimit}}
|
||||
|
||||
@item @strong{--run-mem} @emph{memlimit}
|
||||
@anchor{@strong{--run-mem} @emph{memlimit}}
|
||||
|
||||
@item @strong{--rm} @emph{memlimit}
|
||||
@anchor{@strong{--rm} @emph{memlimit}}
|
||||
|
||||
Run limit. The running program will be slowed down if the system is
|
||||
above the limit. See: @strong{--io}, @strong{--load}, @strong{--mem}, @strong{--noswap}.
|
||||
|
||||
@item @strong{--sensor} @emph{sensor program} (alpha testing)
|
||||
@anchor{@strong{--sensor} @emph{sensor program} (alpha testing)}
|
||||
|
||||
Read sensor. Use @emph{sensor program} to read a sensor.
|
||||
|
||||
This will keep the CPU temperature below 80 deg C on GNU/Linux:
|
||||
|
||||
@verbatim
|
||||
niceload -l 80000 -f 0.001 --sensor 'sort -n /sys/devices/platform/coretemp*/temp*_input' gzip *
|
||||
@end verbatim
|
||||
|
||||
This will stop if the disk space < 100000.
|
||||
|
||||
@verbatim
|
||||
niceload -H -l -100000 --sensor "df . | awk '{ print \$4 }'" echo
|
||||
@end verbatim
|
||||
|
||||
@item @strong{--start-io} @emph{iolimit}
|
||||
@anchor{@strong{--start-io} @emph{iolimit}}
|
||||
|
||||
@item @strong{--si} @emph{iolimit}
|
||||
@anchor{@strong{--si} @emph{iolimit}}
|
||||
|
||||
@item @strong{--start-load} @emph{loadlimit}
|
||||
@anchor{@strong{--start-load} @emph{loadlimit}}
|
||||
|
||||
@item @strong{--sl} @emph{loadlimit}
|
||||
@anchor{@strong{--sl} @emph{loadlimit}}
|
||||
|
||||
@item @strong{--start-mem} @emph{memlimit}
|
||||
@anchor{@strong{--start-mem} @emph{memlimit}}
|
||||
|
||||
@item @strong{--sm} @emph{memlimit}
|
||||
@anchor{@strong{--sm} @emph{memlimit}}
|
||||
|
||||
Start limit. The program will not start until the system is below the
|
||||
limit. See: @strong{--io}, @strong{--load}, @strong{--mem}, @strong{--noswap}.
|
||||
|
||||
@item @strong{--soft}
|
||||
@anchor{@strong{--soft}}
|
||||
|
||||
@item @strong{-S}
|
||||
@anchor{@strong{-S}}
|
||||
|
||||
Soft limit. @strong{niceload} will suspend a process for a while and then
|
||||
let it run for a second thus only slowing down a process while the
|
||||
system is over one of the given limits. This is the default.
|
||||
|
||||
@item @strong{--suspend} @emph{SEC}
|
||||
@anchor{@strong{--suspend} @emph{SEC}}
|
||||
|
||||
@item @strong{-s} @emph{SEC}
|
||||
@anchor{@strong{-s} @emph{SEC}}
|
||||
|
||||
Suspend time. Suspend the command this many seconds when the max load
|
||||
average is reached.
|
||||
|
||||
@item @strong{--recheck} @emph{SEC}
|
||||
@anchor{@strong{--recheck} @emph{SEC}}
|
||||
|
||||
@item @strong{-t} @emph{SEC}
|
||||
@anchor{@strong{-t} @emph{SEC}}
|
||||
|
||||
Recheck load time. Sleep SEC seconds before checking load
|
||||
again. Default is 1 second.
|
||||
|
||||
@item @strong{--verbose}
|
||||
@anchor{@strong{--verbose}}
|
||||
|
||||
@item @strong{-v}
|
||||
@anchor{@strong{-v}}
|
||||
|
||||
Verbose. Print some extra output on what is happening. Use @strong{-v} until
|
||||
you know what your are doing.
|
||||
|
||||
@end table
|
||||
|
||||
@node EXAMPLE: See niceload in action
|
||||
@chapter EXAMPLE: See niceload in action
|
||||
|
||||
In terminal 1 run: top
|
||||
|
||||
In terminal 2 run:
|
||||
|
||||
@strong{niceload -q perl -e '$|=1;do@{$l==$r or print "."; $l=$r@}until(($r=time-$^T)}>@strong{50)'}
|
||||
|
||||
This will print a '.' every second for 50 seconds and eat a lot of
|
||||
CPU. When the load rises to 1.0 the process is suspended.
|
||||
|
||||
@node EXAMPLE: Run updatedb
|
||||
@chapter EXAMPLE: Run updatedb
|
||||
|
||||
Running updatedb can often starve the system for disk I/O and thus result in a high load.
|
||||
|
||||
Run updatedb but suspend updatedb if the load is above 2.00:
|
||||
|
||||
@strong{niceload -L 2 updatedb}
|
||||
|
||||
@node EXAMPLE: Run rsync
|
||||
@chapter EXAMPLE: Run rsync
|
||||
|
||||
rsync can just like updatedb starve the system for disk I/O and thus result in a high load.
|
||||
|
||||
Run rsync but keep load below 3.4. If load reaches 7 sleep for
|
||||
(7-3.4)*12 seconds:
|
||||
|
||||
@strong{niceload -L 3.4 -f 12 rsync -Ha /home/ /backup/home/}
|
||||
|
||||
@node EXAMPLE: Ensure enough disk cache
|
||||
@chapter EXAMPLE: Ensure enough disk cache
|
||||
|
||||
Assume the program @strong{foo} uses 2 GB files intensively. @strong{foo} will run
|
||||
fast if the files are in disk cache and be slow as a crawl if they are
|
||||
not in the cache.
|
||||
|
||||
To ensure 2 GB are reserved for disk cache run:
|
||||
|
||||
@strong{niceload --hard --run-mem 2g foo}
|
||||
|
||||
This will not guarantee that the 2 GB memory will be used for the
|
||||
files for @strong{foo}, but it will stop @strong{foo} if the memory for disk cache
|
||||
is too low.
|
||||
|
||||
@node ENVIRONMENT VARIABLES
|
||||
@chapter ENVIRONMENT VARIABLES
|
||||
|
||||
None. In future versions $NICELOAD will be able to contain default settings.
|
||||
|
||||
@node EXIT STATUS
|
||||
@chapter EXIT STATUS
|
||||
|
||||
Exit status should be the same as the command being run (untested).
|
||||
|
||||
@node REPORTING BUGS
|
||||
@chapter REPORTING BUGS
|
||||
|
||||
Report bugs to <bug-parallel@@gnu.org>.
|
||||
|
||||
@node AUTHOR
|
||||
@chapter AUTHOR
|
||||
|
||||
Copyright (C) 2004-11-19 Ole Tange, http://ole.tange.dk
|
||||
|
||||
Copyright (C) 2005,2006,2006,2008,2009,2010 Ole Tange, http://ole.tange.dk
|
||||
|
||||
Copyright (C) 2010,2011,2012 Ole Tange, http://ole.tange.dk and Free
|
||||
Software Foundation, Inc.
|
||||
|
||||
@node LICENSE
|
||||
@chapter LICENSE
|
||||
|
||||
Copyright (C) 2010,2011,2012 Free Software Foundation, Inc.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 3 of the License, or
|
||||
at your option any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
@menu
|
||||
* Documentation license I::
|
||||
* Documentation license II::
|
||||
@end menu
|
||||
|
||||
@node Documentation license I
|
||||
@section Documentation license I
|
||||
|
||||
Permission is granted to copy, distribute and/or modify this documentation
|
||||
under the terms of the GNU Free Documentation License, Version 1.3 or
|
||||
any later version published by the Free Software Foundation; with no
|
||||
Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
|
||||
Texts. A copy of the license is included in the file fdl.txt.
|
||||
|
||||
@node Documentation license II
|
||||
@section Documentation license II
|
||||
|
||||
You are free:
|
||||
|
||||
@table @asis
|
||||
@item @strong{to Share}
|
||||
@anchor{@strong{to Share}}
|
||||
|
||||
to copy, distribute and transmit the work
|
||||
|
||||
@item @strong{to Remix}
|
||||
@anchor{@strong{to Remix}}
|
||||
|
||||
to adapt the work
|
||||
|
||||
@end table
|
||||
|
||||
Under the following conditions:
|
||||
|
||||
@table @asis
|
||||
@item @strong{Attribution}
|
||||
@anchor{@strong{Attribution}}
|
||||
|
||||
You must attribute the work in the manner specified by the author or
|
||||
licensor (but not in any way that suggests that they endorse you or
|
||||
your use of the work).
|
||||
|
||||
@item @strong{Share Alike}
|
||||
@anchor{@strong{Share Alike}}
|
||||
|
||||
If you alter, transform, or build upon this work, you may distribute
|
||||
the resulting work only under the same, similar or a compatible
|
||||
license.
|
||||
|
||||
@end table
|
||||
|
||||
With the understanding that:
|
||||
|
||||
@table @asis
|
||||
@item @strong{Waiver}
|
||||
@anchor{@strong{Waiver}}
|
||||
|
||||
Any of the above conditions can be waived if you get permission from
|
||||
the copyright holder.
|
||||
|
||||
@item @strong{Public Domain}
|
||||
@anchor{@strong{Public Domain}}
|
||||
|
||||
Where the work or any of its elements is in the public domain under
|
||||
applicable law, that status is in no way affected by the license.
|
||||
|
||||
@item @strong{Other Rights}
|
||||
@anchor{@strong{Other Rights}}
|
||||
|
||||
In no way are any of the following rights affected by the license:
|
||||
|
||||
@itemize
|
||||
@item Your fair dealing or fair use rights, or other applicable
|
||||
copyright exceptions and limitations;
|
||||
|
||||
@item The author's moral rights;
|
||||
|
||||
@item Rights other persons may have either in the work itself or in
|
||||
how the work is used, such as publicity or privacy rights.
|
||||
|
||||
@end itemize
|
||||
|
||||
@end table
|
||||
|
||||
@table @asis
|
||||
@item @strong{Notice}
|
||||
@anchor{@strong{Notice}}
|
||||
|
||||
For any reuse or distribution, you must make clear to others the
|
||||
license terms of this work.
|
||||
|
||||
@end table
|
||||
|
||||
A copy of the full license is included in the file as cc-by-sa.txt.
|
||||
|
||||
@node DEPENDENCIES
|
||||
@chapter DEPENDENCIES
|
||||
|
||||
GNU @strong{niceload} uses Perl, and the Perl modules POSIX, and
|
||||
Getopt::Long.
|
||||
|
||||
@node SEE ALSO
|
||||
@chapter SEE ALSO
|
||||
|
||||
@strong{parallel}(1), @strong{nice}(1), @strong{uptime}(1)
|
||||
|
||||
@bye
|
||||
186
src/parallel
186
src/parallel
|
|
@ -852,7 +852,8 @@ sub parse_options {
|
|||
if(defined $opt::fg) { $Global::semaphore = 1; }
|
||||
if(defined $opt::bg) { $Global::semaphore = 1; }
|
||||
if(defined $opt::wait) { $Global::semaphore = 1; }
|
||||
if(defined $opt::halt_on_error and $opt::halt_on_error=~/%/) { $opt::halt_on_error /= 100; }
|
||||
if(defined $opt::halt_on_error and
|
||||
$opt::halt_on_error=~/%/) { $opt::halt_on_error /= 100; }
|
||||
if(defined $opt::timeout and $opt::timeout !~ /^\d+(\.\d+)?%?$/) {
|
||||
::error("--timeout must be seconds or percentage\n");
|
||||
wait_and_exit(255);
|
||||
|
|
@ -1817,6 +1818,8 @@ sub progress {
|
|||
if($opt::bar) {
|
||||
my $arg = $Global::newest_job ?
|
||||
$Global::newest_job->{'commandline'}->replace_placeholders(["\257<\257>"],0,0) : "";
|
||||
# [\011\013\014] messes up display in the terminal
|
||||
$arg =~ s/[\011\013\014]//g;
|
||||
my $bar_text = sprintf("%d%% %d:%d=%ds %s",
|
||||
$pctcomplete*100, $completed, $left, $this_eta, $arg);
|
||||
my $rev = '[7m';
|
||||
|
|
@ -4229,74 +4232,98 @@ sub control_path_dir {
|
|||
return $self->{'control_path_dir'};
|
||||
}
|
||||
|
||||
|
||||
sub rsync_transfer_cmd {
|
||||
# Command to run to transfer a file
|
||||
# Input:
|
||||
# $file = filename of file to transfer
|
||||
# $workdir = destination dir
|
||||
# Returns:
|
||||
# $cmd = rsync command to run to transfer $file ("" if unreadable)
|
||||
my $self = shift;
|
||||
my $file = shift;
|
||||
my $workdir = shift;
|
||||
if(not -r $file) {
|
||||
::warning($file, " is not readable and will not be transferred.\n");
|
||||
return "true";
|
||||
}
|
||||
my $rsync_destdir;
|
||||
if($file =~ m:^/:) {
|
||||
# rsync /foo/bar /
|
||||
$rsync_destdir = "/";
|
||||
} else {
|
||||
$rsync_destdir = ::shell_quote_file($workdir);
|
||||
}
|
||||
$file = ::shell_quote_file($file);
|
||||
my $sshcmd = $self->sshcommand();
|
||||
my $rsync_opt = "-rlDzR -e" . ::shell_quote_scalar($sshcmd);
|
||||
my $serverlogin = $self->serverlogin();
|
||||
# Make dir if it does not exist
|
||||
return "( $sshcmd $serverlogin mkdir -p $rsync_destdir;" .
|
||||
"rsync $rsync_opt $file $serverlogin:$rsync_destdir )";
|
||||
# Command to run to transfer a file
|
||||
# Input:
|
||||
# $file = filename of file to transfer
|
||||
# $workdir = destination dir
|
||||
# Returns:
|
||||
# $cmd = rsync command to run to transfer $file ("" if unreadable)
|
||||
my $self = shift;
|
||||
my $file = shift;
|
||||
my $workdir = shift;
|
||||
if(not -r $file) {
|
||||
::warning($file, " is not readable and will not be transferred.\n");
|
||||
return "true";
|
||||
}
|
||||
my $rsync_destdir;
|
||||
if($file =~ m:^/:) {
|
||||
# rsync /foo/bar /
|
||||
$rsync_destdir = "/";
|
||||
} else {
|
||||
$rsync_destdir = ::shell_quote_file($workdir);
|
||||
}
|
||||
$file = ::shell_quote_file($file);
|
||||
my $sshcmd = $self->sshcommand();
|
||||
my $rsync_opt = "-rlDzR -e" . ::shell_quote_scalar($sshcmd);
|
||||
my $serverlogin = $self->serverlogin();
|
||||
# Make dir if it does not exist
|
||||
return "( $sshcmd $serverlogin mkdir -p $rsync_destdir;" .
|
||||
rsync()." $rsync_opt $file $serverlogin:$rsync_destdir )";
|
||||
}
|
||||
|
||||
sub cleanup_cmd {
|
||||
# Command to run to remove the remote file
|
||||
# Input:
|
||||
# $file = filename to remove
|
||||
# $workdir = destination dir
|
||||
# Returns:
|
||||
# $cmd = ssh command to run to remove $file and empty parent dirs
|
||||
my $self = shift;
|
||||
my $file = shift;
|
||||
my $workdir = shift;
|
||||
my $f = $file;
|
||||
if($f =~ m:/\./:) {
|
||||
# foo/bar/./baz/quux => workdir/baz/quux
|
||||
# /foo/bar/./baz/quux => workdir/baz/quux
|
||||
$f =~ s:.*/\./:$workdir/:;
|
||||
} elsif($f =~ m:^[^/]:) {
|
||||
# foo/bar => workdir/foo/bar
|
||||
$f = $workdir."/".$f;
|
||||
}
|
||||
my @subdirs = split m:/:, ::dirname($f);
|
||||
my @rmdir;
|
||||
my $dir = "";
|
||||
for(@subdirs) {
|
||||
$dir .= $_."/";
|
||||
unshift @rmdir, ::shell_quote_file($dir);
|
||||
}
|
||||
my $rmdir = @rmdir ? "rmdir @rmdir 2>/dev/null;" : "";
|
||||
if(defined $opt::workdir and $opt::workdir eq "...") {
|
||||
$rmdir .= "rm -rf " . ::shell_quote_file($workdir).';';
|
||||
}
|
||||
# Command to run to remove the remote file
|
||||
# Input:
|
||||
# $file = filename to remove
|
||||
# $workdir = destination dir
|
||||
# Returns:
|
||||
# $cmd = ssh command to run to remove $file and empty parent dirs
|
||||
my $self = shift;
|
||||
my $file = shift;
|
||||
my $workdir = shift;
|
||||
my $f = $file;
|
||||
if($f =~ m:/\./:) {
|
||||
# foo/bar/./baz/quux => workdir/baz/quux
|
||||
# /foo/bar/./baz/quux => workdir/baz/quux
|
||||
$f =~ s:.*/\./:$workdir/:;
|
||||
} elsif($f =~ m:^[^/]:) {
|
||||
# foo/bar => workdir/foo/bar
|
||||
$f = $workdir."/".$f;
|
||||
}
|
||||
my @subdirs = split m:/:, ::dirname($f);
|
||||
my @rmdir;
|
||||
my $dir = "";
|
||||
for(@subdirs) {
|
||||
$dir .= $_."/";
|
||||
unshift @rmdir, ::shell_quote_file($dir);
|
||||
}
|
||||
my $rmdir = @rmdir ? "rmdir @rmdir 2>/dev/null;" : "";
|
||||
if(defined $opt::workdir and $opt::workdir eq "...") {
|
||||
$rmdir .= "rm -rf " . ::shell_quote_file($workdir).';';
|
||||
}
|
||||
|
||||
$f = ::shell_quote_file($f);
|
||||
my $sshcmd = $self->sshcommand();
|
||||
my $serverlogin = $self->serverlogin();
|
||||
return "$sshcmd $serverlogin ".::shell_quote_scalar("(rm -f $f; $rmdir)");
|
||||
$f = ::shell_quote_file($f);
|
||||
my $sshcmd = $self->sshcommand();
|
||||
my $serverlogin = $self->serverlogin();
|
||||
return "$sshcmd $serverlogin ".::shell_quote_scalar("(rm -f $f; $rmdir)");
|
||||
}
|
||||
|
||||
{
|
||||
my $rsync;
|
||||
|
||||
sub rsync {
|
||||
# rsync 3.1.x uses protocol 31 which is unsupported by 2.5.7.
|
||||
# If the version >= 3.1.0: downgrade to protocol 30
|
||||
if(not $rsync) {
|
||||
my @out = `rsync --version`;
|
||||
for (@out) {
|
||||
if(/version (\d+.\d+)(.\d+)?/) {
|
||||
if($1 >= 3.1) {
|
||||
# Version 3.1.0 or later: Downgrade to protocol 30
|
||||
$rsync = "rsync --protocol 30";
|
||||
} else {
|
||||
$rsync = "rsync";
|
||||
}
|
||||
}
|
||||
}
|
||||
$rsync or ::die_bug("Cannot figure out version of rsync: @out");
|
||||
}
|
||||
return $rsync;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
package JobQueue;
|
||||
|
||||
sub new {
|
||||
|
|
@ -4352,9 +4379,20 @@ sub total_jobs {
|
|||
if(not defined $self->{'total_jobs'}) {
|
||||
my $job;
|
||||
my @queue;
|
||||
my $start = time;
|
||||
while($job = $self->get()) {
|
||||
if(time - $start > 30) {
|
||||
::warning("Reading all arguments takes longer than 30 seconds.\n");
|
||||
$opt::eta && ::warning("Consider removing --eta.\n");
|
||||
$opt::bar && ::warning("Consider removing --bar.\n");
|
||||
last;
|
||||
}
|
||||
push @queue, $job;
|
||||
}
|
||||
while($job = $self->get()) {
|
||||
push @queue, $job;
|
||||
}
|
||||
|
||||
$self->unget(@queue);
|
||||
$self->{'total_jobs'} = $#queue+1;
|
||||
}
|
||||
|
|
@ -5042,8 +5080,8 @@ sub transfersize {
|
|||
}
|
||||
|
||||
sub sshtransfer {
|
||||
# Returns for each transfer file:
|
||||
# rsync $file remote:$workdir
|
||||
# Returns for each transfer file:
|
||||
# rsync $file remote:$workdir
|
||||
my $self = shift;
|
||||
my @pre;
|
||||
my $sshlogin = $self->sshlogin();
|
||||
|
|
@ -5109,9 +5147,9 @@ sub sshreturn {
|
|||
my $basename = ::shell_quote_scalar(::shell_quote_file(basename($file)));
|
||||
# --return
|
||||
# mkdir -p /home/tange/dir/subdir/;
|
||||
# rsync -rlDzR --rsync-path="cd /home/tange/dir/subdir/; rsync"
|
||||
# rsync (--protocol 30) -rlDzR --rsync-path="cd /home/tange/dir/subdir/; rsync"
|
||||
# server:file.gz /home/tange/dir/subdir/
|
||||
$pre .= "mkdir -p $basedir$cd; rsync $rsync_cd $rsync_opt $serverlogin:".
|
||||
$pre .= "mkdir -p $basedir$cd; ".$sshlogin->rsync()." $rsync_cd $rsync_opt $serverlogin:".
|
||||
$basename . " ".$basedir.$cd.";";
|
||||
}
|
||||
return $pre;
|
||||
|
|
@ -5365,8 +5403,9 @@ sub tmux_wrap {
|
|||
unlink $tmpfile;
|
||||
my $visual_command = $self->replaced();
|
||||
my $title = ::undef_as_empty($self->{'commandline'}->replace_placeholders(["\257<\257>"],0,0))."";
|
||||
# ascii 194-224 annoys tmux
|
||||
$title =~ s/[\011-\016;\302-\340]//g;
|
||||
# ; causes problems
|
||||
# ascii 194-245 annoys tmux
|
||||
$title =~ s/[\011-\016;\302-\365]//g;
|
||||
|
||||
my $tmux;
|
||||
if($Global::total_running == 0) {
|
||||
|
|
@ -5378,11 +5417,12 @@ sub tmux_wrap {
|
|||
}
|
||||
return "mkfifo $tmpfile; $tmux ".
|
||||
# Run in tmux
|
||||
::shell_quote_scalar("(".$actual_command.');(echo $?$status;echo 255) >'.$tmpfile.";".
|
||||
"echo ".::shell_quote_scalar($visual_command).";".
|
||||
"echo \007Job finished at: `date`;sleep 10").
|
||||
# Run outside tmux
|
||||
"; exit `perl -ne '1..1 and print' $tmpfile;rm $tmpfile` ";
|
||||
::shell_quote_scalar(
|
||||
"(".$actual_command.');(echo $?$status;echo 255) >'.$tmpfile.";".
|
||||
"echo ".::shell_quote_scalar($visual_command).";".
|
||||
"echo \007Job finished at: `date`;sleep 10").
|
||||
# Run outside tmux
|
||||
"; exit `perl -ne 'unlink $ARGV; 1..1 and print' $tmpfile;rm $tmpfile` ";
|
||||
}
|
||||
|
||||
sub is_already_in_results {
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ B<exportf> to export and to set $SHELL to bash:
|
|||
exportf my_func
|
||||
SHELL=/bin/bash parallel "my_func {}" ::: 1 2
|
||||
|
||||
The command cannot contain the character \257 (¯).
|
||||
The command cannot contain the character \257 (macron: ¯).
|
||||
|
||||
=item B<{}> (alpha testing)
|
||||
|
||||
|
|
@ -568,6 +568,26 @@ In Bash I<var> can also be a Bash function - just remember to B<export
|
|||
The variable '_' is special. It will copy all enviroment variables
|
||||
except for the ones mentioned in ~/.parallel/ignored_vars.
|
||||
|
||||
To copy Bash arrays you need an importer function, as Bash arrays
|
||||
cannot be exported:
|
||||
|
||||
import_array () {
|
||||
local func=$1; shift;
|
||||
export $func='() {
|
||||
'"$(for arr in $@; do
|
||||
declare -p $arr|sed '1s/declare -./&g/'
|
||||
done)"'
|
||||
}'
|
||||
}
|
||||
|
||||
declare -A assoc='([one]="1" [two]="2")'
|
||||
declare -a indexed='([0]="one" [1]="two")'
|
||||
|
||||
import_array my_importer assoc indexed
|
||||
|
||||
parallel --env my_importer \
|
||||
'my_importer; echo "{}" "${indexed[{}]}" "${assoc[${indexed[{}]}]}"' ::: "${!indexed[@]}"
|
||||
|
||||
See also: B<--record-env>.
|
||||
|
||||
|
||||
|
|
@ -612,7 +632,7 @@ B<parallel> uses GNU B<parallel> to compute this, so you will get an
|
|||
infinite loop. This will likely be fixed in a later release.
|
||||
|
||||
|
||||
=item B<--gnu>
|
||||
=item B<--gnu>
|
||||
|
||||
Behave like GNU B<parallel>. If B<--tollef> and B<--gnu> are both set,
|
||||
B<--gnu> takes precedence. B<--tollef> is retired, but B<--gnu> is
|
||||
|
|
@ -1094,7 +1114,7 @@ with 'y' or 'Y'. Implies B<-t>.
|
|||
|
||||
Use to define start and end parenthesis for B<{= perl expression =}>. The
|
||||
left and the right parenthesis can be multiple characters and are
|
||||
assumed to be the same length. The default is B<{==}> giving
|
||||
assumed to be the same length. The default is B<{==}> giving
|
||||
B<{=> as the start parenthesis and B<=}> as the end parenthesis.
|
||||
|
||||
Another useful setting is B<,,,,> which would make both parenthesis
|
||||
|
|
@ -1855,7 +1875,7 @@ Compare these two:
|
|||
|
||||
Arguments will be recycled if one input source has more arguments than the others:
|
||||
|
||||
parallel --xapply echo {1} {2} {3} ::: 1 2 ::: I II III ::: a b c d e f g
|
||||
parallel --xapply echo {1} {2} {3} ::: 1 2 ::: I II III ::: a b c d e f g
|
||||
|
||||
See also B<--header>.
|
||||
|
||||
|
|
@ -2459,7 +2479,7 @@ time and be warned if they are ever changed. To do that:
|
|||
(echo 'Host *'; echo StrictHostKeyChecking no) >> .ssh/config
|
||||
parallel --slf my_cluster --nonall true
|
||||
# Remove the disabling of StrictHostKeyChecking
|
||||
mv .ssh/config.backup .ssh/config
|
||||
mv .ssh/config.backup .ssh/config
|
||||
|
||||
The servers in B<.parallel/my_cluster> are now added in B<.ssh/known_hosts>.
|
||||
|
||||
|
|
|
|||
4510
src/parallel.texi
4510
src/parallel.texi
File diff suppressed because it is too large
Load diff
BIN
src/sem.pdf
BIN
src/sem.pdf
Binary file not shown.
359
src/sem.texi
359
src/sem.texi
|
|
@ -1,359 +0,0 @@
|
|||
\input texinfo
|
||||
@setfilename sem.info
|
||||
|
||||
@documentencoding utf-8
|
||||
|
||||
@settitle sem - semaphore for executing shell command lines in parallel
|
||||
|
||||
@node Top
|
||||
@top sem
|
||||
|
||||
@menu
|
||||
* NAME::
|
||||
* SYNOPSIS::
|
||||
* DESCRIPTION::
|
||||
* OPTIONS::
|
||||
* EXAMPLE@asis{:} Gzipping *.log::
|
||||
* EXAMPLE@asis{:} Protecting pod2html from itself::
|
||||
* BUGS::
|
||||
* REPORTING BUGS::
|
||||
* AUTHOR::
|
||||
* LICENSE::
|
||||
* DEPENDENCIES::
|
||||
* SEE ALSO::
|
||||
@end menu
|
||||
|
||||
@node NAME
|
||||
@chapter NAME
|
||||
|
||||
sem - semaphore for executing shell command lines in parallel
|
||||
|
||||
@node SYNOPSIS
|
||||
@chapter SYNOPSIS
|
||||
|
||||
@strong{sem} [--fg] [--id <id>] [--timeout <secs>] [-j <num>] [--wait] command
|
||||
|
||||
@node DESCRIPTION
|
||||
@chapter DESCRIPTION
|
||||
|
||||
GNU @strong{sem} is an alias for GNU @strong{parallel --semaphore}.
|
||||
|
||||
It works as a tool for executing shell commands in parallel. GNU
|
||||
@strong{sem} acts as a counting semaphore. When GNU @strong{sem} is called with
|
||||
command it will start the command in the background. When @emph{num}
|
||||
number of commands are running in the background, GNU @strong{sem} will wait
|
||||
for one of these to complete before starting another command.
|
||||
|
||||
Before looking at the options you may want to check out the examples
|
||||
after the list of options. That will give you an idea of what GNU
|
||||
@strong{sem} is capable of.
|
||||
|
||||
@node OPTIONS
|
||||
@chapter OPTIONS
|
||||
|
||||
@table @asis
|
||||
@item @emph{command}
|
||||
@anchor{@emph{command}}
|
||||
|
||||
Command to execute. The command may be followed by arguments for the command.
|
||||
|
||||
@item @strong{--bg}
|
||||
@anchor{@strong{--bg}}
|
||||
|
||||
Run command in background thus GNU @strong{parallel} will not wait for
|
||||
completion of the command before exiting. This is the default.
|
||||
|
||||
See also: @strong{--fg}
|
||||
|
||||
@item @strong{-j} @emph{N}
|
||||
@anchor{@strong{-j} @emph{N}}
|
||||
|
||||
Run up to N commands in parallel. Default is 1 thus acting like a
|
||||
mutex.
|
||||
|
||||
@item @strong{--jobs} @emph{N}
|
||||
@anchor{@strong{--jobs} @emph{N}}
|
||||
|
||||
@item @strong{-j} @emph{N}
|
||||
@anchor{@strong{-j} @emph{N} 1}
|
||||
|
||||
@item @strong{--max-procs} @emph{N}
|
||||
@anchor{@strong{--max-procs} @emph{N}}
|
||||
|
||||
@item @strong{-P} @emph{N}
|
||||
@anchor{@strong{-P} @emph{N}}
|
||||
|
||||
Run up to N commands in parallel. Default is 1 thus acting like a
|
||||
mutex.
|
||||
|
||||
@item @strong{--jobs} @emph{+N}
|
||||
@anchor{@strong{--jobs} @emph{+N}}
|
||||
|
||||
@item @strong{-j} @emph{+N}
|
||||
@anchor{@strong{-j} @emph{+N}}
|
||||
|
||||
@item @strong{--max-procs} @emph{+N}
|
||||
@anchor{@strong{--max-procs} @emph{+N}}
|
||||
|
||||
@item @strong{-P} @emph{+N}
|
||||
@anchor{@strong{-P} @emph{+N}}
|
||||
|
||||
Add N to the number of CPU cores. Run up to this many jobs in
|
||||
parallel. For compute intensive jobs @strong{-j} +0 is useful as it will run
|
||||
number-of-cpu-cores jobs simultaneously.
|
||||
|
||||
@item @strong{--jobs} @emph{-N}
|
||||
@anchor{@strong{--jobs} @emph{-N}}
|
||||
|
||||
@item @strong{-j} @emph{-N}
|
||||
@anchor{@strong{-j} @emph{-N}}
|
||||
|
||||
@item @strong{--max-procs} @emph{-N}
|
||||
@anchor{@strong{--max-procs} @emph{-N}}
|
||||
|
||||
@item @strong{-P} @emph{-N}
|
||||
@anchor{@strong{-P} @emph{-N}}
|
||||
|
||||
Subtract N from the number of CPU cores. Run up to this many jobs in
|
||||
parallel. If the evaluated number is less than 1 then 1 will be used.
|
||||
See also @strong{--use-cpus-instead-of-cores}.
|
||||
|
||||
@item @strong{--jobs} @emph{N}%
|
||||
@anchor{@strong{--jobs} @emph{N}%}
|
||||
|
||||
@item @strong{-j} @emph{N}%
|
||||
@anchor{@strong{-j} @emph{N}%}
|
||||
|
||||
@item @strong{--max-procs} @emph{N}%
|
||||
@anchor{@strong{--max-procs} @emph{N}%}
|
||||
|
||||
@item @strong{-P} @emph{N}%
|
||||
@anchor{@strong{-P} @emph{N}%}
|
||||
|
||||
Multiply N% with the number of CPU cores. Run up to this many jobs in
|
||||
parallel. If the evaluated number is less than 1 then 1 will be used.
|
||||
See also @strong{--use-cpus-instead-of-cores}.
|
||||
|
||||
@item @strong{--jobs} @emph{procfile}
|
||||
@anchor{@strong{--jobs} @emph{procfile}}
|
||||
|
||||
@item @strong{-j} @emph{procfile}
|
||||
@anchor{@strong{-j} @emph{procfile}}
|
||||
|
||||
@item @strong{--max-procs} @emph{procfile}
|
||||
@anchor{@strong{--max-procs} @emph{procfile}}
|
||||
|
||||
@item @strong{-P} @emph{procfile}
|
||||
@anchor{@strong{-P} @emph{procfile}}
|
||||
|
||||
Read parameter from file. Use the content of @emph{procfile} as parameter
|
||||
for @emph{-j}. E.g. @emph{procfile} could contain the string 100% or +2 or
|
||||
10.
|
||||
|
||||
@item @strong{--semaphorename} @emph{name}
|
||||
@anchor{@strong{--semaphorename} @emph{name}}
|
||||
|
||||
@item @strong{--id} @emph{name}
|
||||
@anchor{@strong{--id} @emph{name}}
|
||||
|
||||
Use @strong{name} as the name of the semaphore. Default is the name of the
|
||||
controlling tty (output from @strong{tty}).
|
||||
|
||||
The default normally works as expected when used interactively, but
|
||||
when used in a script @emph{name} should be set. @emph{$$} or @emph{my_task_name}
|
||||
are often a good value.
|
||||
|
||||
The semaphore is stored in ~/.parallel/semaphores/
|
||||
|
||||
@item @strong{--fg}
|
||||
@anchor{@strong{--fg}}
|
||||
|
||||
Do not put command in background.
|
||||
|
||||
@item @strong{--timeout} @emph{secs} (not implemented)
|
||||
@anchor{@strong{--timeout} @emph{secs} (not implemented)}
|
||||
|
||||
@item @strong{-t} @emph{secs} (not implemented)
|
||||
@anchor{@strong{-t} @emph{secs} (not implemented)}
|
||||
|
||||
If the semaphore is not released within @emph{secs} seconds, take it anyway.
|
||||
|
||||
@item @strong{--wait}
|
||||
@anchor{@strong{--wait}}
|
||||
|
||||
@item @strong{-w}
|
||||
@anchor{@strong{-w}}
|
||||
|
||||
Wait for all commands to complete.
|
||||
|
||||
@end table
|
||||
|
||||
@node EXAMPLE: Gzipping *.log
|
||||
@chapter EXAMPLE: Gzipping *.log
|
||||
|
||||
Run one gzip process per CPU core. Block until a CPU core becomes
|
||||
available.
|
||||
|
||||
@verbatim
|
||||
for i in *.log ; do
|
||||
echo $i
|
||||
sem -j+0 gzip $i ";" echo done
|
||||
done
|
||||
sem --wait
|
||||
@end verbatim
|
||||
|
||||
@node EXAMPLE: Protecting pod2html from itself
|
||||
@chapter EXAMPLE: Protecting pod2html from itself
|
||||
|
||||
pod2html creates two files: pod2htmd.tmp and pod2htmi.tmp which it
|
||||
does not clean up. It uses these two files for a short time. But if
|
||||
you run multiple pod2html in parallel (e.g. in a Makefile with make
|
||||
-j) you need to protect pod2html from running twice at the same
|
||||
time. @strong{sem} running as a mutex will do just that:
|
||||
|
||||
@verbatim
|
||||
sem --fg --id pod2html pod2html foo.pod > foo.html
|
||||
sem --fg --id pod2html rm -f pod2htmd.tmp pod2htmi.tmp
|
||||
@end verbatim
|
||||
|
||||
@node BUGS
|
||||
@chapter BUGS
|
||||
|
||||
None known.
|
||||
|
||||
@node REPORTING BUGS
|
||||
@chapter REPORTING BUGS
|
||||
|
||||
Report bugs to <bug-parallel@@gnu.org>.
|
||||
|
||||
@node AUTHOR
|
||||
@chapter AUTHOR
|
||||
|
||||
Copyright (C) 2010,2011,2012,2013 Ole Tange, http://ole.tange.dk and Free
|
||||
Software Foundation, Inc.
|
||||
|
||||
@node LICENSE
|
||||
@chapter LICENSE
|
||||
|
||||
Copyright (C) 2010,2011,2012,2013 Free Software Foundation, Inc.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 3 of the License, or
|
||||
at your option any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
@menu
|
||||
* Documentation license I::
|
||||
* Documentation license II::
|
||||
@end menu
|
||||
|
||||
@node Documentation license I
|
||||
@section Documentation license I
|
||||
|
||||
Permission is granted to copy, distribute and/or modify this documentation
|
||||
under the terms of the GNU Free Documentation License, Version 1.3 or
|
||||
any later version published by the Free Software Foundation; with no
|
||||
Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
|
||||
Texts. A copy of the license is included in the file fdl.txt.
|
||||
|
||||
@node Documentation license II
|
||||
@section Documentation license II
|
||||
|
||||
You are free:
|
||||
|
||||
@table @asis
|
||||
@item @strong{to Share}
|
||||
@anchor{@strong{to Share}}
|
||||
|
||||
to copy, distribute and transmit the work
|
||||
|
||||
@item @strong{to Remix}
|
||||
@anchor{@strong{to Remix}}
|
||||
|
||||
to adapt the work
|
||||
|
||||
@end table
|
||||
|
||||
Under the following conditions:
|
||||
|
||||
@table @asis
|
||||
@item @strong{Attribution}
|
||||
@anchor{@strong{Attribution}}
|
||||
|
||||
You must attribute the work in the manner specified by the author or
|
||||
licensor (but not in any way that suggests that they endorse you or
|
||||
your use of the work).
|
||||
|
||||
@item @strong{Share Alike}
|
||||
@anchor{@strong{Share Alike}}
|
||||
|
||||
If you alter, transform, or build upon this work, you may distribute
|
||||
the resulting work only under the same, similar or a compatible
|
||||
license.
|
||||
|
||||
@end table
|
||||
|
||||
With the understanding that:
|
||||
|
||||
@table @asis
|
||||
@item @strong{Waiver}
|
||||
@anchor{@strong{Waiver}}
|
||||
|
||||
Any of the above conditions can be waived if you get permission from
|
||||
the copyright holder.
|
||||
|
||||
@item @strong{Public Domain}
|
||||
@anchor{@strong{Public Domain}}
|
||||
|
||||
Where the work or any of its elements is in the public domain under
|
||||
applicable law, that status is in no way affected by the license.
|
||||
|
||||
@item @strong{Other Rights}
|
||||
@anchor{@strong{Other Rights}}
|
||||
|
||||
In no way are any of the following rights affected by the license:
|
||||
|
||||
@itemize
|
||||
@item Your fair dealing or fair use rights, or other applicable
|
||||
copyright exceptions and limitations;
|
||||
|
||||
@item The author's moral rights;
|
||||
|
||||
@item Rights other persons may have either in the work itself or in
|
||||
how the work is used, such as publicity or privacy rights.
|
||||
|
||||
@end itemize
|
||||
|
||||
@end table
|
||||
|
||||
@table @asis
|
||||
@item @strong{Notice}
|
||||
@anchor{@strong{Notice}}
|
||||
|
||||
For any reuse or distribution, you must make clear to others the
|
||||
license terms of this work.
|
||||
|
||||
@end table
|
||||
|
||||
A copy of the full license is included in the file as cc-by-sa.txt.
|
||||
|
||||
@node DEPENDENCIES
|
||||
@chapter DEPENDENCIES
|
||||
|
||||
GNU @strong{sem} uses Perl, and the Perl modules Getopt::Long,
|
||||
Symbol, Fcntl.
|
||||
|
||||
@node SEE ALSO
|
||||
@chapter SEE ALSO
|
||||
|
||||
@strong{parallel}(1)
|
||||
|
||||
@bye
|
||||
540
src/sql.texi
540
src/sql.texi
|
|
@ -1,540 +0,0 @@
|
|||
\input texinfo
|
||||
@setfilename sql.info
|
||||
|
||||
@documentencoding utf-8
|
||||
|
||||
@settitle sql - execute a command on a database determined by a dburl
|
||||
|
||||
@node Top
|
||||
@top sql
|
||||
|
||||
@menu
|
||||
* NAME::
|
||||
* SYNOPSIS::
|
||||
* DESCRIPTION::
|
||||
* DBURL::
|
||||
* EXAMPLES::
|
||||
* REPORTING BUGS::
|
||||
* AUTHOR::
|
||||
* LICENSE::
|
||||
* DEPENDENCIES::
|
||||
* FILES::
|
||||
* SEE ALSO::
|
||||
@end menu
|
||||
|
||||
@node NAME
|
||||
@chapter NAME
|
||||
|
||||
sql - execute a command on a database determined by a dburl
|
||||
|
||||
@node SYNOPSIS
|
||||
@chapter SYNOPSIS
|
||||
|
||||
@strong{sql} [options] @emph{dburl} [@emph{commands}]
|
||||
|
||||
@strong{sql} [options] @emph{dburl} < commandfile
|
||||
|
||||
@strong{#!/usr/bin/sql} @strong{--shebang} [options] @emph{dburl}
|
||||
|
||||
@node DESCRIPTION
|
||||
@chapter DESCRIPTION
|
||||
|
||||
GNU @strong{sql} aims to give a simple, unified interface for accessing
|
||||
databases through all the different databases' command line
|
||||
clients. So far the focus has been on giving a common way to specify
|
||||
login information (protocol, username, password, hostname, and port
|
||||
number), size (database and table size), and running queries.
|
||||
|
||||
The database is addressed using a DBURL. If @emph{commands} are left out
|
||||
you will get that database's interactive shell.
|
||||
|
||||
GNU @strong{sql} is often used in combination with GNU @strong{parallel}.
|
||||
|
||||
@table @asis
|
||||
@item @emph{dburl}
|
||||
@anchor{@emph{dburl}}
|
||||
|
||||
A DBURL has the following syntax:
|
||||
[sql:]vendor://
|
||||
[[user][:password]@@][host][:port]/[database][?sqlquery]
|
||||
|
||||
See the section DBURL below.
|
||||
|
||||
@item @emph{commands}
|
||||
@anchor{@emph{commands}}
|
||||
|
||||
The SQL commands to run. Each argument will have a newline
|
||||
appended.
|
||||
|
||||
Example: "SELECT * FROM foo;" "SELECT * FROM bar;"
|
||||
|
||||
If the arguments contain '\n' or '\x0a' this will be replaced with a
|
||||
newline:
|
||||
|
||||
Example: "SELECT * FROM foo;\n SELECT * FROM bar;"
|
||||
|
||||
If no commands are given SQL is read from the keyboard or STDIN.
|
||||
|
||||
Example: echo 'SELECT * FROM foo;' | sql mysql:///
|
||||
|
||||
@item @strong{--db-size}
|
||||
@anchor{@strong{--db-size}}
|
||||
|
||||
@item @strong{--dbsize}
|
||||
@anchor{@strong{--dbsize}}
|
||||
|
||||
Size of database. Show the size of the database on disk. For Oracle
|
||||
this requires access to read the table @emph{dba_data_files} - the user
|
||||
@emph{system} has that.
|
||||
|
||||
@item @strong{--help}
|
||||
@anchor{@strong{--help}}
|
||||
|
||||
@item @strong{-h}
|
||||
@anchor{@strong{-h}}
|
||||
|
||||
Print a summary of the options to GNU @strong{sql} and exit.
|
||||
|
||||
@item @strong{--html}
|
||||
@anchor{@strong{--html}}
|
||||
|
||||
HTML output. Turn on HTML tabular output.
|
||||
|
||||
@item @strong{--show-processlist}
|
||||
@anchor{@strong{--show-processlist}}
|
||||
|
||||
@item @strong{--proclist}
|
||||
@anchor{@strong{--proclist}}
|
||||
|
||||
@item @strong{--listproc}
|
||||
@anchor{@strong{--listproc}}
|
||||
|
||||
Show the list of running queries.
|
||||
|
||||
@item @strong{--show-databases}
|
||||
@anchor{@strong{--show-databases}}
|
||||
|
||||
@item @strong{--showdbs}
|
||||
@anchor{@strong{--showdbs}}
|
||||
|
||||
@item @strong{--list-databases}
|
||||
@anchor{@strong{--list-databases}}
|
||||
|
||||
@item @strong{--listdbs}
|
||||
@anchor{@strong{--listdbs}}
|
||||
|
||||
List the databases (table spaces) in the database.
|
||||
|
||||
@item @strong{--show-tables}
|
||||
@anchor{@strong{--show-tables}}
|
||||
|
||||
@item @strong{--list-tables}
|
||||
@anchor{@strong{--list-tables}}
|
||||
|
||||
@item @strong{--table-list}
|
||||
@anchor{@strong{--table-list}}
|
||||
|
||||
List the tables in the database.
|
||||
|
||||
@item @strong{--noheaders}
|
||||
@anchor{@strong{--noheaders}}
|
||||
|
||||
@item @strong{--no-headers}
|
||||
@anchor{@strong{--no-headers}}
|
||||
|
||||
@item @strong{-n}
|
||||
@anchor{@strong{-n}}
|
||||
|
||||
Remove headers and footers and print only tuples. Bug in Oracle: it
|
||||
still prints number of rows found.
|
||||
|
||||
@item @strong{-p} @emph{pass-through}
|
||||
@anchor{@strong{-p} @emph{pass-through}}
|
||||
|
||||
The string following -p will be given to the database connection
|
||||
program as arguments. Multiple -p's will be joined with
|
||||
space. Example: pass '-U' and the user name to the program:
|
||||
|
||||
@emph{-p "-U scott"} can also be written @emph{-p -U -p scott}.
|
||||
|
||||
@item @strong{-r}
|
||||
@anchor{@strong{-r}}
|
||||
|
||||
Try 3 times. Short version of @emph{--retries 3}.
|
||||
|
||||
@item @strong{--retries} @emph{ntimes}
|
||||
@anchor{@strong{--retries} @emph{ntimes}}
|
||||
|
||||
Try @emph{ntimes} times. If the client program returns with an error,
|
||||
retry the command. Default is @emph{--retries 1}.
|
||||
|
||||
@item @strong{--sep} @emph{string}
|
||||
@anchor{@strong{--sep} @emph{string}}
|
||||
|
||||
@item @strong{-s} @emph{string}
|
||||
@anchor{@strong{-s} @emph{string}}
|
||||
|
||||
Field separator. Use @emph{string} as separator between columns.
|
||||
|
||||
@item @strong{--skip-first-line}
|
||||
@anchor{@strong{--skip-first-line}}
|
||||
|
||||
Do not use the first line of input (used by GNU @strong{sql} itself
|
||||
when called with @strong{--shebang}).
|
||||
|
||||
@item @strong{--table-size}
|
||||
@anchor{@strong{--table-size}}
|
||||
|
||||
@item @strong{--tablesize}
|
||||
@anchor{@strong{--tablesize}}
|
||||
|
||||
Size of tables. Show the size of the tables in the database.
|
||||
|
||||
@item @strong{--verbose}
|
||||
@anchor{@strong{--verbose}}
|
||||
|
||||
@item @strong{-v}
|
||||
@anchor{@strong{-v}}
|
||||
|
||||
Print which command is sent.
|
||||
|
||||
@item @strong{--version}
|
||||
@anchor{@strong{--version}}
|
||||
|
||||
@item @strong{-V}
|
||||
@anchor{@strong{-V}}
|
||||
|
||||
Print the version GNU @strong{sql} and exit.
|
||||
|
||||
@item @strong{--shebang}
|
||||
@anchor{@strong{--shebang}}
|
||||
|
||||
@item @strong{-Y}
|
||||
@anchor{@strong{-Y}}
|
||||
|
||||
GNU @strong{sql} can be called as a shebang (#!) command as the first line of a script. Like this:
|
||||
|
||||
@verbatim
|
||||
#!/usr/bin/sql -Y mysql:///
|
||||
|
||||
SELECT * FROM foo;
|
||||
@end verbatim
|
||||
|
||||
For this to work @strong{--shebang} or @strong{-Y} must be set as the first option.
|
||||
|
||||
@end table
|
||||
|
||||
@node DBURL
|
||||
@chapter DBURL
|
||||
|
||||
A DBURL has the following syntax:
|
||||
[sql:]vendor://
|
||||
[[user][:password]@@][host][:port]/[database][?sqlquery]
|
||||
|
||||
To quote special characters use %-encoding specified in
|
||||
http://tools.ietf.org/html/rfc3986#section-2.1 (E.g. a password
|
||||
containing '/' would contain '%2F').
|
||||
|
||||
Examples:
|
||||
mysql://scott:tiger@@my.example.com/mydb
|
||||
sql:oracle://scott:tiger@@ora.example.com/xe
|
||||
postgresql://scott:tiger@@pg.example.com/pgdb
|
||||
pg:///
|
||||
postgresqlssl://scott@@pg.example.com:3333/pgdb
|
||||
sql:sqlite2:////tmp/db.sqlite?SELECT * FROM foo;
|
||||
sqlite3:///../db.sqlite3?SELECT%20*%20FROM%20foo;
|
||||
|
||||
Currently supported vendors: MySQL (mysql), MySQL with SSL (mysqls,
|
||||
mysqlssl), Oracle (oracle, ora), PostgreSQL (postgresql, pg, pgsql,
|
||||
postgres), PostgreSQL with SSL (postgresqlssl, pgs, pgsqlssl,
|
||||
postgresssl, pgssl, postgresqls, pgsqls, postgress), SQLite2 (sqlite,
|
||||
sqlite2), SQLite3 (sqlite3).
|
||||
|
||||
Aliases must start with ':' and are read from
|
||||
/etc/sql/aliases and ~/.sql/aliases. The user's own
|
||||
~/.sql/aliases should only be readable by the user.
|
||||
|
||||
Example of aliases:
|
||||
|
||||
@verbatim
|
||||
:myalias1 pg://scott:tiger@pg.example.com/pgdb
|
||||
:myalias2 ora://scott:tiger@ora.example.com/xe
|
||||
# Short form of mysql://`whoami`:nopassword@localhost:3306/`whoami`
|
||||
:myalias3 mysql:///
|
||||
# Short form of mysql://`whoami`:nopassword@localhost:33333/mydb
|
||||
:myalias4 mysql://:33333/mydb
|
||||
# Alias for an alias
|
||||
:m :myalias4
|
||||
# the sortest alias possible
|
||||
: sqlite2:////tmp/db.sqlite
|
||||
# Including an SQL query
|
||||
:query sqlite:////tmp/db.sqlite?SELECT * FROM foo;
|
||||
@end verbatim
|
||||
|
||||
@node EXAMPLES
|
||||
@chapter EXAMPLES
|
||||
|
||||
@menu
|
||||
* Get an interactive prompt::
|
||||
* Run a query::
|
||||
* Copy a PostgreSQL database::
|
||||
* Empty all tables in a MySQL database::
|
||||
* Drop all tables in a PostgreSQL database::
|
||||
* Run as a script::
|
||||
* Use --colsep to process multiple columns::
|
||||
* Retry if the connection fails::
|
||||
* Get info about the running database system::
|
||||
@end menu
|
||||
|
||||
@node Get an interactive prompt
|
||||
@section Get an interactive prompt
|
||||
|
||||
The most basic use of GNU @strong{sql} is to get an interactive prompt:
|
||||
|
||||
@strong{sql sql:oracle://scott:tiger@@ora.example.com/xe}
|
||||
|
||||
If you have setup an alias you can do:
|
||||
|
||||
@strong{sql :myora}
|
||||
|
||||
@node Run a query
|
||||
@section Run a query
|
||||
|
||||
To run a query directly from the command line:
|
||||
|
||||
@strong{sql :myalias "SELECT * FROM foo;"}
|
||||
|
||||
Oracle requires newlines after each statement. This can be done like
|
||||
this:
|
||||
|
||||
@strong{sql :myora "SELECT * FROM foo;" "SELECT * FROM bar;"}
|
||||
|
||||
Or this:
|
||||
|
||||
@strong{sql :myora "SELECT * FROM foo;\nSELECT * FROM bar;"}
|
||||
|
||||
@node Copy a PostgreSQL database
|
||||
@section Copy a PostgreSQL database
|
||||
|
||||
To copy a PostgreSQL database use pg_dump to generate the dump and GNU
|
||||
@strong{sql} to import it:
|
||||
|
||||
@strong{pg_dump pg_database | sql pg://scott:tiger@@pg.example.com/pgdb}
|
||||
|
||||
@node Empty all tables in a MySQL database
|
||||
@section Empty all tables in a MySQL database
|
||||
|
||||
Using GNU @strong{parallel} it is easy to empty all tables without dropping them:
|
||||
|
||||
@strong{sql -n mysql:/// 'show tables' | parallel sql mysql:/// DELETE FROM @{@};}
|
||||
|
||||
@node Drop all tables in a PostgreSQL database
|
||||
@section Drop all tables in a PostgreSQL database
|
||||
|
||||
To drop all tables in a PostgreSQL database do:
|
||||
|
||||
@strong{sql -n pg:/// '\dt' | parallel --colsep '\|' -r sql pg:/// DROP TABLE @{2@};}
|
||||
|
||||
@node Run as a script
|
||||
@section Run as a script
|
||||
|
||||
Instead of doing:
|
||||
|
||||
@strong{sql mysql:/// < sqlfile}
|
||||
|
||||
you can combine the sqlfile with the DBURL to make a
|
||||
UNIX-script. Create a script called @emph{demosql}:
|
||||
|
||||
@strong{#!/usr/bin/sql -Y mysql:///}
|
||||
|
||||
@strong{SELECT * FROM foo;}
|
||||
|
||||
Then do:
|
||||
|
||||
@strong{chmod +x demosql; ./demosql}
|
||||
|
||||
@node Use --colsep to process multiple columns
|
||||
@section Use --colsep to process multiple columns
|
||||
|
||||
Use GNU @strong{parallel}'s @strong{--colsep} to separate columns:
|
||||
|
||||
@strong{sql -s '\t' :myalias 'SELECT * FROM foo;' | parallel --colsep '\t' do_stuff @{4@} @{1@}}
|
||||
|
||||
@node Retry if the connection fails
|
||||
@section Retry if the connection fails
|
||||
|
||||
If the access to the database fails occasionally @strong{--retries} can help
|
||||
make sure the query succeeds:
|
||||
|
||||
@strong{sql --retries 5 :myalias 'SELECT * FROM really_big_foo;'}
|
||||
|
||||
@node Get info about the running database system
|
||||
@section Get info about the running database system
|
||||
|
||||
Show how big the database is:
|
||||
|
||||
@strong{sql --db-size :myalias}
|
||||
|
||||
List the tables:
|
||||
|
||||
@strong{sql --list-tables :myalias}
|
||||
|
||||
List the size of the tables:
|
||||
|
||||
@strong{sql --table-size :myalias}
|
||||
|
||||
List the running processes:
|
||||
|
||||
@strong{sql --show-processlist :myalias}
|
||||
|
||||
@node REPORTING BUGS
|
||||
@chapter REPORTING BUGS
|
||||
|
||||
GNU @strong{sql} is part of GNU @strong{parallel}. Report bugs to <bug-parallel@@gnu.org>.
|
||||
|
||||
@node AUTHOR
|
||||
@chapter AUTHOR
|
||||
|
||||
When using GNU @strong{sql} for a publication please cite:
|
||||
|
||||
O. Tange (2011): GNU SQL - A Command Line Tool for Accessing Different
|
||||
Databases Using DBURLs, ;login: The USENIX Magazine, April 2011:29-32.
|
||||
|
||||
Copyright (C) 2008,2009,2010 Ole Tange http://ole.tange.dk
|
||||
|
||||
Copyright (C) 2010,2011 Ole Tange, http://ole.tange.dk and Free
|
||||
Software Foundation, Inc.
|
||||
|
||||
@node LICENSE
|
||||
@chapter LICENSE
|
||||
|
||||
Copyright (C) 2007,2008,2009,2010,2011 Free Software Foundation, Inc.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 3 of the License, or
|
||||
at your option any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
@menu
|
||||
* Documentation license I::
|
||||
* Documentation license II::
|
||||
@end menu
|
||||
|
||||
@node Documentation license I
|
||||
@section Documentation license I
|
||||
|
||||
Permission is granted to copy, distribute and/or modify this documentation
|
||||
under the terms of the GNU Free Documentation License, Version 1.3 or
|
||||
any later version published by the Free Software Foundation; with no
|
||||
Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
|
||||
Texts. A copy of the license is included in the file fdl.txt.
|
||||
|
||||
@node Documentation license II
|
||||
@section Documentation license II
|
||||
|
||||
You are free:
|
||||
|
||||
@table @asis
|
||||
@item @strong{to Share}
|
||||
@anchor{@strong{to Share}}
|
||||
|
||||
to copy, distribute and transmit the work
|
||||
|
||||
@item @strong{to Remix}
|
||||
@anchor{@strong{to Remix}}
|
||||
|
||||
to adapt the work
|
||||
|
||||
@end table
|
||||
|
||||
Under the following conditions:
|
||||
|
||||
@table @asis
|
||||
@item @strong{Attribution}
|
||||
@anchor{@strong{Attribution}}
|
||||
|
||||
You must attribute the work in the manner specified by the author or
|
||||
licensor (but not in any way that suggests that they endorse you or
|
||||
your use of the work).
|
||||
|
||||
@item @strong{Share Alike}
|
||||
@anchor{@strong{Share Alike}}
|
||||
|
||||
If you alter, transform, or build upon this work, you may distribute
|
||||
the resulting work only under the same, similar or a compatible
|
||||
license.
|
||||
|
||||
@end table
|
||||
|
||||
With the understanding that:
|
||||
|
||||
@table @asis
|
||||
@item @strong{Waiver}
|
||||
@anchor{@strong{Waiver}}
|
||||
|
||||
Any of the above conditions can be waived if you get permission from
|
||||
the copyright holder.
|
||||
|
||||
@item @strong{Public Domain}
|
||||
@anchor{@strong{Public Domain}}
|
||||
|
||||
Where the work or any of its elements is in the public domain under
|
||||
applicable law, that status is in no way affected by the license.
|
||||
|
||||
@item @strong{Other Rights}
|
||||
@anchor{@strong{Other Rights}}
|
||||
|
||||
In no way are any of the following rights affected by the license:
|
||||
|
||||
@itemize
|
||||
@item Your fair dealing or fair use rights, or other applicable
|
||||
copyright exceptions and limitations;
|
||||
|
||||
@item The author's moral rights;
|
||||
|
||||
@item Rights other persons may have either in the work itself or in
|
||||
how the work is used, such as publicity or privacy rights.
|
||||
|
||||
@end itemize
|
||||
|
||||
@item @strong{Notice}
|
||||
@anchor{@strong{Notice}}
|
||||
|
||||
For any reuse or distribution, you must make clear to others the
|
||||
license terms of this work.
|
||||
|
||||
@end table
|
||||
|
||||
A copy of the full license is included in the file as cc-by-sa.txt.
|
||||
|
||||
@node DEPENDENCIES
|
||||
@chapter DEPENDENCIES
|
||||
|
||||
GNU @strong{sql} uses Perl. If @strong{mysql} is installed, MySQL dburls will
|
||||
work. If @strong{psql} is installed, PostgreSQL dburls will work. If
|
||||
@strong{sqlite} is installed, SQLite2 dburls will work. If @strong{sqlite3} is
|
||||
installed, SQLite3 dburls will work. If @strong{sqlplus} is installed,
|
||||
Oracle dburls will work. If @strong{rlwrap} is installed, GNU @strong{sql} will
|
||||
have a command history for Oracle.
|
||||
|
||||
@node FILES
|
||||
@chapter FILES
|
||||
|
||||
~/.sql/aliases - user's own aliases with DBURLs
|
||||
|
||||
/etc/sql/aliases - common aliases with DBURLs
|
||||
|
||||
@node SEE ALSO
|
||||
@chapter SEE ALSO
|
||||
|
||||
@strong{mysql}(1), @strong{psql}(1), @strong{rlwrap}(1), @strong{sqlite}(1), @strong{sqlite3}(1), @strong{sqlplus}(1)
|
||||
|
||||
@bye
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
echo '### Test --return of weirdly named file'
|
||||
### Test --return of weirdly named file
|
||||
stdout parallel --return {} -vv -S parallel\@parallel-server3 echo '>'{} ::: 'aa<${#}" b'; rm 'aa<${#}" b'
|
||||
ssh -tt -oLogLevel=quiet parallel@parallel-server3 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \>aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b;_EXIT_status=$?; mkdir -p ./.; rsync --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status;
|
||||
ssh -tt -oLogLevel=quiet parallel@parallel-server3 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \>aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh parallel@parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status;
|
||||
echo '### Test if remote login shell is csh'
|
||||
### Test if remote login shell is csh
|
||||
stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c
|
||||
|
|
|
|||
Loading…
Reference in a new issue