Added --sshloginfile .. or -S .. means use ~/.parallel/sshloginfile.

Changed .parallelrc to .parallel/config to avoid having two files in ~/
This commit is contained in:
Ole Tange 2010-08-14 22:51:09 +02:00
parent a038ade0de
commit 48c89d6929
7 changed files with 91 additions and 30 deletions

View file

@ -1,5 +1,7 @@
Default sshloginfile ~/.parallel/sshloginfile Added --sshloginfile .. or -S .. means use ~/.parallel/sshloginfile
--sshloginfile .. or -S .. means use default sshloginfile
Changed .parallelrc to .parallel/config to avoid having two files in ~/
# Allow 7 to run. After then 7th is started, block untill one is dead # Allow 7 to run. After then 7th is started, block untill one is dead
parallel --mutex uniqidentifier -j7 command parallel --mutex uniqidentifier -j7 command

View file

@ -600,6 +600,8 @@ The sshlogin must not require a password.
The sshlogin ':' is special, it means 'no ssh' and will therefore run The sshlogin ':' is special, it means 'no ssh' and will therefore run
on the local computer. on the local computer.
The sshlogin '..' is special, it read sshlogins from ~/.parallel/sshloginfile
To specify more sshlogins separate the sshlogins by comma or repeat To specify more sshlogins separate the sshlogins by comma or repeat
the options multiple times. the options multiple times.
@ -633,6 +635,9 @@ lines. Empty lines and lines starting with '#' are ignored. Example:
When using a different ssh program the last argument must be the hostname. When using a different ssh program the last argument must be the hostname.
The sshloginfile '..' is special, it read sshlogins from
~/.parallel/sshloginfile
=item B<--silent> =item B<--silent>
@ -1329,14 +1334,15 @@ B<-j> take an argument and thus both need to be at the end of a group.
=back =back
=head1 INIT FILE (RC FILE) =head1 CONFIG FILE
The file ~/.parallelrc will be read if it exists. It should be The file ~/.parallel/config will be read if it exists. It should be
formatted like the environment variable $PARALLEL. Lines starting with formatted like the environment variable $PARALLEL. Lines starting with
'#' will be ignored. '#' will be ignored.
Options on the command line takes precedence over the environment Options on the command line takes precedence over the environment
variable $PARALLEL which takes precedence over the file ~/.parallelrc. variable $PARALLEL which takes precedence over the file
~/.parallel/config.
=head1 EXIT STATUS =head1 EXIT STATUS
@ -1945,10 +1951,10 @@ sub parse_options {
$Global::trim = 'n'; $Global::trim = 'n';
Getopt::Long::Configure ("bundling","require_order"); Getopt::Long::Configure ("bundling","require_order");
# Add options from .parallelrc # Add options from .parallel/config
my $parallelrc = $ENV{'HOME'}."/.parallelrc"; my $parallel_config = $ENV{'HOME'}."/.parallel/config";
if(-r $parallelrc) { if(-r $parallel_config) {
open (IN, "<", $parallelrc) || die; open (IN, "<", $parallel_config) || die;
while(<IN>) { while(<IN>) {
/^\s*\#/ and next; /^\s*\#/ and next;
chomp; chomp;
@ -3670,6 +3676,9 @@ sub __REMOTE_SSH__ {}
sub read_sshloginfile { sub read_sshloginfile {
# Returns: N/A # Returns: N/A
my $file = shift; my $file = shift;
if($file eq "..") {
$file = $ENV{'HOME'}."/.parallel/sshloginfile";
}
open(IN, $file) || die "Cannot open $file"; open(IN, $file) || die "Cannot open $file";
while(<IN>) { while(<IN>) {
chomp; chomp;
@ -3684,7 +3693,13 @@ sub parse_sshlogin {
if(not @Global::sshlogin) { @Global::sshlogin = (":"); } if(not @Global::sshlogin) { @Global::sshlogin = (":"); }
for my $sshlogin (@Global::sshlogin) { for my $sshlogin (@Global::sshlogin) {
# Split up -S sshlogin,sshlogin # Split up -S sshlogin,sshlogin
push (@login, (split /,/, $sshlogin)); for my $s (split /,/, $sshlogin) {
if ($s eq "..") {
read_sshloginfile($s);
} else {
push (@login, $s);
}
}
} }
for my $sshlogin (@login) { for my $sshlogin (@login) {
if($sshlogin =~ s:^(\d*)/::) { if($sshlogin =~ s:^(\d*)/::) {

View file

@ -1,12 +1,16 @@
#!/bin/bash #!/bin/bash
PAR=parallel
SERVER1=parallel-server3 SERVER1=parallel-server3
SERVER2=parallel-server2 SERVER2=parallel-server2
echo '### Check -S .. and --serverloginfile ..'
echo $SERVER1 > ~/.parallel/sshloginfile
echo parallel@$SERVER2 >> ~/.parallel/sshloginfile
seq 1 20 | parallel -k -S .. echo
seq 1 20 | parallel -k --sshloginfile .. echo
echo '### Check warning if --transfer but file not found' echo '### Check warning if --transfer but file not found'
echo /tmp/noexistant/file | stdout $PAR -k -S $SERVER1 --transfer echo echo /tmp/noexistant/file | stdout parallel -k -S $SERVER1 --transfer echo
echo '### Transfer for file starting with :' echo '### Transfer for file starting with :'
cd /tmp cd /tmp
@ -18,34 +22,34 @@ cat /tmp/test18 | parallel -j1 --trc {}.{.} -S $SERVER1,parallel@$SERVER2,: \
cat /tmp/test18 | parallel -j1 -k 'cat {}.{.}' cat /tmp/test18 | parallel -j1 -k 'cat {}.{.}'
echo '### Check warning if --transfer but not --sshlogin' echo '### Check warning if --transfer but not --sshlogin'
echo | stdout $PAR -k --transfer echo echo | stdout parallel -k --transfer echo
echo '### Check warning if --return but not --sshlogin' echo '### Check warning if --return but not --sshlogin'
echo | stdout $PAR -k --return {} echo echo | stdout parallel -k --return {} echo
echo '### Check warning if --cleanup but not --sshlogin' echo '### Check warning if --cleanup but not --sshlogin'
echo | stdout $PAR -k --cleanup echo echo | stdout parallel -k --cleanup echo
echo '### Test --sshlogin -S --sshloginfile' echo '### Test --sshlogin -S --sshloginfile'
echo localhost >/tmp/parallel-sshlogin echo localhost >/tmp/parallel-sshlogin
seq 1 3 | $PAR -k --sshlogin 8/$SERVER1 -S "7/ssh -l parallel $SERVER2",: --sshloginfile /tmp/parallel-sshlogin echo seq 1 3 | parallel -k --sshlogin 8/$SERVER1 -S "7/ssh -l parallel $SERVER2",: --sshloginfile /tmp/parallel-sshlogin echo
echo '### Test --sshloginfile with extra content' echo '### Test --sshloginfile with extra content'
echo "2/ssh -l parallel $SERVER2" >>/tmp/parallel-sshlogin echo "2/ssh -l parallel $SERVER2" >>/tmp/parallel-sshlogin
echo ":" >>/tmp/parallel-sshlogin echo ":" >>/tmp/parallel-sshlogin
echo "#2/ssh -l tange nothing" >>/tmp/parallel-sshlogin echo "#2/ssh -l tange nothing" >>/tmp/parallel-sshlogin
seq 1 10 | $PAR -k --sshloginfile /tmp/parallel-sshlogin echo seq 1 10 | parallel -k --sshloginfile /tmp/parallel-sshlogin echo
echo '### Check forced number of CPUs being respected' echo '### Check forced number of CPUs being respected'
stdout seq 1 20 | parallel -k -j+0 -S 1/:,9/$SERVER1 "hostname; echo {} >/dev/null" stdout seq 1 20 | parallel -k -j+0 -S 1/:,9/$SERVER1 "hostname; echo {} >/dev/null"
echo '### Check more than 9 simultaneous sshlogins' echo '### Check more than 9 simultaneous sshlogins'
seq 1 11 | $PAR -k -j0 -S "/ssh $SERVER1" echo seq 1 11 | parallel -k -j0 -S "/ssh $SERVER1" echo
echo '### Check more than 9(relative) simultaneous sshlogins' echo '### Check more than 9(relative) simultaneous sshlogins'
seq 1 11 | $PAR -k -j10000% -S "ssh $SERVER1" echo seq 1 11 | parallel -k -j10000% -S "ssh $SERVER1" echo
echo '### Check -S syntax' echo '### Check -S syntax'
seq 1 11 | $PAR -k -j100% -S "/:" echo seq 1 11 | parallel -k -j100% -S "/:" echo

View file

@ -1,18 +1,17 @@
#!/bin/bash #!/bin/bash
PAR=parallel
SERVER1=parallel-server3 SERVER1=parallel-server3
SERVER2=parallel-server2 SERVER2=parallel-server2
echo '### Test $PARALLEL' echo '### Test $PARALLEL'
echo | PARALLEL=--number-of-cpus $PAR echo | PARALLEL=--number-of-cpus parallel
seq 1 2 | PARALLEL="-S$SERVER1 seq 1 2 | PARALLEL="-S$SERVER1
-Sssh -l parallel $SERVER2 -Sssh -l parallel $SERVER2
-j1" $PAR -kv echo -j1" parallel -kv echo
echo '### Test ~/.parallelrc' echo '### Test ~/.parallel/config'
echo "-S$SERVER1 echo "-S$SERVER1
-Sssh -l parallel $SERVER2 -Sssh -l parallel $SERVER2
-j1" > ~/.parallelrc -j1" > ~/.parallel/config
seq 1 2 | $PAR -kv echo seq 1 2 | parallel -kv echo
rm ~/.parallelrc rm ~/.parallel/config

View file

@ -13,7 +13,7 @@ echo '### Test of eof string on :::'
parallel -E ole echo ::: foo ole bar parallel -E ole echo ::: foo ole bar
echo '### Test of ignore-empty string on :::' echo '### Test of ignore-empty string on :::'
parallel -r echo ::: foo '' ole bar parallel -k -r echo ::: foo '' ole bar
echo '### Test of trailing space continuation' echo '### Test of trailing space continuation'
(echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 echo (echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 echo

View file

@ -1,3 +1,44 @@
### Check -S .. and --serverloginfile ..
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
### Check warning if --transfer but file not found ### Check warning if --transfer but file not found
Warning: /tmp/noexistant/file is not readable and will not be transferred Warning: /tmp/noexistant/file is not readable and will not be transferred
/tmp/noexistant/file /tmp/noexistant/file

View file

@ -4,7 +4,7 @@ ssh -l parallel parallel-server2 echo\ 1;
1 1
ssh parallel-server3 echo\ 2; ssh parallel-server3 echo\ 2;
2 2
### Test ~/.parallelrc ### Test ~/.parallel/config
ssh -l parallel parallel-server2 echo\ 1; ssh -l parallel parallel-server2 echo\ 1;
1 1
ssh parallel-server3 echo\ 2; ssh parallel-server3 echo\ 2;