From 48c89d6929881a0e458461fe89b1dc7f871f5527 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 14 Aug 2010 22:51:09 +0200 Subject: [PATCH] Added --sshloginfile .. or -S .. means use ~/.parallel/sshloginfile. Changed .parallelrc to .parallel/config to avoid having two files in ~/ --- doc/FUTURE_IDEAS | 6 +++-- src/parallel | 31 ++++++++++++++++++------- unittest/tests-to-run/test18.sh | 26 ++++++++++++--------- unittest/tests-to-run/test21.sh | 13 +++++------ unittest/tests-to-run/test29.sh | 2 +- unittest/wanted-results/test18 | 41 +++++++++++++++++++++++++++++++++ unittest/wanted-results/test21 | 2 +- 7 files changed, 91 insertions(+), 30 deletions(-) diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index b6270598..cfbdf526 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,5 +1,7 @@ -Default sshloginfile ~/.parallel/sshloginfile ---sshloginfile .. or -S .. means use default sshloginfile +Added --sshloginfile .. or -S .. means use ~/.parallel/sshloginfile + +Changed .parallelrc to .parallel/config to avoid having two files in ~/ + # Allow 7 to run. After then 7th is started, block untill one is dead parallel --mutex uniqidentifier -j7 command diff --git a/src/parallel b/src/parallel index caa1bce7..9fe4ac7f 100755 --- a/src/parallel +++ b/src/parallel @@ -600,6 +600,8 @@ The sshlogin must not require a password. The sshlogin ':' is special, it means 'no ssh' and will therefore run on the local computer. +The sshlogin '..' is special, it read sshlogins from ~/.parallel/sshloginfile + To specify more sshlogins separate the sshlogins by comma or repeat the options multiple times. @@ -633,6 +635,9 @@ lines. Empty lines and lines starting with '#' are ignored. Example: When using a different ssh program the last argument must be the hostname. +The sshloginfile '..' is special, it read sshlogins from +~/.parallel/sshloginfile + =item B<--silent> @@ -1329,14 +1334,15 @@ B<-j> take an argument and thus both need to be at the end of a group. =back -=head1 INIT FILE (RC FILE) +=head1 CONFIG FILE -The file ~/.parallelrc will be read if it exists. It should be +The file ~/.parallel/config will be read if it exists. It should be formatted like the environment variable $PARALLEL. Lines starting with '#' will be ignored. Options on the command line takes precedence over the environment -variable $PARALLEL which takes precedence over the file ~/.parallelrc. +variable $PARALLEL which takes precedence over the file +~/.parallel/config. =head1 EXIT STATUS @@ -1945,10 +1951,10 @@ sub parse_options { $Global::trim = 'n'; Getopt::Long::Configure ("bundling","require_order"); - # Add options from .parallelrc - my $parallelrc = $ENV{'HOME'}."/.parallelrc"; - if(-r $parallelrc) { - open (IN, "<", $parallelrc) || die; + # Add options from .parallel/config + my $parallel_config = $ENV{'HOME'}."/.parallel/config"; + if(-r $parallel_config) { + open (IN, "<", $parallel_config) || die; while() { /^\s*\#/ and next; chomp; @@ -3670,6 +3676,9 @@ sub __REMOTE_SSH__ {} sub read_sshloginfile { # Returns: N/A my $file = shift; + if($file eq "..") { + $file = $ENV{'HOME'}."/.parallel/sshloginfile"; + } open(IN, $file) || die "Cannot open $file"; while() { chomp; @@ -3684,7 +3693,13 @@ sub parse_sshlogin { if(not @Global::sshlogin) { @Global::sshlogin = (":"); } for my $sshlogin (@Global::sshlogin) { # Split up -S sshlogin,sshlogin - push (@login, (split /,/, $sshlogin)); + for my $s (split /,/, $sshlogin) { + if ($s eq "..") { + read_sshloginfile($s); + } else { + push (@login, $s); + } + } } for my $sshlogin (@login) { if($sshlogin =~ s:^(\d*)/::) { diff --git a/unittest/tests-to-run/test18.sh b/unittest/tests-to-run/test18.sh index 8c59d901..f57ef88b 100644 --- a/unittest/tests-to-run/test18.sh +++ b/unittest/tests-to-run/test18.sh @@ -1,12 +1,16 @@ #!/bin/bash -PAR=parallel - SERVER1=parallel-server3 SERVER2=parallel-server2 +echo '### Check -S .. and --serverloginfile ..' +echo $SERVER1 > ~/.parallel/sshloginfile +echo parallel@$SERVER2 >> ~/.parallel/sshloginfile +seq 1 20 | parallel -k -S .. echo +seq 1 20 | parallel -k --sshloginfile .. echo + echo '### Check warning if --transfer but file not found' -echo /tmp/noexistant/file | stdout $PAR -k -S $SERVER1 --transfer echo +echo /tmp/noexistant/file | stdout parallel -k -S $SERVER1 --transfer echo echo '### Transfer for file starting with :' cd /tmp @@ -18,34 +22,34 @@ cat /tmp/test18 | parallel -j1 --trc {}.{.} -S $SERVER1,parallel@$SERVER2,: \ cat /tmp/test18 | parallel -j1 -k 'cat {}.{.}' echo '### Check warning if --transfer but not --sshlogin' -echo | stdout $PAR -k --transfer echo +echo | stdout parallel -k --transfer echo echo '### Check warning if --return but not --sshlogin' -echo | stdout $PAR -k --return {} echo +echo | stdout parallel -k --return {} echo echo '### Check warning if --cleanup but not --sshlogin' -echo | stdout $PAR -k --cleanup echo +echo | stdout parallel -k --cleanup echo echo '### Test --sshlogin -S --sshloginfile' echo localhost >/tmp/parallel-sshlogin -seq 1 3 | $PAR -k --sshlogin 8/$SERVER1 -S "7/ssh -l parallel $SERVER2",: --sshloginfile /tmp/parallel-sshlogin echo +seq 1 3 | parallel -k --sshlogin 8/$SERVER1 -S "7/ssh -l parallel $SERVER2",: --sshloginfile /tmp/parallel-sshlogin echo echo '### Test --sshloginfile with extra content' echo "2/ssh -l parallel $SERVER2" >>/tmp/parallel-sshlogin echo ":" >>/tmp/parallel-sshlogin echo "#2/ssh -l tange nothing" >>/tmp/parallel-sshlogin -seq 1 10 | $PAR -k --sshloginfile /tmp/parallel-sshlogin echo +seq 1 10 | parallel -k --sshloginfile /tmp/parallel-sshlogin echo echo '### Check forced number of CPUs being respected' stdout seq 1 20 | parallel -k -j+0 -S 1/:,9/$SERVER1 "hostname; echo {} >/dev/null" echo '### Check more than 9 simultaneous sshlogins' -seq 1 11 | $PAR -k -j0 -S "/ssh $SERVER1" echo +seq 1 11 | parallel -k -j0 -S "/ssh $SERVER1" echo echo '### Check more than 9(relative) simultaneous sshlogins' -seq 1 11 | $PAR -k -j10000% -S "ssh $SERVER1" echo +seq 1 11 | parallel -k -j10000% -S "ssh $SERVER1" echo echo '### Check -S syntax' -seq 1 11 | $PAR -k -j100% -S "/:" echo +seq 1 11 | parallel -k -j100% -S "/:" echo diff --git a/unittest/tests-to-run/test21.sh b/unittest/tests-to-run/test21.sh index 5bed50a5..8924e7f4 100644 --- a/unittest/tests-to-run/test21.sh +++ b/unittest/tests-to-run/test21.sh @@ -1,18 +1,17 @@ #!/bin/bash -PAR=parallel SERVER1=parallel-server3 SERVER2=parallel-server2 echo '### Test $PARALLEL' -echo | PARALLEL=--number-of-cpus $PAR +echo | PARALLEL=--number-of-cpus parallel seq 1 2 | PARALLEL="-S$SERVER1 -Sssh -l parallel $SERVER2 --j1" $PAR -kv echo +-j1" parallel -kv echo -echo '### Test ~/.parallelrc' +echo '### Test ~/.parallel/config' echo "-S$SERVER1 -Sssh -l parallel $SERVER2 --j1" > ~/.parallelrc -seq 1 2 | $PAR -kv echo -rm ~/.parallelrc +-j1" > ~/.parallel/config +seq 1 2 | parallel -kv echo +rm ~/.parallel/config diff --git a/unittest/tests-to-run/test29.sh b/unittest/tests-to-run/test29.sh index fc8e6f2b..2cfbc282 100644 --- a/unittest/tests-to-run/test29.sh +++ b/unittest/tests-to-run/test29.sh @@ -13,7 +13,7 @@ echo '### Test of eof string on :::' parallel -E ole echo ::: foo ole bar echo '### Test of ignore-empty string on :::' -parallel -r echo ::: foo '' ole bar +parallel -k -r echo ::: foo '' ole bar echo '### Test of trailing space continuation' (echo foo; echo '';echo 'ole ';echo bar;echo quux) | xargs -r -L2 echo diff --git a/unittest/wanted-results/test18 b/unittest/wanted-results/test18 index 381e7f39..9f30d16e 100644 --- a/unittest/wanted-results/test18 +++ b/unittest/wanted-results/test18 @@ -1,3 +1,44 @@ +### Check -S .. and --serverloginfile .. +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 ### Check warning if --transfer but file not found Warning: /tmp/noexistant/file is not readable and will not be transferred /tmp/noexistant/file diff --git a/unittest/wanted-results/test21 b/unittest/wanted-results/test21 index fb071aab..7514d2b0 100644 --- a/unittest/wanted-results/test21 +++ b/unittest/wanted-results/test21 @@ -4,7 +4,7 @@ ssh -l parallel parallel-server2 echo\ 1; 1 ssh parallel-server3 echo\ 2; 2 -### Test ~/.parallelrc +### Test ~/.parallel/config ssh -l parallel parallel-server2 echo\ 1; 1 ssh parallel-server3 echo\ 2;