mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
parallel: implemented --minversion
This commit is contained in:
Ole Tange
parent
7ed9090560
commit
0a1d43bbda
276
doc/FUTURE_IDEAS
276
doc/FUTURE_IDEAS
|
|
@ -1,11 +1,138 @@
|
||||||
|
Til QUOTING:
|
||||||
|
|
||||||
|
FN="two spaces"
|
||||||
|
echo 1 | parallel -q echo {} "$FN"
|
||||||
|
# Prints 2 spaces between 'two' and 'spaces'
|
||||||
|
|
||||||
|
-q will not work with composed commands as it will quote the ; as
|
||||||
|
well. So composed commands have to be quoted by hand:
|
||||||
|
|
||||||
|
# Using export:
|
||||||
|
FN2="two spaces"
|
||||||
|
export FN2
|
||||||
|
echo 1 | parallel echo {} \"\$FN2\" \; echo \"\$FN2\" {}
|
||||||
|
# Prints 2 spaces between 'two' and 'spaces'
|
||||||
|
|
||||||
|
# Without export:
|
||||||
|
FN3="two spaces"
|
||||||
|
echo 1 | parallel echo {} \""$FN3"\" \; echo \'"$FN3"\' {}
|
||||||
|
|
||||||
|
# By quoting the space in the variable
|
||||||
|
FN4='two\ \ spaces'
|
||||||
|
echo 1 | parallel echo {} $FN4 \; echo $FN4 {}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
= Bug? ==
|
||||||
|
|
||||||
|
locate .gz | parallel -X find {} -size +1000 -size -2000 | parallel --workdir ... -S .. --trc {/}.bz2 'zcat {} | bzip2 > {/}.bz2'
|
||||||
|
|
||||||
|
|
||||||
|
== Compare ==
|
||||||
|
|
||||||
|
http://code.google.com/p/spawntool/
|
||||||
|
http://code.google.com/p/push/
|
||||||
|
|
||||||
|
== Bug? ==
|
||||||
|
|
||||||
|
.parallel/config with --long-options
|
||||||
|
|
||||||
|
|
||||||
|
== SQL ==
|
||||||
|
|
||||||
|
Example with %0a as newline
|
||||||
|
sql :my_postgres?'\dt %0a SELECT * FROM users'
|
||||||
|
|
||||||
|
cat ~/.sql/aliases | parallel --colsep '\s' sql {1} '"select 0.14+3;" | grep -q 3.14 || (echo dead: {1}; exit 1)'
|
||||||
|
|
||||||
|
== FEX ==
|
||||||
|
|
||||||
|
fex syntax for splitting fields
|
||||||
|
http://www.semicomplete.com/projects/fex/
|
||||||
|
sql :foo 'select * from bar' | parallel --fex '|{1,2}' do_stuff {2} {1}
|
||||||
|
|
||||||
|
|
||||||
|
--autocolsep: Læs alle linjer.
|
||||||
|
Prøv fastlængde: Find tegn, som står i alle linjer på de samme pladser. Risiko for falske pos
|
||||||
|
Prøv fieldsep: Find eet tegn, som optræder det samme antal gange i alle linjer (tab sep)
|
||||||
|
Prøv klyngesep: Find den samme klynge tegn, som står samme antal gange i alle linjer (' | ' sep)
|
||||||
|
Fjern whitespace før og efter colonne
|
||||||
|
|
||||||
|
hvis der er n af tegn A og 2n af tegn B, så
|
||||||
|
|
||||||
|
a | b | c
|
||||||
|
|
||||||
|
Simpleste: tab sep
|
||||||
|
|
||||||
|
for hver linje
|
||||||
|
max,min count for hver char
|
||||||
|
|
||||||
|
for hver char
|
||||||
|
if max == min :
|
||||||
|
potentiel
|
||||||
|
min_potentiel = min(min_potentiel,min)
|
||||||
|
|
||||||
|
for potentiel:
|
||||||
|
if min % min_potentiel = 0: sepchars += potentiel,no of sepchars += min / min_potentiel
|
||||||
|
|
||||||
|
colsep = [sepchars]{no_of_sepchars}
|
||||||
|
|
||||||
|
|
||||||
|
# TODO max_line_length on remote
|
||||||
|
# TODO compute how many can be transferred within max_line_length
|
||||||
|
|
||||||
|
|
||||||
|
Postkort:
|
||||||
|
- Forside kun figur
|
||||||
|
- Bagside:
|
||||||
|
- Logo med figur - evt gnu.org/s/parallel
|
||||||
|
- kort grå tekst, der forklarer hvad det er.
|
||||||
|
- Eet eksempel: parallel gzip ::: *
|
||||||
|
- Link til video. http://nd.gd/0s
|
||||||
|
|
||||||
|
GNU parallel is a UNIX-tool for running commands in parallel.
|
||||||
|
To gzip all files running one job per CPU write:
|
||||||
|
parallel gzip ::: *
|
||||||
|
Watch the intro video to learn more: http://nd.gd/0s
|
||||||
|
Or read more about GNU parallel: www.gnu.org/s/parallel
|
||||||
|
|
||||||
|
job->start():
|
||||||
|
$jobslot = Global::jobslot->$sshlogin
|
||||||
|
|
||||||
|
sub get_jobslot {
|
||||||
|
my $sshlogin = shift;
|
||||||
|
my $jobslot_id = pop @Global::jobslots{$sshlogin};
|
||||||
|
if not defined $jobslot_id {
|
||||||
|
$jobslot_id = ++$Global::max_jobslot_id;
|
||||||
|
}
|
||||||
|
return $jobslot_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub release_jobslot {
|
||||||
|
my $sshlogin = shift;
|
||||||
|
my $jobslot_id = shift;
|
||||||
|
push @Global::jobslots{$sshlogin}, $jobslot_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
Test sshlogins in parallel. Assume parallel is in path
|
||||||
|
|
||||||
|
seq 1 10 | parallel -I {o} 'seq 1 255 | parallel echo ssh -oNoHostAuthenticationForLocalhost=true 127.0.{o}.{}' >/tmp/sshloginfile
|
||||||
|
seq 1 1000 | parallel --sshloginfile /tmp/sshloginfile echo
|
||||||
|
|
||||||
|
ssh -F /tmp/
|
||||||
|
|
||||||
|
Example:
|
||||||
|
Chop mbox into emails
|
||||||
|
Parallel sort
|
||||||
|
|
||||||
|
codecoverage
|
||||||
|
|
||||||
|
Testsuite: sem without ~/.parallel
|
||||||
|
|
||||||
Dont start:
|
Dont start:
|
||||||
|
|
||||||
* load
|
|
||||||
* memory free | head -n 2 | awk \{print\ \$4+\$6+\$7\}
|
|
||||||
* swap
|
|
||||||
* seek
|
* seek
|
||||||
|
|
||||||
vmstat\ 1\ 2\ \|\ tail\ -n1\ \|\ awk\ \{print\\\ \\\$7*\\\$8\}
|
|
||||||
|
|
||||||
|
|
||||||
Video 30. 36. 41. 48
|
Video 30. 36. 41. 48
|
||||||
|
|
@ -113,137 +240,6 @@ parallel convert -resize {3} -quality {2} {1} {1.}_{3}_q{2}.jpg :::: - ::: 25 50
|
||||||
# Find GNU Parallel at http://www.gnu.org/software/parallel/
|
# Find GNU Parallel at http://www.gnu.org/software/parallel/
|
||||||
|
|
||||||
|
|
||||||
Postkort:
|
|
||||||
- Forside kun figur
|
|
||||||
- Bagside:
|
|
||||||
- Logo med figur - evt gnu.org/s/parallel
|
|
||||||
- kort grå tekst, der forklarer hvad det er.
|
|
||||||
- Eet eksempel: parallel gzip ::: *
|
|
||||||
- Link til video. http://nd.gd/0s
|
|
||||||
|
|
||||||
GNU parallel is a UNIX-tool for running commands in parallel.
|
|
||||||
To gzip all files running one job per CPU write:
|
|
||||||
parallel gzip ::: *
|
|
||||||
Watch the intro video to learn more: http://nd.gd/0s
|
|
||||||
Or read more about GNU parallel: www.gnu.org/s/parallel
|
|
||||||
|
|
||||||
job->start():
|
|
||||||
$jobslot = Global::jobslot->$sshlogin
|
|
||||||
|
|
||||||
sub get_jobslot {
|
|
||||||
my $sshlogin = shift;
|
|
||||||
my $jobslot_id = pop @Global::jobslots{$sshlogin};
|
|
||||||
if not defined $jobslot_id {
|
|
||||||
$jobslot_id = ++$Global::max_jobslot_id;
|
|
||||||
}
|
|
||||||
return $jobslot_id;
|
|
||||||
}
|
|
||||||
|
|
||||||
sub release_jobslot {
|
|
||||||
my $sshlogin = shift;
|
|
||||||
my $jobslot_id = shift;
|
|
||||||
push @Global::jobslots{$sshlogin}, $jobslot_id;
|
|
||||||
}
|
|
||||||
|
|
||||||
Test sshlogins in parallel. Assume parallel is in path
|
|
||||||
|
|
||||||
seq 1 10 | parallel -I {o} 'seq 1 255 | parallel echo ssh -oNoHostAuthenticationForLocalhost=true 127.0.{o}.{}' >/tmp/sshloginfile
|
|
||||||
seq 1 1000 | parallel --sshloginfile /tmp/sshloginfile echo
|
|
||||||
|
|
||||||
ssh -F /tmp/
|
|
||||||
|
|
||||||
Example:
|
|
||||||
Chop mbox into emails
|
|
||||||
Parallel sort
|
|
||||||
|
|
||||||
codecoverage
|
|
||||||
|
|
||||||
Testsuite: sem without ~/.parallel
|
|
||||||
|
|
||||||
Til QUOTING:
|
|
||||||
|
|
||||||
FN="two spaces"
|
|
||||||
echo 1 | parallel -q echo {} "$FN"
|
|
||||||
# Prints 2 spaces between 'two' and 'spaces'
|
|
||||||
|
|
||||||
-q will not work with composed commands as it will quote the ; as
|
|
||||||
well. So composed commands have to be quoted by hand:
|
|
||||||
|
|
||||||
# Using export:
|
|
||||||
FN2="two spaces"
|
|
||||||
export FN2
|
|
||||||
echo 1 | parallel echo {} \"\$FN2\" \; echo \"\$FN2\" {}
|
|
||||||
# Prints 2 spaces between 'two' and 'spaces'
|
|
||||||
|
|
||||||
# Without export:
|
|
||||||
FN3="two spaces"
|
|
||||||
echo 1 | parallel echo {} \""$FN3"\" \; echo \'"$FN3"\' {}
|
|
||||||
|
|
||||||
# By quoting the space in the variable
|
|
||||||
FN4='two\ \ spaces'
|
|
||||||
echo 1 | parallel echo {} $FN4 \; echo $FN4 {}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
= Bug? ==
|
|
||||||
|
|
||||||
locate .gz | parallel -X find {} -size +1000 -size -2000 | parallel --workdir ... -S .. --trc {/}.bz2 'zcat {} | bzip2 > {/}.bz2'
|
|
||||||
|
|
||||||
|
|
||||||
== Compare ==
|
|
||||||
|
|
||||||
http://code.google.com/p/spawntool/
|
|
||||||
http://code.google.com/p/push/
|
|
||||||
|
|
||||||
== Bug? ==
|
|
||||||
|
|
||||||
.parallel/config with --long-options
|
|
||||||
|
|
||||||
time find . -type f | parallel -j+0 --eta -S..,: --progress --trc {}.gz gzip {}
|
|
||||||
|
|
||||||
== SQL ==
|
|
||||||
|
|
||||||
Example with %0a as newline
|
|
||||||
sql :my_postgres?'\dt %0a SELECT * FROM users'
|
|
||||||
|
|
||||||
cat ~/.sql/aliases | parallel --colsep '\s' sql {1} '"select 0.14+3;" | grep -q 3.14 || (echo dead: {1}; exit 1)'
|
|
||||||
|
|
||||||
== FEX ==
|
|
||||||
|
|
||||||
fex syntax for splitting fields
|
|
||||||
http://www.semicomplete.com/projects/fex/
|
|
||||||
sql :foo 'select * from bar' | parallel --fex '|{1,2}' do_stuff {2} {1}
|
|
||||||
|
|
||||||
|
|
||||||
--autocolsep: Læs alle linjer.
|
|
||||||
Prøv fastlængde: Find tegn, som står i alle linjer på de samme pladser. Risiko for falske pos
|
|
||||||
Prøv fieldsep: Find eet tegn, som optræder det samme antal gange i alle linjer (tab sep)
|
|
||||||
Prøv klyngesep: Find den samme klynge tegn, som står samme antal gange i alle linjer (' | ' sep)
|
|
||||||
Fjern whitespace før og efter colonne
|
|
||||||
|
|
||||||
hvis der er n af tegn A og 2n af tegn B, så
|
|
||||||
|
|
||||||
a | b | c
|
|
||||||
|
|
||||||
Simpleste: tab sep
|
|
||||||
|
|
||||||
for hver linje
|
|
||||||
max,min count for hver char
|
|
||||||
|
|
||||||
for hver char
|
|
||||||
if max == min :
|
|
||||||
potentiel
|
|
||||||
min_potentiel = min(min_potentiel,min)
|
|
||||||
|
|
||||||
for potentiel:
|
|
||||||
if min % min_potentiel = 0: sepchars += potentiel,no of sepchars += min / min_potentiel
|
|
||||||
|
|
||||||
colsep = [sepchars]{no_of_sepchars}
|
|
||||||
|
|
||||||
|
|
||||||
# TODO max_line_length on remote
|
|
||||||
# TODO compute how many can be transferred within max_line_length
|
|
||||||
|
|
||||||
=head1 YouTube video --pipe
|
=head1 YouTube video --pipe
|
||||||
|
|
||||||
cp parallel.fasta parallel.mbox lucene.tar
|
cp parallel.fasta parallel.mbox lucene.tar
|
||||||
|
|
@ -615,16 +611,6 @@ Test if -0 works on filenames ending in '\n'
|
||||||
|
|
||||||
One char options not used: A F G K O Q R Z c f
|
One char options not used: A F G K O Q R Z c f
|
||||||
|
|
||||||
=head1 sem
|
|
||||||
|
|
||||||
Parallelize so this can be done:
|
|
||||||
mdm.screen find dir -execdir mdm-run cmd {} \;
|
|
||||||
Maybe:
|
|
||||||
find dir -execdir par$ --communication-file /tmp/comfile cmd {} \;
|
|
||||||
|
|
||||||
find dir -execdir mutex -j4 -b cmd {} \;
|
|
||||||
|
|
||||||
|
|
||||||
=head1 Unlikely
|
=head1 Unlikely
|
||||||
|
|
||||||
Accept signal INT instead of TERM to complete current running jobs but
|
Accept signal INT instead of TERM to complete current running jobs but
|
||||||
|
|
|
||||||
|
|
@ -403,6 +403,7 @@ sub options_hash {
|
||||||
"interactive|p" => \$::opt_p,
|
"interactive|p" => \$::opt_p,
|
||||||
"verbose|t" => \$::opt_verbose,
|
"verbose|t" => \$::opt_verbose,
|
||||||
"version|V" => \$::opt_version,
|
"version|V" => \$::opt_version,
|
||||||
|
"minversion|min-version=i" => \$::opt_minversion,
|
||||||
"show-limits|showlimits" => \$::opt_show_limits,
|
"show-limits|showlimits" => \$::opt_show_limits,
|
||||||
"exit|x" => \$::opt_x,
|
"exit|x" => \$::opt_x,
|
||||||
# Semaphore
|
# Semaphore
|
||||||
|
|
@ -530,6 +531,14 @@ sub parse_options {
|
||||||
if(defined $::opt_fg) { $Global::semaphore = 1; }
|
if(defined $::opt_fg) { $Global::semaphore = 1; }
|
||||||
if(defined $::opt_bg) { $Global::semaphore = 1; }
|
if(defined $::opt_bg) { $Global::semaphore = 1; }
|
||||||
if(defined $::opt_wait) { $Global::semaphore = 1; }
|
if(defined $::opt_wait) { $Global::semaphore = 1; }
|
||||||
|
if(defined $::opt_minversion) {
|
||||||
|
version();
|
||||||
|
if($Global::version < $::opt_minversion) {
|
||||||
|
wait_and_exit(255);
|
||||||
|
} else {
|
||||||
|
wait_and_exit(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
if(defined $::opt_nonall) {
|
if(defined $::opt_nonall) {
|
||||||
# Append a dummy empty argument
|
# Append a dummy empty argument
|
||||||
push @ARGV, ":::", "";
|
push @ARGV, ":::", "";
|
||||||
|
|
|
||||||
|
|
@ -621,6 +621,16 @@ See also B<-X> for context replace. If in doubt use B<-X> as that will
|
||||||
most likely do what is needed.
|
most likely do what is needed.
|
||||||
|
|
||||||
|
|
||||||
|
=item B<--minversion> I<version> (alpha testing)
|
||||||
|
|
||||||
|
Print the version GNU B<parallel> and exit. If the current version of
|
||||||
|
GNU B<parallel> is less than I<version> the exit code is
|
||||||
|
255. Otherwise it is 0.
|
||||||
|
|
||||||
|
This is useful for scripts that depend on features only available from
|
||||||
|
a certain version of GNU B<parallel>.
|
||||||
|
|
||||||
|
|
||||||
=item B<--onall> (alpha testing)
|
=item B<--onall> (alpha testing)
|
||||||
|
|
||||||
Run all the jobs on all computers given with B<--sshlogin>. GNU
|
Run all the jobs on all computers given with B<--sshlogin>. GNU
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue