mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-21 21:47:54 +00:00
parallel: Give better error message if file not found when using --pipepart.
This commit is contained in:
Ole Tange
parent
eac8944b1d
commit
73f554ad8f
|
|
@ -23,12 +23,6 @@
|
|||
# $ bash install.sh
|
||||
|
||||
run() {
|
||||
# tail on openindiana must be /usr/xpg4/bin/tail
|
||||
tail=$(echo | tail -n 1 2>/dev/null && echo tail ||
|
||||
(echo | /usr/xpg4/bin/tail -n 1 && echo /usr/xpg4/bin/tail))
|
||||
# grep on openindiana must be /usr/xpg4/bin/grep
|
||||
grep=$(echo | grep -vE . 2>/dev/null && echo grep ||
|
||||
(echo | /usr/xpg4/bin/grep -vE . && echo /usr/xpg4/bin/grep))
|
||||
# FreeBSD prefers 'fetch', MacOS prefers 'curl', Linux prefers 'wget'
|
||||
get=$(
|
||||
(lynx -source /dev/null && echo lynx -source) ||
|
||||
|
|
@ -49,7 +43,7 @@ run() {
|
|||
LANG=C
|
||||
latest=$($get http://ftpmirror.gnu.org/parallel |
|
||||
perl -ne '/.*(parallel-\d{8})/ and print $1."\n"' |
|
||||
sort | $tail -n1)
|
||||
perl -e 'print ((reverse sort <>)[0])')
|
||||
if test \! -e $latest.tar.bz2; then
|
||||
# Source tar does not exist
|
||||
rm -f $latest.tar.bz2 $latest.tar.bz2.sig
|
||||
|
|
@ -91,7 +85,7 @@ run() {
|
|||
# Check signature - in case ftpmirror.gnu.org is compromised
|
||||
if fetch_keys; then
|
||||
if gpg --with-fingerprint $latest.tar.bz2.sig 2>&1 |
|
||||
$grep -E '^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888' ; then
|
||||
perl -e 'exit not grep /^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888/, <>'; then
|
||||
# Source code signed by Ole Tange <ole@tange.dk>
|
||||
# KeyID FFFFFFF1/88888888
|
||||
true
|
||||
|
|
|
|||
8
NEWS
8
NEWS
|
|
@ -1,4 +1,4 @@
|
|||
20190822
|
||||
20190922
|
||||
|
||||
* --nice is now inherited by the nice level that GNU Parallel is
|
||||
started at. So 'nice -n10 parallel' will also cause remote jobs to
|
||||
|
|
@ -9,9 +9,11 @@
|
|||
* GNU Parallel will be presented at Driving IT 2019:
|
||||
https://ida.dk/arrangementer-og-kurser/konferencer/driving-it/tools
|
||||
|
||||
* Greenland Ice Sheet solid ice discharge from 1986 through 2017 https://www.earth-syst-sci-data.net/11/769/2019/essd-11-769-2019.pdf
|
||||
* Greenland Ice Sheet solid ice discharge from 1986 through 2017
|
||||
https://www.earth-syst-sci-data.net/11/769/2019/essd-11-769-2019.pdf
|
||||
|
||||
* App Center Cloud Build Machines > Installed Software > Utilities https://docs.microsoft.com/en-us/appcenter/build/software
|
||||
* App Center Cloud Build Machines > Installed Software > Utilities
|
||||
https://docs.microsoft.com/en-us/appcenter/build/software
|
||||
|
||||
* Curryfinger - SNI & Host header spoofing utility
|
||||
https://dualuse.io/blog/curryfinger/
|
||||
|
|
|
|||
10
doc/haikus
10
doc/haikus
|
|
@ -1,7 +1,10 @@
|
|||
Quote of the month:
|
||||
|
||||
IMHO, SQLite and GNU Parallel are among the world's great software.
|
||||
-- singe@reddit
|
||||
Well anyway, It was blazingly fast and astonished by performance. guess I'll never use xargs.
|
||||
-- (Not) Akaming @_Akamig@twitter
|
||||
|
||||
GNU parallel has helped me kill a Hadoop cluster before.
|
||||
-- Travis Campbell @hcoyote@twitter
|
||||
|
||||
Yeah, GNU parallel is a beast when used accordingly.
|
||||
-- @lsde@twitter
|
||||
|
|
@ -55,6 +58,9 @@ Quote of the month:
|
|||
|
||||
|
||||
=== Used ===
|
||||
IMHO, SQLite and GNU Parallel are among the world's great software.
|
||||
-- singe@reddit
|
||||
|
||||
It is, beyond absolutely any doubt whatsoever, the single most
|
||||
important tool I use in making me a productive bioinformatician.
|
||||
-- A-N-Other@reddit.com
|
||||
|
|
|
|||
|
|
@ -209,9 +209,9 @@ from:tange@gnu.org
|
|||
to:parallel@gnu.org, bug-parallel@gnu.org
|
||||
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
||||
|
||||
Subject: GNU Parallel 20190822 ('Dorian/Stallman') released <<[stable]>>
|
||||
Subject: GNU Parallel 20191022 ('Driving IT') released <<[stable]>>
|
||||
|
||||
GNU Parallel 20190822 ('Dorian') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
GNU Parallel 20191022 ('Driving IT') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
|
||||
<<No new functionality was introduced so this is a good candidate for a stable release.>>
|
||||
|
||||
|
|
@ -225,32 +225,16 @@ Quote of the month:
|
|||
|
||||
New in this release:
|
||||
|
||||
* --delay is now accurate to around 5 ms.
|
||||
Uses GNU Parallel https://github.com/pirovc/genome_updater
|
||||
|
||||
* --nice is now inherited by the nice level that GNU Parallel is started at. So 'nice -n10 parallel' will also cause remote jobs to be run at nice level 10.
|
||||
Using GNU-Parallel for bioinformatics https://www.danielecook.com/using-gnu-parallel-for-bioinformatics/
|
||||
|
||||
* GNU Parallel will be presented at Driving IT 2019: https://ida.dk/arrangementer-og-kurser/konferencer/driving-it/tools
|
||||
Speeding up PostgreSQL ETL pipeline with the help of GODS https://cfengine.com/company/blog-detail/speeding-up-postgresql-etl-pipeline-with-the-help-of-gods/
|
||||
|
||||
* Greenland Ice Sheet solid ice discharge from 1986 through 2017 https://www.earth-syst-sci-data.net/11/769/2019/essd-11-769-2019.pdf
|
||||
|
||||
* App Center Cloud Build Machines > Installed Software > Utilities https://docs.microsoft.com/en-us/appcenter/build/software
|
||||
https://readthedocs.org/projects/curc/downloads/pdf/latest/
|
||||
|
||||
https://dualuse.io/blog/curryfinger/
|
||||
|
||||
https://wiki.cac.washington.edu/display/hyakusers/Hyak+parallel-sql
|
||||
|
||||
https://www.reddit.com/r/perl/comments/cyo2qc/a_meditation_on_analysing_data_with_perl_in/
|
||||
|
||||
https://www.eventbrite.ca/e/inscription-accelerer-ses-taches-avec-gnu-parallel-et-les-lots-de-taches-ul-71412840729
|
||||
|
||||
https://curc.readthedocs.io/en/latest/software/GNUParallel.html
|
||||
|
||||
https://petelawson.com/post/parallel-in-shell/
|
||||
|
||||
https://www.usenix.org/conference/lisa19/presentation/maheshwari
|
||||
|
||||
http://atomicer.cn/2018/01/18/gnu-parallel-nfs-%E5%AE%9E%E7%8E%B0%E5%85%B1%E4%BA%AB%E6%96%87%E4%BB%B6%E7%BD%91%E7%BB%9C%E7%9A%84%E5%B9%B6%E5%8F%91%E8%AE%A1%E7%AE%97%E7%AD%89%E6%93%8D%E4%BD%9C/#more
|
||||
Gnu/parallel + Nfs 实现共享文件网络的并发计算等操作
|
||||
* 如何使用Parallel在Shell中并行执行命令https://www.myfreax.com/gnu-parallel/
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
|
|
|
|||
46
src/parallel
46
src/parallel
|
|
@ -149,9 +149,12 @@ sub pipepart_setup() {
|
|||
$size += -s $_;
|
||||
} elsif(-b $_) {
|
||||
$size += size_of_block_dev($_);
|
||||
} else {
|
||||
} elsif(-e $_) {
|
||||
::error("$_ is neither a file nor a block device");
|
||||
wait_and_exit(255);
|
||||
} else {
|
||||
::error("File not found: $_");
|
||||
wait_and_exit(255);
|
||||
}
|
||||
}
|
||||
# Run in total $job_slots*(- $blocksize) jobs
|
||||
|
|
@ -1528,6 +1531,7 @@ sub options_hash() {
|
|||
"linkinputsource|xapplyinputsource=i" => \@opt::linkinputsource,
|
||||
# Before changing this line, please read
|
||||
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
|
||||
# https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
|
||||
"bibtex|citation" => \$opt::citation,
|
||||
"wc|willcite|will-cite|nn|nonotice|no-notice" => \$opt::willcite,
|
||||
# Termination and retries
|
||||
|
|
@ -1636,6 +1640,7 @@ sub parse_options(@) {
|
|||
|
||||
# Before changing this line, please read
|
||||
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
|
||||
# https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
|
||||
if(defined $opt::citation) {
|
||||
citation(\@argv_before,\@ARGV);
|
||||
wait_and_exit(0);
|
||||
|
|
@ -1937,11 +1942,14 @@ sub parse_options(@) {
|
|||
# the alternatives instead?
|
||||
# See a list in: 'man parallel_alternatives'
|
||||
#
|
||||
# If you want GNU Parallel to be maintained in the future you should not
|
||||
# change this line.
|
||||
# If you want GNU Parallel to be maintained in the future keep
|
||||
# this line.
|
||||
citation_notice();
|
||||
# Seriously: You will be harming free software by removing the notice.
|
||||
# You make it harder to justify spending time developing it.
|
||||
# Seriously: YOU will be harming free software by removing the
|
||||
# notice. You make it harder to justify spending time developing
|
||||
# it. If you *do* remove the line, please email
|
||||
# hallofshame@tange.dk if you want to avoid being put in a hall of
|
||||
# shame.
|
||||
|
||||
parse_halt();
|
||||
|
||||
|
|
@ -2060,6 +2068,7 @@ sub init_globals() {
|
|||
# Defaults:
|
||||
$Global::version = 20190922;
|
||||
$Global::progname = 'parallel';
|
||||
$::name = "GNU Parallel";
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
$Global::verbose = 0;
|
||||
|
|
@ -3403,10 +3412,11 @@ sub drain_job_queue(@) {
|
|||
my $sleep = 0.2;
|
||||
do {
|
||||
while($Global::total_running > 0) {
|
||||
debug($Global::total_running, "==", scalar
|
||||
debug("init",$Global::total_running, "==", scalar
|
||||
keys %Global::running," slots: ", $Global::max_jobs_running);
|
||||
if($opt::pipe) {
|
||||
# When using --pipe sometimes file handles are not closed properly
|
||||
# When using --pipe sometimes file handles are not
|
||||
# closed properly
|
||||
for my $job (values %Global::running) {
|
||||
close $job->fh(0,"w");
|
||||
}
|
||||
|
|
@ -4011,7 +4021,7 @@ sub setup_basefile() {
|
|||
}
|
||||
debug("init", "basesetup: @cmd\n");
|
||||
my ($exitstatus,$stdout_ref,$stderr_ref) =
|
||||
run_parallel((join "\n",@cmd),"-j0","--retries",5);
|
||||
run_gnu_parallel((join "\n",@cmd),"-j0","--retries",5);
|
||||
if($exitstatus) {
|
||||
my @stdout = @$stdout_ref;
|
||||
my @stderr = @$stderr_ref;
|
||||
|
|
@ -4041,7 +4051,7 @@ sub cleanup_basefile() {
|
|||
}
|
||||
debug("init", "basecleanup: @cmd\n");
|
||||
my ($exitstatus,$stdout_ref,$stderr_ref) =
|
||||
run_parallel(join("\n",@cmd),"-j0","--retries",5);
|
||||
run_gnu_parallel(join("\n",@cmd),"-j0","--retries",5);
|
||||
if($exitstatus) {
|
||||
my @stdout = @$stdout_ref;
|
||||
my @stderr = @$stderr_ref;
|
||||
|
|
@ -4050,14 +4060,14 @@ sub cleanup_basefile() {
|
|||
}
|
||||
}
|
||||
|
||||
sub run_parallel() {
|
||||
sub run_gnu_parallel() {
|
||||
my ($stdin,@args) = @_;
|
||||
my $cmd = join "",map { " $_ & " } split /\n/, $stdin;
|
||||
print $Global::original_stderr ` $cmd wait` ;
|
||||
return 0
|
||||
}
|
||||
|
||||
sub _run_parallel() {
|
||||
sub _run_gnu_parallel() {
|
||||
# Run GNU Parallel
|
||||
# This should ideally just fork an internal copy
|
||||
# and not start it through a shell
|
||||
|
|
@ -4793,6 +4803,7 @@ sub usage() {
|
|||
"",
|
||||
# Before changing this line, please read
|
||||
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
|
||||
# https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
|
||||
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
|
||||
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
|
||||
"",
|
||||
|
|
@ -4823,6 +4834,7 @@ sub citation_notice() {
|
|||
"",
|
||||
# Before changing this line, please read
|
||||
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
|
||||
# https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
|
||||
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
|
||||
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
|
||||
"",
|
||||
|
|
@ -4986,7 +4998,7 @@ sub citation() {
|
|||
"",
|
||||
"If you use '--will-cite' in scripts to be run by others you are making",
|
||||
"it harder for others to see the citation notice. The development of",
|
||||
"GNU parallel is indirectly financed through citations, so if users",
|
||||
"GNU Parallel is indirectly financed through citations, so if users",
|
||||
"do not know they should cite then you are making it harder to finance",
|
||||
"development. However, if you pay 10000 EUR, you should feel free to",
|
||||
"use '--will-cite' in scripts.",
|
||||
|
|
@ -5066,7 +5078,7 @@ parallel() {
|
|||
# [1]+ Done cat
|
||||
|
||||
# Make a temporary fifo that perl can read from
|
||||
_fifo_with_parallel_source=`perl -e 'use POSIX qw(mkfifo);
|
||||
_fifo_with_GNU_Parallel_source=`perl -e 'use POSIX qw(mkfifo);
|
||||
do {
|
||||
$f = "/tmp/parallel-".join"",
|
||||
map { (0..9,"a".."z","A".."Z")[rand(62)] } (1..5);
|
||||
|
|
@ -5075,9 +5087,9 @@ parallel() {
|
|||
print $f;'`
|
||||
# Put source code into temporary file
|
||||
# so it is easy to copy to the fifo
|
||||
_file_with_parallel_source=`mktemp`;
|
||||
_file_with_GNU_Parallel_source=`mktemp`;
|
||||
!,
|
||||
"cat <<'$randomstring' > \$_file_with_parallel_source\n",
|
||||
"cat <<'$randomstring' > \$_file_with_GNU_Parallel_source\n",
|
||||
@source,
|
||||
$randomstring,"\n",
|
||||
q!
|
||||
|
|
@ -5085,10 +5097,10 @@ parallel() {
|
|||
# and remove the file and fifo ASAP
|
||||
# 'sh -c' is needed to avoid
|
||||
# [1]+ Done cat
|
||||
sh -c "(rm $_file_with_parallel_source; cat >$_fifo_with_parallel_source; rm $_fifo_with_parallel_source) < $_file_with_parallel_source &"
|
||||
sh -c "(rm $_file_with_GNU_Parallel_source; cat >$_fifo_with_GNU_Parallel_source; rm $_fifo_with_GNU_Parallel_source) < $_file_with_GNU_Parallel_source &"
|
||||
|
||||
# Read the source from the fifo
|
||||
perl $_fifo_with_parallel_source "$@"
|
||||
perl $_fifo_with_GNU_Parallel_source "$@"
|
||||
}
|
||||
!,
|
||||
@env_parallel_source,
|
||||
|
|
|
|||
|
|
@ -1610,8 +1610,8 @@ so you are unlikely to ever use this option.
|
|||
|
||||
Setting B<--nice> will override this nice level. If the nice level is
|
||||
smaller than the current nice level, it will only affect remote jobs
|
||||
(e.g. current level is 10 and B<--nice 5> will cause local jobs to be
|
||||
run at level 10, but remote jobs run at nice level 5).
|
||||
(e.g. if current level is 10 then B<--nice 5> will cause local jobs to
|
||||
be run at level 10, but remote jobs run at nice level 5).
|
||||
|
||||
|
||||
=item B<--interactive>
|
||||
|
|
@ -2786,13 +2786,14 @@ Print the version GNU B<parallel> and exit.
|
|||
|
||||
=item B<--wd> I<mydir>
|
||||
|
||||
Jobs will be run in the dir I<mydir>.
|
||||
|
||||
Files transferred using B<--transferfile> and B<--return> will be
|
||||
relative to I<mydir> on remote computers, and the command will be
|
||||
executed in the dir I<mydir>.
|
||||
relative to I<mydir> on remote computers.
|
||||
|
||||
The special I<mydir> value B<...> will create working dirs under
|
||||
B<~/.parallel/tmp/> on the remote computers. If B<--cleanup> is given
|
||||
these dirs will be removed.
|
||||
B<~/.parallel/tmp/>. If B<--cleanup> is given these dirs will be
|
||||
removed.
|
||||
|
||||
The special I<mydir> value B<.> uses the current working dir. If the
|
||||
current working dir is beneath your home dir, the value B<.> is
|
||||
|
|
|
|||
Loading…
Reference in a new issue