From e6c9dfe0107c5046a14497b1133f3552494647f8 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 29 Aug 2020 22:12:46 +0200 Subject: [PATCH] Fixed bug #59006: rsync version 3.2.3 is not detected correctly. --- doc/haikus | 10 ++++--- doc/release_new_version | 11 +++++--- packager/obs/Makefile | 11 ++++---- src/niceload | 6 +++-- src/parallel | 6 +++-- src/parcat.pod | 19 +++++++++----- src/parsort | 16 +++++++++++- src/sql | 2 +- testsuite/tests-to-run/parallel-local-ssh6.sh | 10 +++++++ testsuite/tests-to-run/sql01.sh | 21 +++++++-------- testsuite/wanted-results/parallel-local-ssh6 | 3 +++ testsuite/wanted-results/sql01 | 26 +++++++++---------- 12 files changed, 94 insertions(+), 47 deletions(-) diff --git a/doc/haikus b/doc/haikus index aed96cd2..8418b7a5 100644 --- a/doc/haikus +++ b/doc/haikus @@ -1,14 +1,15 @@ Quote of the month: - Gnu parallel is also awesome, fwiw. - -- Rogan Dawes @RoganDawes@twitter + Great tool, gets jobs done fast. + Great tool, gets jobs done fast. + Great tool, gets jobs done fast. + -- Paul F. De La Cruz @pdelacruzcc I get a weird sense of satisfaction every single time I see the lovely logo of #GNU Parallel (plus, what an underrated piece of great software!) -- Emre Sevinç @EmreSevinc@twitter - I have gotten a *ton* of mileage out of jq, awk, and GNU parallel, even at multi-GB sizes. -- Eric Wolak @ericthewolak@twitter @@ -68,6 +69,9 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html === Used === + Gnu parallel is also awesome, fwiw. + -- Rogan Dawes @RoganDawes@twitter + With multicore systems everywhere GNU Parallel is a must have tool. -- Neil H. Watson @neil_h_watson@twitter diff --git a/doc/release_new_version b/doc/release_new_version index e3efdc44..73125dad 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -123,7 +123,7 @@ torsocks git push origin $YYYYMMDD == Zenodo == -Add tar.bz2 and publish. +Add tar.bz2 [Start upload] and [Publish]. == Update documentation == @@ -134,6 +134,7 @@ export YYYYMMDD=${YYYYMMDD:0:6}23 echo $YYYYMMDD perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/parallel perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/sql +perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/parsort perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/niceload Unmodified beta since last version => production @@ -189,9 +190,9 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20200822 ('Beirut') released <<[stable]>> +Subject: GNU Parallel 20200922 ('Belarus/Lukashenko') released <<[stable]>> -GNU Parallel 20200822 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20200922 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> @@ -207,6 +208,10 @@ New in this release: News about GNU Parallel: +https://floki.blog/2020/08/more_unix_tools + +https://computeontario.ca/event/webinar-options-for-solving-jobs-with-many-tasks/ + Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html diff --git a/packager/obs/Makefile b/packager/obs/Makefile index f95d4bf2..11f77370 100644 --- a/packager/obs/Makefile +++ b/packager/obs/Makefile @@ -13,12 +13,13 @@ all: cd home\:tange/parallel/ && osc up cd home\:tange/parallel/ && parallel osc add ::: *.spec *.dsc *.tar.gz *.tar.bz2 && echo Src added OK || true cd home\:tange/parallel/ && osc ci -m "New release" -# wait for "building" state + # wait for "building" state cd home\:tange/parallel/ && yes building | parallel -j1 --delay 10 --halt now,success=1 'osc results|G -E {}' -# wait for "building" state to end - cd home\:tange/parallel/ && yes building | parallel -j1 --delay 10 --halt now,fail=1 'osc results|G -v Mageia -E {}' || true -# wait for "finished" state of .deb + # wait for "building" state to end + # Ignore Mageia and Raspbian_9.0 that are broken + cd home\:tange/parallel/ && yes building | parallel -j1 --delay 10 --halt now,fail=1 'osc results|G -v Mageia -v Raspbian_9.0 -E {}' || true + # wait for "finished" state of .deb cd home\:tange/parallel/ && echo succeeded | parallel -j1 --retries 30 --delay 10 --halt now,success=1 'osc results|G -E "(Debian|Ubuntu).*{}"' -# wait for "finished" state of .rpm + # wait for "finished" state of .rpm cd home\:tange/parallel/ && echo succeeded | parallel -j1 --retries 30 --delay 10 --halt now,success=1 'osc results|G -E "(SUSE|SLE|Scientific|RHEL|Fedora|CentOS).*{}"' diff --git a/src/niceload b/src/niceload index 7ea7de81..ae3a1506 100755 --- a/src/niceload +++ b/src/niceload @@ -23,7 +23,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20200822; +$Global::version = 20200823; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { @@ -1147,7 +1147,9 @@ sub io_status_linux { my @iostat; for(reverse @iostat_out) { /Device/ and last; - push @iostat, (split(/\s+/,$_))[13]; + my @col = (split(/\s+/,$_)); + # Util% is last column + push @iostat, pop @col; } my $io = ::max(@iostat); return undef_as_zero($io)/10; diff --git a/src/parallel b/src/parallel index a047fd94..f3fd3dd9 100755 --- a/src/parallel +++ b/src/parallel @@ -2160,7 +2160,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20200822; + $Global::version = 20200823; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -7836,7 +7836,9 @@ sub cleanup_cmd($$$) { if(not $rsync) { my @out = `rsync --version`; for (@out) { - if(/version (\d+.\d+)(.\d+)?/) { + # rsync version 3.1.3 protocol version 31 + # rsync version v3.2.3 protocol version 31 + if(/version v?(\d+.\d+)(.\d+)?/) { if($1 >= 3.1) { # Version 3.1.0 or later: Downgrade to protocol 30 $rsync = "rsync --protocol 30"; diff --git a/src/parcat.pod b/src/parcat.pod index b558488d..59b4749a 100644 --- a/src/parcat.pod +++ b/src/parcat.pod @@ -16,6 +16,10 @@ you use: (cat file1 & cat file2 &) | ... +It is faster than doing: + + parallel -j0 --lb cat ::: file* + Arguments can be given on the command line or passed in on stdin (standard input). @@ -44,13 +48,16 @@ the files. =head2 Simple line buffered output -GNU B saves output to tempfiles. If the amount of data is -bigger than the free disk space, then you can use this technique to do -line buffering without saving to disk: +B will often print half a line. If run in parallel, two +instances may half-lines of their output. This can be avoided by +saving the output to a fifo and then using B to read the two +fifos in parallel: + + mkfifo freenetproject.org.fifo tange.dk.fifo + traceroute freenetproject.org > freenetproject.org.fifo & + traceroute tange.dk > tange.dk.fifo & + parcat --rm *fifo - mkfifo slot-{1..5} - seq 10000000 | parallel -j5 --round --pipe 'cat > slot-{%}' & - parcat slot-{1..5} | wc =head1 REPORTING BUGS diff --git a/src/parsort b/src/parsort index 132a5646..9c962fd8 100755 --- a/src/parsort +++ b/src/parsort @@ -117,7 +117,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20200412; +$Global::version = 20200823; if($opt::version) { version(); exit 0; } @Global::sortoptions = @ARGV_before[0..($#ARGV_before-$#ARGV-1)]; #if($opt::zero_terminated) { $/ = "\0"; } @@ -256,6 +256,20 @@ sub which { return wantarray ? @which : $which[0]; } +sub version() { + # Returns: N/A + print join + ("\n", + "GNU $Global::progname $Global::version", + "Copyright (C) 2020 Ole Tange, http://ole.tange.dk and Free Software", + "Foundation, Inc.", + "License GPLv3+: GNU GPL version 3 or later ", + "This is free software: you are free to change and redistribute it.", + "GNU $Global::progname comes with no warranty.", + "", + "Web site: https://www.gnu.org/software/parallel\n", + ); +} if(@ARGV) { sort_files(@ARGV); diff --git a/src/sql b/src/sql index b1e3e9c0..80c5f112 100755 --- a/src/sql +++ b/src/sql @@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20200822; + $Global::version = 20200823; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-ssh6.sh b/testsuite/tests-to-run/parallel-local-ssh6.sh index 52c6b30a..eb9f01e2 100644 --- a/testsuite/tests-to-run/parallel-local-ssh6.sh +++ b/testsuite/tests-to-run/parallel-local-ssh6.sh @@ -147,6 +147,16 @@ par_timeout_onall() { echo jobs failed: $? } +par_rsync_3.2.3() { + echo "bug #59006: rsync version 3.2.3 is not detected correctly" + PATH=$HOME/bin/rsync:$PATH + rsync --version | grep version + rm -f bug59006 + parallel --return {} -Sparallel@lo touch ::: bug59006 + ls bug59006 + rm -f bug59006 +} + export -f $(compgen -A function | grep par_) #compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1' compgen -A function | grep par_ | sort | diff --git a/testsuite/tests-to-run/sql01.sh b/testsuite/tests-to-run/sql01.sh index c1242fbd..4aa7b940 100755 --- a/testsuite/tests-to-run/sql01.sh +++ b/testsuite/tests-to-run/sql01.sh @@ -1,18 +1,17 @@ #!/bin/bash -mysqlrootpass=${mysqlrootpass:-M-b+Ydjq4ejT4E} -MYSQL_ADMIN_DBURL=mysql://root:$mysqlrootpass@/mysql - exec 2>&1 -# Setup -sql $MYSQL_ADMIN_DBURL "drop user 'sqlunittest'@'localhost'" -sql $MYSQL_ADMIN_DBURL DROP DATABASE sqlunittest; -sql $MYSQL_ADMIN_DBURL CREATE DATABASE sqlunittest; -sql $MYSQL_ADMIN_DBURL "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';" -sql $MYSQL_ADMIN_DBURL "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';" +## Setup +# mysqlrootpass=${mysqlrootpass:-M-b+Ydjq4ejT4E} +# MYSQL_ADMIN_DBURL=mysql://root:$mysqlrootpass@/mysql +# sql $MYSQL_ADMIN_DBURL "drop user 'sqlunittest'@'localhost'" +# sql $MYSQL_ADMIN_DBURL DROP DATABASE sqlunittest; +# sql $MYSQL_ADMIN_DBURL CREATE DATABASE sqlunittest; +# sql $MYSQL_ADMIN_DBURL "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';" +# sql $MYSQL_ADMIN_DBURL "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';" -MYSQL_TEST_DBURL=mysql://sqlunittest:CB5A1FFFA5A@/ +MYSQL_TEST_DBURL=mysql://tange:tange@/ echo '### Test of #! -Y with file as input' cat >/tmp/shebang <