parallel: rsync version 3.2.7 support.

This commit is contained in:
Ole Tange 2023-02-27 21:48:16 +01:00
parent d53a4b044c
commit bbfb4a8d4e
4 changed files with 75 additions and 22 deletions

View file

@ -4,6 +4,25 @@
Quote of the month: Quote of the month:
Recently learned how to use GNU parallel (from ChatGPT, no less!) and I've gone mad with power
-- Mark, Anthropomorphic Anuran @reject_resubmit@twitter
gnu parallel is actually like. really easy
-- tom (era) @slimefiend@twitter
GNU parallel is magic, half of my work uses it, to the point where they're referenced and thanked in my thesis
-- Best Catboy Key Grip @alamogordoglass@twitter
Love to make a dual processor workstation absolutely whir running dozens of analysis scripts at once
-- Best Catboy Key Grip @alamogordoglass@twitter
i really didn't expect it to be as simple as it is, took me all of 5 minutes to implement and the results are exactly what i wanted
gnu parallel babey, it's really simple actually lol
-- tom (era) @slimefiend@twitter
GNU parallel is your friend. Unleash your cores! #GNU
-- Blake L @BlakeDL@twitter
--line-buffer, a flag --line-buffer, a flag

View file

@ -262,9 +262,9 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm> stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20230222 ('Gaziantep') released Subject: GNU Parallel 20230322 ('Grækenland Larissa tog/Tiktok?') released
GNU Parallel 20230222 ('Gaziantep') has been released. It is available for download at: lbry://@GnuParallel:4 GNU Parallel 20230322 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4
Quote of the month: Quote of the month:

View file

@ -5876,9 +5876,6 @@ sub citation() {
"https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html", "https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html",
"https://www.gnu.org/software/parallel/parallel_design.html#citation-notice", "https://www.gnu.org/software/parallel/parallel_design.html#citation-notice",
"https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt", "https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt",
"",
"If you send a copy of your published article to tange\@gnu.org, it will be",
"mentioned in the release notes of next version of GNU Parallel.",
"" ""
); );
while(not grep { -e "$_/will-cite" } @Global::config_dirs) { while(not grep { -e "$_/will-cite" } @Global::config_dirs) {
@ -8833,15 +8830,17 @@ sub rsync_transfer_cmd($) {
} }
{ {
my $rsync_protocol; my $rsync_fix;
my $rsync_version;
sub rsync($) { sub rsync($) {
sub rsync_protocol { # rsync 3.1.x uses protocol 31 which is unsupported by 2.5.7.
# rsync 3.1.x uses protocol 31 which is unsupported by 2.5.7. # If the version >= 3.1.0: downgrade to protocol 30
# If the version >= 3.1.0: downgrade to protocol 30 # rsync 3.2.4 introduces a quoting bug: Add --old-args for that
# Returns: # Returns:
# $rsync = "rsync" or "rsync --protocol 30" # $rsync = "rsync" or "rsync --protocol 30 --old-args"
if(not $rsync_protocol) { sub rsync_version {
if(not $rsync_version) {
my @out = `rsync --version`; my @out = `rsync --version`;
if(not @out) { if(not @out) {
if(::which("rsync")) { if(::which("rsync")) {
@ -8854,23 +8853,37 @@ sub rsync_transfer_cmd($) {
for (@out) { for (@out) {
# rsync version 3.1.3 protocol version 31 # rsync version 3.1.3 protocol version 31
# rsync version v3.2.3 protocol version 31 # rsync version v3.2.3 protocol version 31
if(/version v?(\d+.\d+)(.\d+)?/) { if(/version v?(\d+)\.(\d+)(\.(\d+))?/) {
if($1 >= 3.1) { # 3.2.27 => 03.0227
# Version 3.1.0 or later: Downgrade to protocol 30 $rsync_version = sprintf "%02d.%02d%02d",$1,$2,$4;
$rsync_protocol = "rsync --protocol 30";
} else {
$rsync_protocol = "rsync";
}
} }
} }
$rsync_protocol or $rsync_version or
::die_bug("Cannot figure out version of rsync: @out"); ::die_bug("Cannot figure out version of rsync: @out");
} }
return $rsync_protocol; }
sub rsync_fixup {
# rsync 3.1.x uses protocol 31 which is unsupported by 2.5.7.
# If the version >= 3.1.0: downgrade to protocol 30
# Returns:
# $rsync = "rsync" or "rsync --protocol 30"
if(not $rsync_fix) {
rsync_version();
if($rsync_version >= 3.01) {
# Version 3.1.0 or later: Downgrade to protocol 30
$rsync_fix .= " --protocol 30";
}
if($rsync_version >= 3.0204) {
# Version 3.2.4 .. 3.2.8: --old-args
$rsync_fix .= " --old-args";
}
}
return $rsync_fix;
} }
my $self = shift; my $self = shift;
return rsync_protocol()." ".$ENV{'PARALLEL_RSYNC_OPTS'}. return "rsync".rsync_fixup()." ".$ENV{'PARALLEL_RSYNC_OPTS'}.
" -e".::Q($self->sshcmd()); " -e".::Q($self->sshcmd());
} }
} }

View file

@ -21,6 +21,27 @@ par_z_sshloginfile() {
rm -f "$tmp" rm -f "$tmp"
} }
par__test_different_rsync_versions() {
echo '### different versions of rsync need fixups'
echo '### no output is good'
doit() {
rm -f 'a`b`c\<d\$e\{#\}g\"h\ i'$2 'a`b`c\<d\$e\{#\}g\"h\ i'$2.out
touch 'a`b`c\<d\$e\{#\}g\"h\ i'$2
TMPDIR=/tmp tmp=$(mktemp -d)
(
echo "#!/bin/bash"
echo $1' "$@"'
) > "$tmp"/rsync
chmod +x "$tmp"/rsync
PATH="$tmp":"$PATH"
parallel --trc {}.out -S sh@lo cp {} {}.out ::: 'a`b`c\<d\$e\{#\}g\"h\ i'$2
rm 'a`b`c\<d\$e\{#\}g\"h\ i'$2 'a`b`c\<d\$e\{#\}g\"h\ i'$2.out
rm -rf "$tmp"
}
export -f doit
stdout parallel --tagstring {/} -k doit {} {/} ::: /usr/local/bin/rsync-v*
}
par_--nonall_results() { par_--nonall_results() {
echo '### --results --onall' echo '### --results --onall'
tmp="$TMPDIR"/onall tmp="$TMPDIR"/onall