parallel: better support for wide (Asian) characters in --latest-line.

2024-11-21 13:37:56 +00:00 · 2023-03-22 20:32:12 +01:00 · 2023-03-22 20:32:12 +01:00 · 3309f8b2c4
parent bbfb4a8d4e
commit 3309f8b2c4
37 changed files with 3615 additions and 83 deletions
--- a/11
+++ b/11
@ -1,3 +1,14 @@
+20230322
+
+New in this release:
+
+* Better support for wide characters in --latest-line.
+
+* Support for rsync 3.2.7.
+
+* Bug fixes and man page updates.
+
+
 20230222

 New in this release:
--- a/doc/haikus
+++ b/doc/haikus
@ -10,9 +10,6 @@ Quote of the month:
  gnu parallel is actually like. really easy
    -- tom (era) @slimefiend@twitter
  
-  GNU parallel is magic, half of my work uses it, to the point where they're referenced and thanked in my thesis
-    -- Best Catboy Key Grip @alamogordoglass@twitter
-  
  Love to make a dual processor workstation absolutely whir running dozens of analysis scripts at once
    -- Best Catboy Key Grip @alamogordoglass@twitter
  
@ -233,6 +230,9 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html

 === Used ===

+  GNU parallel is magic, half of my work uses it, to the point where they're referenced and thanked in my thesis
+    -- Best Catboy Key Grip @alamogordoglass@twitter
+  
  Praise GNU parallel, though. That gets me pretty far.
    -- Your Obed. Servant, J. B. @Jeffinatorator

--- a/doc/release_new_version
+++ b/doc/release_new_version
@ -262,25 +262,27 @@ from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>

-Subject: GNU Parallel 20230322 ('Grækenland Larissa tog/Tiktok?') released
+Subject: GNU Parallel 20230322 ('ICC Putin/Arrest Warrant/Fosbury/Grækenland Larissa tog/Tiktok?') released

 GNU Parallel 20230322 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4

 Quote of the month:

-  Praise GNU parallel, though. That gets me pretty far.
-    -- Your Obed. Servant, J. B. @Jeffinatorator
+  GNU parallel is magic, half of my work uses it, to the point where they're referenced and thanked in my thesis
+    -- Best Catboy Key Grip @alamogordoglass@twitter

 New in this release:

-* parsort: --parallel now does closer to what you expect.
+* Better support for wide characters in --latest-line.

-* parallel: --files0 is --files but \0 separated.
+* Support for rsync 3.2.7.

 * Bug fixes and man page updates.

 News about GNU Parallel:

+* Analyzing multi-gigabyte JSON files locally https://thenybble.de/posts/json-analysis/
+
 * 5 great Perl scripts to keep in your sysadmin toolbox https://www.redhat.com/sysadmin/perl-scripts


--- a/src/parallel
+++ b/src/parallel
@ -2503,6 +2503,7 @@ sub parse_options(@) {
    if(defined $opt::bar) { $opt::progress = $opt::bar; }
    if(defined $opt::bar or defined $opt::latestline) {
 	my $fh = $Global::status_fd || *STDERR;
+	# Activate decode_utf8
 	eval q{
 	    # Enable utf8 if possible
 	    use utf8;
@ -2515,6 +2516,20 @@ sub parse_options(@) {
 	    # UTF8-decode not supported: Dummy decode
 	    eval q{sub decode_utf8($;$) { $_[0]; }};
 	}
+	# Activate decode_utf8
+	eval q{
+	    # Enable utf8 if possible
+	    use utf8;
+	    use Encode             qw( encode_utf8 );
+	    use Text::CharWidth    qw( mbswidth );
+	    use Unicode::Normalize qw( NFC NFD );
+	};
+	if(eval { mbswidth("ヌー平行") }) {
+	    # Great: mbswidth works
+	} else {
+	    # mbswidth not supported: Dummy mbswidth
+	    eval q{ sub mbswidth { return length @_; } };
+	}
    }

    # If you want GNU Parallel to be maintained in the future you
@ -2589,7 +2604,6 @@ sub parse_options(@) {
    # If you want GNU Parallel to be maintained in the future you
    # should keep this line.
    citation_notice();
-    
    # This is because _YOU_ actively make it harder to justify
    # spending time developing GNU Parallel by removing it.

@ -4558,6 +4572,17 @@ sub progress() {
    }
 }

+sub untabify($) {
+    # Convert \t into spaces
+    my @out;
+    my ($src);
+    # Deal with multi-byte characters
+    for my $src (split("\t",$_[0])) {
+	push @out, $src. " "x(8-mbswidth($src)%8);
+    }
+    return join "",@out;
+}
+
 # Prototype forwarding
 sub get_job_with_sshlogin($);
 sub get_job_with_sshlogin($) {
@ -11421,6 +11446,22 @@ sub print_files($) {
 	}
    }

+    sub truncate_mbs($$) {
+	my $str = shift;
+	my $len = shift;
+	if(::mbswidth($str) == length($str)) {
+	    $str = substr($str,0,$len);
+	} else {
+	    # Highly inefficient truncator
+	    while(::mbswidth($str) > $len) {
+		do {
+		    chop $str;
+		} while(::mbswidth($str) < 0);
+	    }
+	}
+	return $str;
+    }
+    
    sub print_latest_line($) {
 	my $self = shift;
 	my $out_fh = shift;
@ -11437,37 +11478,35 @@ sub print_files($) {
 	    eval q{ binmode $out_fh, "encoding(utf8)"; };
 	}
 	my ($color,$reset_color) = $self->color();
-	# Strings with TABs give the wrong length. Untabify strings
 	my $termcol = ::terminal_columns();
-	my $untabify_tag = ::decode_utf8($self->untabtag());
-	my $taglen = length $untabify_tag;
+	my $untabify_tag = $self->untabtag();
+	my $untabify_str = ::untabify($self->{$out_fh,'latestline'});
+	# -1 to make space for $truncated_str
+	my $maxtaglen = $termcol - 1;
+	$untabify_tag = truncate_mbs($untabify_tag,$maxtaglen);
+	my $taglen = ::mbswidth($untabify_tag);
+	my $maxstrlen = $termcol - $taglen - 1;
+	$untabify_str = truncate_mbs($untabify_str,$maxstrlen);
+	my $strlen = ::mbswidth($untabify_str);
 	my $truncated_tag = "";
-	my $untabify_str = ::decode_utf8($self->{$out_fh,'latestline'});
-	$untabify_str =~ s/\t/$tab{$-[0]%8}/g;
-	# -1 to make space for $truncated
-	my $strlen = $termcol - $taglen - 1;
-	my $strspc = $strlen - length $untabify_str;
-	if($strlen < 0) { $strlen = 0;}
-	# Line is shorter than terminal width: add " "
-	# Line is longer than terminal width: add ">"
-	my $truncated = ($strspc >= 0) ? " " : ">";
-	if($taglen > $termcol) {
-	    # Tag is longer than terminal width: add ">" to tag
-	    # Remove $truncated (it will not be shown at all)
-	    $taglen = $termcol - 1;
+	my $truncated_str = "";
+	if($termcol - $taglen < 2) {
 	    $truncated_tag = ">";
-	    $truncated = "";
+	} else {
+	    if($termcol - $taglen - $strlen <= 2) {
+		$truncated_str = ">";
+	    }
 	}
 	$maxrow = ($row > $maxrow) ? $row : $maxrow;
 	printf($out_fh
 	       ("%s%s%s%s". # up down \r eol
-		"%.${taglen}s%s". # tag trunc_tag
-		"%s%.${strlen}s%s%s". # color line trunc reset_color
+		"%s%s". # tag trunc_tag
+		"%s%s%s%s". # color line trunc reset_color
 		"%s" # down
 	       ),
 	       "$up"x($currow - $row), "\n"x($row - $currow), "\r", $eol,
-	       $untabify_tag,$truncated_tag,
-	       $color, $untabify_str, $truncated, $reset_color,
+	       ::decode_utf8($untabify_tag),$truncated_tag,
+	       $color, ::decode_utf8($untabify_str), $truncated_str, $reset_color,
 	       "\n"x($maxrow - $row + 1));
 	$currow = $maxrow + 1;
    }
@ -11898,9 +11937,7 @@ sub untabtag($) {
    my $self = shift;
    my $tag = $self->tag();
    if(not defined $self->{'untab'}{$tag}) {
-	my $t = $tag;
-	$t =~ s/\t/" "x(8-($-[0]%8))/eg;
-	$self->{'untab'}{$tag} = $t;
+	$self->{'untab'}{$tag} = ::untabify($tag);
    }
    return $self->{'untab'}{$tag};
 }
--- a/src/parallel.pod
+++ b/src/parallel.pod
@ -1586,9 +1586,9 @@ Similar to B<--memfree>.
 See also: B<--memfree> B<--load>


-=item B<--latest-line> (beta testing)
+=item B<--latest-line> (alpha testing)

-=item B<--ll> (beta testing)
+=item B<--ll> (alpha testing)

 Print the lastest line. Each job gets a single line that is updated
 with the lastest output from the job.
@ -1858,13 +1858,13 @@ Similar to B<--tty> but does not set B<--jobs> or B<--ungroup>.
 See also: B<--tty>


-=item B<--output-as-files> (alpha testing)
+=item B<--output-as-files> (beta testing)

-=item B<--outputasfiles> (alpha testing)
+=item B<--outputasfiles> (beta testing)

-=item B<--files> (alpha testing)
+=item B<--files> (beta testing)

-=item B<--files0> (alpha testing)
+=item B<--files0> (beta testing)

 Save output to files.

--- a/src/parsort
+++ b/src/parsort
@ -30,7 +30,7 @@ Same as B<sort>. Except:

 =over 4

-=item B<--parallel=>I<N> (alpha testing)
+=item B<--parallel=>I<N> (beta testing)

 Change the number of sorts run concurrently to I<N>. I<N> will be
 increased to number of files if B<parsort> is given more than I<N>
--- a/testsuite/input-files/perllib/Text/Abbrev.pm
+++ b/testsuite/input-files/perllib/Text/Abbrev.pm
@ -0,0 +1,83 @@
+package Text::Abbrev;
+require 5.005;		# Probably works on earlier versions too.
+require Exporter;
+
+our $VERSION = '1.00';
+
+=head1 NAME
+
+abbrev - create an abbreviation table from a list
+
+=head1 SYNOPSIS
+
+    use Text::Abbrev;
+    abbrev $hashref, LIST
+
+
+=head1 DESCRIPTION
+
+Stores all unambiguous truncations of each element of LIST
+as keys in the associative array referenced by C<$hashref>.
+The values are the original list elements.
+
+=head1 EXAMPLE
+
+    $hashref = abbrev qw(list edit send abort gripe);
+
+    %hash = abbrev qw(list edit send abort gripe);
+
+    abbrev $hashref, qw(list edit send abort gripe);
+
+    abbrev(*hash, qw(list edit send abort gripe));
+
+=cut
+
+@ISA = qw(Exporter);
+@EXPORT = qw(abbrev);
+
+# Usage:
+#	abbrev \%foo, LIST;
+#	...
+#	$long = $foo{$short};
+
+sub abbrev {
+    my ($word, $hashref, $glob, %table, $returnvoid);
+
+    if (ref($_[0])) {           # hash reference preferably
+      $hashref = shift;
+      $returnvoid = 1;
+    } elsif (ref \$_[0] eq 'GLOB') {  # is actually a glob (deprecated)
+      $hashref = \%{shift()};
+      $returnvoid = 1;
+    }
+    %{$hashref} = ();
+
+    WORD: foreach $word (@_) {
+        for (my $len = (length $word) - 1; $len > 0; --$len) {
+	    my $abbrev = substr($word,0,$len);
+	    my $seen = ++$table{$abbrev};
+	    if ($seen == 1) {	    # We're the first word so far to have
+	    			    # this abbreviation.
+	        $hashref->{$abbrev} = $word;
+	    } elsif ($seen == 2) {  # We're the second word to have this
+	    			    # abbreviation, so we can't use it.
+	        delete $hashref->{$abbrev};
+	    } else {		    # We're the third word to have this
+	    			    # abbreviation, so skip to the next word.
+	        next WORD;
+	    }
+	}
+    }
+    # Non-abbreviations always get entered, even if they aren't unique
+    foreach $word (@_) {
+        $hashref->{$word} = $word;
+    }
+    return if $returnvoid;
+    if (wantarray) {
+      %{$hashref};
+    } else {
+      $hashref;
+    }
+}
+
+1;
--- a/testsuite/input-files/perllib/Text/Balanced.pm
+++ b/testsuite/input-files/perllib/Text/Balanced.pm
--- a/testsuite/input-files/perllib/Text/Balanced/Changes
+++ b/testsuite/input-files/perllib/Text/Balanced/Changes
@ -0,0 +1,263 @@
+Revision history for Perl extension Text::Balanced.
+
+1.00  Mon Aug 11 12:42:56 1997
+
+	- original version
+
+
+1.01  Mon Sep  8 18:09:18 EST 1997
+
+	- changed "quotemeta" to "quotemeta $_" to work 
+	  around bug in Perl 5.002 and 5.003
+
+
+1.10  Tue Sep 30 17:23:23 EST 1997
+
+	- reworked extract_quotelike to correct handling of some obscure cases
+
+
+1.21  Sat Oct  4 17:21:54 EST 1997
+
+	- synchronised with Parse::RecDescent distribution (version number
+	  will now reflect that package)
+
+1.23  Fri Oct 17 10:26:38 EST 1997
+
+	- changed behaviour in scalar and void contexts. Scalar contexts
+	  now return only the extracted string. Void contexts now remove
+	  the extracted string from the first argument (or $_).
+
+1.24
+
+	- changed behaviour in scalar contexts. Scalar contexts
+	  now return the extracted string _and_ remove it from the
+	  first argument (or $_).
+
+	- changed return values on failure (all contexts return undef
+	  for invalid return fields)
+
+	- fixed some lurking bugs with trailing modifier handling
+	
+	- added :ALL tag to simplify wholesale importing of functions
+
+	- fixed serious bug with embedded division operators ("/")
+	  This now also allows the ?...? form of pattern matching!
+
+1.35	Wed Jun 24 09:53:31 1998
+
+	- fixed handling of :: quantifiers in extract_variable()
+
+	- numerous trivial lexical changes to make xemacs happy
+
+
+1.36	Tue Jul 14 12:26:04 1998
+
+	- Reinstated POD file missing from previous distribution
+
+	- Added undocumented fourth parameter to extract_codeblock
+	  so as to correctly handle (?) and (s?) modifiers in
+	  RecDescent grammars.
+
+
+1.40	Tue Aug  4 13:54:52 1998
+
+        - Added (optional) handling of embedded quoted text to
+          extract_delimited (see revised entry in Balanced.pod)
+
+        - Added extract_tagged which extracts text between arbitrary,
+          optionally nested start and end tags (see new entry in
+          Balanced.pod).
+
+        - Added delimited_pat which builds a pattern which matches a
+          string delimited by any of the delimiters specified (see new
+          entry in Balanced.pod).
+
+	- Added test.pl
+
+
+1.41	Mon Aug 10 14:51:50 1998
+
+	- Reinstated change to extract_codeblock from 1.36 which were
+	  mysteriously lost in 1.40
+
+
+1.50	Thu Aug 27 09:20:19 1998
+
+	- Improved the structure of the regex generated by
+	  delimited_pat (and used in extract_delimited). It's
+	  considerably more complex, but also more robust and 
+	  much faster in the worst case.
+
+	- Altered extract_variable to accept whitespace in variables,
+	  e.g. '$ a -> {'b'}     ->  [2]'
+	  
+
+
+1.51	Sat Feb 13 10:31:55 1999
+
+	- Fixed bugs in prefix matching in extract_variable:
+	  	* incorrectly used default if '' specified
+		* now handles $#array correctly
+
+	- Fixed bugs in extract_codeblock:
+		  * Now handles !~ properly
+		  * Now handles embedded comments better.
+		  * Now handles "raw" pattern matches better.
+
+	- Added support for single strings or qr's as
+	  'reject' and 'ignore' args to extract_tagged()
+
+	- Added gen_extract_tagged() to "precompile"
+	  a specific tag extractor for repeated use
+	  (approximately 3 times faster!)
+
+
+1.52	Thu Mar  4 12:43:38 1999
+
+	- Added CSV parsing example to documentation of extract_multiple.
+
+	- Fixed a bug with extract_codeblock in "RecDescent" mode
+	  (it would accept "subrule(s?)" and "subrule(?)", but
+	   not "subrule(s)"). Thanks, Jan.
+
+
+1.66	Fri Jul  2 13:29:22 1999
+
+	- Added ability to use quotelike operators in extract_bracketed
+
+	- Fixed bug under 5.003 ('foreach my $func' not understood)
+
+	- Added escape specification as fourth arg to &extract_delimited
+
+	- Fixed handling of &delimited_pat and &extract_delimited
+	  when delimiter is same as escape
+
+	- Fixed handling of ->, =>, and >> in &extract_code
+	  when delimiters are "<>"
+
+
+1.76	Fri Nov 19 06:51:54 1999
+
+	- IMPORTANT: Now requires 5.005 or better.
+
+	- IMPORTANT: Made extract methods sensitive to the pos()
+	  value of the text they are parsing. In other words,
+	  all extract subroutines now act like patterns of the form
+	  /\G.../gc. See documentation for details.
+
+	- IMPORTANT: Changed semantics of extract_multiple, in line
+	  with the above change, and to simplify the semantics to
+	  something vaguely predictable. See documentation for details.
+
+	- Added ability to use qr/../'s and raw strings as extractors
+	  in extract_multiple. See documentation.
+
+	- Added fourth argument to extract_codeblock to allow
+	  outermost brackets to be separately specified. See
+	  documentation for details.
+
+	- Reimplemented internals of all extraction subroutines
+	  for significant speed-ups (between 100% and 2000%
+	  improvement).
+
+	- Fixed nasty bug in extract_variable and extract_codeblock
+	  (they were returning prefix as well in scalar context)
+
+	- Allowed read-only strings to be used as arguments in
+	  scalar contexts.
+
+	- Renamed delimited_pat to gen-delimited pat (in line with
+	  gen_extract_tagged). Old name still works, but is now deprecated.
+
+	- Tweaked all extraction subs so they correctly handle 
+	  zero-length prefix matches after another zero-length match.
+
+
+1.77	Mon Nov 22 06:08:23 1999
+
+	- Fixed major bug in extract_codeblock (would not 
+	  terminate if there was trailing whitespace)
+
+	- Improved /.../ pattern parsing within codeblocks
+
+
+1.81	Wed Sep 13 11:58:49 2000
+
+	- Fixed test count in extract_codeblock.t
+
+	- Fixed improbable bug with trailing ->'s in extract_variable
+
+	- Fixed (HT|X)ML tag extraction in extract_tagged (thanks, Tim)
+
+	- Added explanatory note about prefix matching (thanks again, Tim)
+
+	- Added handling of globs and sub refs to extract_variable
+
+	- Pod tweak (thanks Abigail)
+
+	- Allowed right tags to be run-time evaluated, so
+	  extract_tagged($text, '/([a-z]+)', '/end$1') works
+	  as expected.
+
+	- Added optional blessing of matches via extract_multiple
+
+	- Fixed bug in autogeneration of closing tags in extract_tagged
+	  (Thanks, Coke)
+
+	- Fixed bug in interaction between extract_multiple and
+	  gen_extract_tagged (Thanks Anthony)
+
+
+1.82	Sun Jan 14 16:56:04 2001
+
+	- Fixed nit in extract_variable.t
+	  (tested more cases than it promised to)
+
+	- Fixed bug extracting prefix in extract_quotelike (Thanks Michael)
+
+	- Added handling of Perl 4 package qualifier: $Package'var, etc.
+
+	- Added handling of here docs (see documentation for limitations)
+
+	- Added reporting of failure position via $@->{pos} (see documentation)
+
+
+1.83	Mon Jan 15 12:43:12 2001
+
+	- Fixed numerous bugs in here doc extraction (many thanks Tim)
+
+
+1.84	Thu Apr 26 11:58:13 2001
+
+	- Fixed bug in certain extractions not matching strings
+	  with embedded newlines (thanks Robin)
+
+
+1.85	Sun Jun  3 07:47:18 2001
+
+	- Fixed bug in extract_variable recognizing method calls that
+	  start with an underscore (thanks Jeff)
+
+
+1.86	Mon Sep  3 06:57:08 2001
+
+	- Revised licence for inclusion in core distribution
+
+	- Consolidated POD in .pm file
+
+	- renamed tests to let DOS cope with them
+
+
+1.87	Thu Nov 15 21:25:35 2001
+
+	- Made extract_multiple aware of skipped prefixes returned
+	  by subroutine extractors (such as extract_quotelike, etc.)
+
+	- Made extract_variable aware of punctuation variables
+
+	- Corified tests
+
+
+1.89	Sun Nov 18 22:49:50 2001
+
+	- Fixed extvar.t tests
--- a/testsuite/input-files/perllib/Text/Balanced/README
+++ b/testsuite/input-files/perllib/Text/Balanced/README
@ -0,0 +1,84 @@
+==============================================================================
+                  Release of version 1.89 of Text::Balanced
+==============================================================================
+
+
+NAME
+
+    Text::Balanced - Extract delimited text sequences from strings.
+
+
+SUMMARY (see Balanced.pod for full details)
+
+    Text::Balanced::extract_delimited
+    
+        `extract_delimited' extracts the initial substring of a string
+        which is delimited by a user-specified set of single-character
+        delimiters, whilst ignoring any backslash-escaped delimiter
+        characters.
+
+    Text::Balanced::extract_bracketed
+    
+        `extract_bracketed' extracts a balanced-bracket-delimited substring
+        (using any one (or more) of the user-specified delimiter brackets:
+        '(..)', '{..}', '[..]', or '<..>').
+    
+    Text::Balanced::extract_quotelike
+    
+        `extract_quotelike' attempts to recognize and extract any one of the
+        various Perl quote and quotelike operators (see "perlop(3)"). Embedded
+        backslashed delimiters, nested bracket delimiters (for the
+        quotelike operators), and trailing modifiers are all correctly handled.
+    
+    Text::Balanced::extract_codeblock
+    
+        `extract_codeblock' attempts to recognize and extract a
+        balanced bracket-delimited substring which may also contain
+        unbalanced brackets inside Perl quotes or quotelike
+        operations. That is, `extract_codeblock' is like a combination
+        of `extract_bracketed' and `extract_quotelike'.
+
+    Text::Balanced::extract_tagged
+    
+        `extract_tagged' attempts to recognize and extract a
+        substring between two arbitrary "tag" patterns (a start tag
+	and an end tag).
+
+    
+INSTALLATION
+
+    It's all pure Perl, so just put the .pm file in its appropriate
+    local Perl subdirectory.
+
+
+AUTHOR
+
+    Damian Conway (damian@cs.monash.edu.au)
+
+
+COPYRIGHT
+
+     Copyright (c) 1997-2001, Damian Conway. All Rights Reserved.
+     This module is free software. It may be used, redistributed
+	  and/or modified under the same terms as Perl itself.
+
+
+
+==============================================================================
+
+CHANGES IN VERSION 1.89
+
+
+	- Fixed extvar.t tests
+
+
+==============================================================================
+
+AVAILABILITY
+
+Text::Balanced has been uploaded to the CPAN
+and is also available from:
+
+	http://www.csse.monash.edu.au/~damian/CPAN/Text-Balanced.tar.gz
+
+==============================================================================
--- a/testsuite/input-files/perllib/Text/ParseWords.pm
+++ b/testsuite/input-files/perllib/Text/ParseWords.pm
@ -0,0 +1,263 @@
+package Text::ParseWords;
+
+use vars qw($VERSION @ISA @EXPORT $PERL_SINGLE_QUOTE);
+$VERSION = "3.21";
+
+require 5.000;
+
+use Exporter;
+@ISA = qw(Exporter);
+@EXPORT = qw(shellwords quotewords nested_quotewords parse_line);
+@EXPORT_OK = qw(old_shellwords);
+
+
+sub shellwords {
+    local(@lines) = @_;
+    $lines[$#lines] =~ s/\s+$//;
+    return(quotewords('\s+', 0, @lines));
+}
+
+
+
+sub quotewords {
+    my($delim, $keep, @lines) = @_;
+    my($line, @words, @allwords);
+    
+
+    foreach $line (@lines) {
+	@words = parse_line($delim, $keep, $line);
+	return() unless (@words || !length($line));
+	push(@allwords, @words);
+    }
+    return(@allwords);
+}
+
+
+
+sub nested_quotewords {
+    my($delim, $keep, @lines) = @_;
+    my($i, @allwords);
+    
+    for ($i = 0; $i < @lines; $i++) {
+	@{$allwords[$i]} = parse_line($delim, $keep, $lines[$i]);
+	return() unless (@{$allwords[$i]} || !length($lines[$i]));
+    }
+    return(@allwords);
+}
+
+
+
+sub parse_line {
+	# We will be testing undef strings
+	no warnings;
+	use re 'taint'; # if it's tainted, leave it as such
+
+    my($delimiter, $keep, $line) = @_;
+    my($quote, $quoted, $unquoted, $delim, $word, @pieces);
+
+    while (length($line)) {
+
+	($quote, $quoted, undef, $unquoted, $delim, undef) =
+	    $line =~ m/^(["'])                 # a $quote
+                        ((?:\\.|(?!\1)[^\\])*)    # and $quoted text
+                        \1 		       # followed by the same quote
+                        ([\000-\377]*)	       # and the rest
+		       |                       # --OR--
+                       ^((?:\\.|[^\\"'])*?)    # an $unquoted text
+		      (\Z(?!\n)|(?-x:$delimiter)|(?!^)(?=["']))  
+                                               # plus EOL, delimiter, or quote
+                      ([\000-\377]*)	       # the rest
+		      /x;		       # extended layout
+	return() unless( $quote || length($unquoted) || length($delim));
+
+	$line = $+;
+
+        if ($keep) {
+	    $quoted = "$quote$quoted$quote";
+	}
+        else {
+	    $unquoted =~ s/\\(.)/$1/g;
+	    if (defined $quote) {
+		$quoted =~ s/\\(.)/$1/g if ($quote eq '"');
+		$quoted =~ s/\\([\\'])/$1/g if ( $PERL_SINGLE_QUOTE && $quote eq "'");
+            }
+	}
+        $word .= defined $quote ? $quoted : $unquoted;
+ 
+        if (length($delim)) {
+            push(@pieces, $word);
+            push(@pieces, $delim) if ($keep eq 'delimiters');
+            undef $word;
+        }
+        if (!length($line)) {
+            push(@pieces, $word);
+	}
+    }
+    return(@pieces);
+}
+
+
+
+sub old_shellwords {
+
+    # Usage:
+    #	use ParseWords;
+    #	@words = old_shellwords($line);
+    #	or
+    #	@words = old_shellwords(@lines);
+
+    local($_) = join('', @_);
+    my(@words,$snippet,$field);
+
+    s/^\s+//;
+    while ($_ ne '') {
+	$field = '';
+	for (;;) {
+	    if (s/^"(([^"\\]|\\.)*)"//) {
+		($snippet = $1) =~ s#\\(.)#$1#g;
+	    }
+	    elsif (/^"/) {
+		return();
+	    }
+	    elsif (s/^'(([^'\\]|\\.)*)'//) {
+		($snippet = $1) =~ s#\\(.)#$1#g;
+	    }
+	    elsif (/^'/) {
+		return();
+	    }
+	    elsif (s/^\\(.)//) {
+		$snippet = $1;
+	    }
+	    elsif (s/^([^\s\\'"]+)//) {
+		$snippet = $1;
+	    }
+	    else {
+		s/^\s+//;
+		last;
+	    }
+	    $field .= $snippet;
+	}
+	push(@words, $field);
+    }
+    @words;
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+Text::ParseWords - parse text into an array of tokens or array of arrays
+
+=head1 SYNOPSIS
+
+  use Text::ParseWords;
+  @lists = &nested_quotewords($delim, $keep, @lines);
+  @words = &quotewords($delim, $keep, @lines);
+  @words = &shellwords(@lines);
+  @words = &parse_line($delim, $keep, $line);
+  @words = &old_shellwords(@lines); # DEPRECATED!
+
+=head1 DESCRIPTION
+
+The &nested_quotewords() and &quotewords() functions accept a delimiter 
+(which can be a regular expression)
+and a list of lines and then breaks those lines up into a list of
+words ignoring delimiters that appear inside quotes.  &quotewords()
+returns all of the tokens in a single long list, while &nested_quotewords()
+returns a list of token lists corresponding to the elements of @lines.
+&parse_line() does tokenizing on a single string.  The &*quotewords()
+functions simply call &parse_lines(), so if you're only splitting
+one line you can call &parse_lines() directly and save a function
+call.
+
+The $keep argument is a boolean flag.  If true, then the tokens are
+split on the specified delimiter, but all other characters (quotes,
+backslashes, etc.) are kept in the tokens.  If $keep is false then the
+&*quotewords() functions remove all quotes and backslashes that are
+not themselves backslash-escaped or inside of single quotes (i.e.,
+&quotewords() tries to interpret these characters just like the Bourne
+shell).  NB: these semantics are significantly different from the
+original version of this module shipped with Perl 5.000 through 5.004.
+As an additional feature, $keep may be the keyword "delimiters" which
+causes the functions to preserve the delimiters in each string as
+tokens in the token lists, in addition to preserving quote and
+backslash characters.
+
+&shellwords() is written as a special case of &quotewords(), and it
+does token parsing with whitespace as a delimiter-- similar to most
+Unix shells.
+
+=head1 EXAMPLES
+
+The sample program:
+
+  use Text::ParseWords;
+  @words = &quotewords('\s+', 0, q{this   is "a test" of\ quotewords \"for you});
+  $i = 0;
+  foreach (@words) {
+      print "$i: <$_>\n";
+      $i++;
+  }
+
+produces:
+
+  0: <this>
+  1: <is>
+  2: <a test>
+  3: <of quotewords>
+  4: <"for>
+  5: <you>
+
+demonstrating:
+
+=over 4
+
+=item 0
+
+a simple word
+
+=item 1
+
+multiple spaces are skipped because of our $delim
+
+=item 2
+
+use of quotes to include a space in a word
+
+=item 3
+
+use of a backslash to include a space in a word
+
+=item 4
+
+use of a backslash to remove the special meaning of a double-quote
+
+=item 5
+
+another simple word (note the lack of effect of the
+backslashed double-quote)
+
+=back
+
+Replacing C<&quotewords('\s+', 0, q{this   is...})>
+with C<&shellwords(q{this   is...})>
+is a simpler way to accomplish the same thing.
+
+=head1 AUTHORS
+
+Maintainer is Hal Pomeranz <pomeranz@netcom.com>, 1994-1997 (Original
+author unknown).  Much of the code for &parse_line() (including the
+primary regexp) from Joerk Behrends <jbehrends@multimediaproduzenten.de>.
+
+Examples section another documentation provided by John Heidemann 
+<johnh@ISI.EDU>
+
+Bug reports, patches, and nagging provided by lots of folks-- thanks
+everybody!  Special thanks to Michael Schwern <schwern@envirolink.org>
+for assuring me that a &nested_quotewords() would be useful, and to 
+Jeff Friedl <jfriedl@yahoo-inc.com> for telling me not to worry about
+error-checking (sort of-- you had to be there).
+
+=cut
--- a/testsuite/input-files/perllib/Text/Soundex.pm
+++ b/testsuite/input-files/perllib/Text/Soundex.pm
@ -0,0 +1,150 @@
+package Text::Soundex;
+require 5.000;
+require Exporter;
+
+@ISA = qw(Exporter);
+@EXPORT = qw(&soundex $soundex_nocode);
+
+$VERSION = '1.01';
+
+# $Id: soundex.pl,v 1.2 1994/03/24 00:30:27 mike Exp $
+#
+# Implementation of soundex algorithm as described by Knuth in volume
+# 3 of The Art of Computer Programming, with ideas stolen from Ian
+# Phillipps <ian@pipex.net>.
+#
+# Mike Stok <Mike.Stok@meiko.concord.ma.us>, 2 March 1994.
+#
+# Knuth's test cases are:
+# 
+# Euler, Ellery -> E460
+# Gauss, Ghosh -> G200
+# Hilbert, Heilbronn -> H416
+# Knuth, Kant -> K530
+# Lloyd, Ladd -> L300
+# Lukasiewicz, Lissajous -> L222
+#
+# $Log: soundex.pl,v $
+# Revision 1.2  1994/03/24  00:30:27  mike
+# Subtle bug (any excuse :-) spotted by Rich Pinder <rpinder@hsc.usc.edu>
+# in the way I handles leasing characters which were different but had
+# the same soundex code.  This showed up comparing it with Oracle's
+# soundex output.
+#
+# Revision 1.1  1994/03/02  13:01:30  mike
+# Initial revision
+#
+#
+##############################################################################
+
+# $soundex_nocode is used to indicate a string doesn't have a soundex
+# code, I like undef other people may want to set it to 'Z000'.
+
+$soundex_nocode = undef;
+
+sub soundex
+{
+  local (@s, $f, $fc, $_) = @_;
+
+  push @s, '' unless @s;	# handle no args as a single empty string
+
+  foreach (@s)
+  {
+    $_ = uc $_;
+    tr/A-Z//cd;
+
+    if ($_ eq '')
+    {
+      $_ = $soundex_nocode;
+    }
+    else
+    {
+      ($f) = /^(.)/;
+      tr/AEHIOUWYBFPVCGJKQSXZDTLMNR/00000000111122222222334556/;
+      ($fc) = /^(.)/;
+      s/^$fc+//;
+      tr///cs;
+      tr/0//d;
+      $_ = $f . $_ . '000';
+      s/^(.{4}).*/$1/;
+    }
+  }
+
+  wantarray ? @s : shift @s;
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+Text::Soundex - Implementation of the Soundex Algorithm as Described by Knuth
+
+=head1 SYNOPSIS
+
+  use Text::Soundex;
+
+  $code = soundex $string;            # get soundex code for a string
+  @codes = soundex @list;             # get list of codes for list of strings
+
+  # set value to be returned for strings without soundex code
+
+  $soundex_nocode = 'Z000';
+
+=head1 DESCRIPTION
+
+This module implements the soundex algorithm as described by Donald Knuth
+in Volume 3 of B<The Art of Computer Programming>.  The algorithm is
+intended to hash words (in particular surnames) into a small space using a
+simple model which approximates the sound of the word when spoken by an English
+speaker.  Each word is reduced to a four character string, the first
+character being an upper case letter and the remaining three being digits.
+
+If there is no soundex code representation for a string then the value of
+C<$soundex_nocode> is returned.  This is initially set to C<undef>, but
+many people seem to prefer an I<unlikely> value like C<Z000>
+(how unlikely this is depends on the data set being dealt with.)  Any value
+can be assigned to C<$soundex_nocode>.
+
+In scalar context C<soundex> returns the soundex code of its first
+argument, and in list context a list is returned in which each element is the 
+soundex code for the corresponding argument passed to C<soundex> e.g.
+
+  @codes = soundex qw(Mike Stok);
+
+leaves C<@codes> containing C<('M200', 'S320')>.
+
+=head1 EXAMPLES
+
+Knuth's examples of various names and the soundex codes they map to
+are listed below:
+
+  Euler, Ellery -> E460
+  Gauss, Ghosh -> G200
+  Hilbert, Heilbronn -> H416
+  Knuth, Kant -> K530
+  Lloyd, Ladd -> L300
+  Lukasiewicz, Lissajous -> L222
+
+so:
+
+  $code = soundex 'Knuth';              # $code contains 'K530'
+  @list = soundex qw(Lloyd Gauss);	# @list contains 'L300', 'G200'
+
+=head1 LIMITATIONS
+
+As the soundex algorithm was originally used a B<long> time ago in the US
+it considers only the English alphabet and pronunciation.
+
+As it is mapping a large space (arbitrary length strings) onto a small
+space (single letter plus 3 digits) no inference can be made about the
+similarity of two strings which end up with the same soundex code.  For 
+example, both C<Hilbert> and C<Heilbronn> end up with a soundex code
+of C<H416>.
+
+=head1 AUTHOR
+
+This code was implemented by Mike Stok (C<stok@cybercom.net>) from the 
+description given by Knuth.  Ian Phillipps (C<ian@pipex.net>) and Rich Pinder 
+(C<rpinder@hsc.usc.edu>) supplied ideas and spotted mistakes.
--- a/testsuite/input-files/perllib/Text/Tabs.pm
+++ b/testsuite/input-files/perllib/Text/Tabs.pm
@ -0,0 +1,97 @@
+
+package Text::Tabs;
+
+require Exporter;
+
+@ISA = (Exporter);
+@EXPORT = qw(expand unexpand $tabstop);
+
+use vars qw($VERSION $tabstop $debug);
+$VERSION = 98.112801;
+
+use strict;
+
+BEGIN	{
+	$tabstop = 8;
+	$debug = 0;
+}
+
+sub expand
+{
+	my (@l) = @_;
+	for $_ (@l) {
+		1 while s/(^|\n)([^\t\n]*)(\t+)/
+			$1. $2 . (" " x 
+				($tabstop * length($3)
+				- (length($2) % $tabstop)))
+			/sex;
+	}
+	return @l if wantarray;
+	return $l[0];
+}
+
+sub unexpand
+{
+	my (@l) = @_;
+	my @e;
+	my $x;
+	my $line;
+	my @lines;
+	my $lastbit;
+	for $x (@l) {
+		@lines = split("\n", $x, -1);
+		for $line (@lines) {
+			$line = expand($line);
+			@e = split(/(.{$tabstop})/,$line,-1);
+			$lastbit = pop(@e);
+			$lastbit = '' unless defined $lastbit;
+			$lastbit = "\t"
+				if $lastbit eq " "x$tabstop;
+			for $_ (@e) {
+				if ($debug) {
+					my $x = $_;
+					$x =~ s/\t/^I\t/gs;
+					print "sub on '$x'\n";
+				}
+				s/  +$/\t/;
+			}
+			$line = join('',@e, $lastbit);
+		}
+		$x = join("\n", @lines);
+	}
+	return @l if wantarray;
+	return $l[0];
+}
+
+1;
+__END__
+
+
+=head1 NAME
+
+Text::Tabs -- expand and unexpand tabs per the unix expand(1) and unexpand(1)
+
+=head1 SYNOPSIS
+
+  use Text::Tabs;
+
+  $tabstop = 4;
+  @lines_without_tabs = expand(@lines_with_tabs);
+  @lines_with_tabs = unexpand(@lines_without_tabs);
+
+=head1 DESCRIPTION
+
+Text::Tabs does about what the unix utilities expand(1) and unexpand(1)
+do.  Given a line with tabs in it, expand will replace the tabs with
+the appropriate number of spaces.  Given a line with or without tabs in
+it, unexpand will add tabs when it can save bytes by doing so.  Invisible
+compression with plain ascii!
+
+=head1 BUGS
+
+expand doesn't handle newlines very quickly -- do not feed it an
+entire document in one string.  Instead feed it an array of lines.
+
+=head1 AUTHOR
+
+David Muir Sharnoff <muir@idiom.com>
--- a/testsuite/input-files/perllib/Text/TabsWrap/CHANGELOG
+++ b/testsuite/input-files/perllib/Text/TabsWrap/CHANGELOG
@ -0,0 +1,74 @@
+= 2001/09/29
+
+Philip Newton <Philip.Newton@gmx.net> sent in a clean patch that
+added support for defining words differently; that prevents 
+Text::Wrap from untainting strings; and that fixes a documentation
+bug.
+
+So that fill.t can be used in the version included in the perl
+distribution, fill.t no longer uses File::Slurp.
+
+Both Sweth Chandramouli <svc@sweth.net> and Drew Degentesh 
+<ddegentesh@daed.com> both objected to the automatic unexpand
+that Text::Wrap does on its results.  Drew sent a patch which
+has been integrated.
+
+Way back in '97, Joel Earl <jrearl@VNET.IBM.COM> asked that
+it be possible to use a line separator other than \n when
+adding new lines.  There is now support for that.
+
+= 2001/01/30
+
+Bugfix by Michael G Schwern <schwern@pobox.com>: don't add extra
+whitespace when working one an array of input (as opposed to a 
+single string).
+
+Performance rewrite: use m/\G/ rather than s///.
+
+You can now specify that words that are too long to wrap can simply
+overflow the line.  Feature requested by James Hoagland 
+<hoagland@SiliconDefense.com> and by John Porter <jdporter@min.net>.
+
+Documentation changes from Rich Bowen <Rich@cre8tivegroup.com>.
+
+= 1998/11/29
+
+Combined Fill.pm into Wrap.pm.  It appears there are versions of
+Wrap.pm with fill in them.
+
+= 1998/11/28
+
+Over the last couple of years, many people sent in various
+rewrites of Text::Wrap.  I should have done something about
+updating it long ago.  If someone wants to take it over from
+me, discuss it in perl-porters.  I'll be happy to hand it
+over.
+
+Anyway, I have a bunch of people to thank.  I didn't
+use what any of them sent in, but I did take ideas from
+all of them.  Many sent in complete new implamentations.
+
+	Ivan Brawley <ibrawley@awadi.com.au> 
+
+	Jacqui Caren <Jacqui.Caren@ig.co.uk>
+
+	Jeff Kowalski <jeff.kowalski@autodesk.com>
+
+	Allen Smith <easmith@beatrice.rutgers.edu>
+
+	Sullivan N. Beck <sbeck@cise.ufl.edu>
+
+The end result is a very slight change in the API.  There
+is now an additional package variable: $Text::Wrap::huge.
+When $huge is set to 'die' then long words will cause 
+wrap() to die.  When it is set to 'wrap', long words will
+be wrapped.  The default is 'wrap'.
+
+<shout>LONG WORDS WILL NOW BE WRAPPED BY DEFAULT</shout>.  
+This is a change in behavior.
+
+At the bottom of Text::Wrap, there was a function (fill())
+sitting there unpublished.  There was a note that Tim Pierce
+had a faster version, but a search on CPAN failed to turn it
+up.  Text::Fill is now available.
+
--- a/testsuite/input-files/perllib/Text/Wrap.pm
+++ b/testsuite/input-files/perllib/Text/Wrap.pm
@ -0,0 +1,212 @@
+package Text::Wrap;
+
+require Exporter;
+
+@ISA = qw(Exporter);
+@EXPORT = qw(wrap fill);
+@EXPORT_OK = qw($columns $break $huge);
+
+$VERSION = 2001.0929;
+
+use vars qw($VERSION $columns $debug $break $huge $unexpand $tabstop
+	$separator);
+use strict;
+
+BEGIN	{
+	$columns = 76;  # <= screen width
+	$debug = 0;
+	$break = '\s';
+	$huge = 'wrap'; # alternatively: 'die' or 'overflow'
+	$unexpand = 1;
+	$tabstop = 8;
+	$separator = "\n";
+}
+
+use Text::Tabs qw(expand unexpand);
+
+sub wrap
+{
+	my ($ip, $xp, @t) = @_;
+
+	local($Text::Tabs::tabstop) = $tabstop;
+	my $r = "";
+	my $tail = pop(@t);
+	my $t = expand(join("", (map { /\s+\z/ ? ( $_ ) : ($_, ' ') } @t), $tail));
+	my $lead = $ip;
+	my $ll = $columns - length(expand($ip)) - 1;
+	$ll = 0 if $ll < 0;
+	my $nll = $columns - length(expand($xp)) - 1;
+	my $nl = "";
+	my $remainder = "";
+
+	use re 'taint';
+
+	pos($t) = 0;
+	while ($t !~ /\G\s*\Z/gc) {
+		if ($t =~ /\G([^\n]{0,$ll})($break|\z)/xmgc) {
+			$r .= $unexpand 
+				? unexpand($nl . $lead . $1)
+				: $nl . $lead . $1;
+			$remainder = $2;
+		} elsif ($huge eq 'wrap' && $t =~ /\G([^\n]{$ll})/gc) {
+			$r .= $unexpand 
+				? unexpand($nl . $lead . $1)
+				: $nl . $lead . $1;
+			$remainder = $separator;
+		} elsif ($huge eq 'overflow' && $t =~ /\G([^\n]*?)($break|\z)/xmgc) {
+			$r .= $unexpand 
+				? unexpand($nl . $lead . $1)
+				: $nl . $lead . $1;
+			$remainder = $2;
+		} elsif ($huge eq 'die') {
+			die "couldn't wrap '$t'";
+		} else {
+			die "This shouldn't happen";
+		}
+			
+		$lead = $xp;
+		$ll = $nll;
+		$nl = $separator;
+	}
+	$r .= $remainder;
+
+	print "-----------$r---------\n" if $debug;
+
+	print "Finish up with '$lead'\n" if $debug;
+
+	$r .= $lead . substr($t, pos($t), length($t)-pos($t))
+		if pos($t) ne length($t);
+
+	print "-----------$r---------\n" if $debug;;
+
+	return $r;
+}
+
+sub fill 
+{
+	my ($ip, $xp, @raw) = @_;
+	my @para;
+	my $pp;
+
+	for $pp (split(/\n\s+/, join("\n",@raw))) {
+		$pp =~ s/\s+/ /g;
+		my $x = wrap($ip, $xp, $pp);
+		push(@para, $x);
+	}
+
+	# if paragraph_indent is the same as line_indent, 
+	# separate paragraphs with blank lines
+
+	my $ps = ($ip eq $xp) ? "\n\n" : "\n";
+	return join ($ps, @para);
+}
+
+1;
+__END__
+
+=head1 NAME
+
+Text::Wrap - line wrapping to form simple paragraphs
+
+=head1 SYNOPSIS 
+
+B<Example 1>
+
+	use Text::Wrap
+
+	$initial_tab = "\t";	# Tab before first line
+	$subsequent_tab = "";	# All other lines flush left
+
+	print wrap($initial_tab, $subsequent_tab, @text);
+	print fill($initial_tab, $subsequent_tab, @text);
+
+	@lines = wrap($initial_tab, $subsequent_tab, @text);
+
+	@paragraphs = fill($initial_tab, $subsequent_tab, @text);
+
+B<Example 2>
+
+	use Text::Wrap qw(wrap $columns $huge);
+
+	$columns = 132;		# Wrap at 132 characters
+	$huge = 'die';
+	$huge = 'wrap';
+	$huge = 'overflow';
+
+B<Example 3>
+
+	use Text::Wrap
+
+	$Text::Wrap::columns = 72;
+	print wrap('', '', @text);
+
+=head1 DESCRIPTION
+
+C<Text::Wrap::wrap()> is a very simple paragraph formatter.  It formats a
+single paragraph at a time by breaking lines at word boundries.
+Indentation is controlled for the first line (C<$initial_tab>) and
+all subsequent lines (C<$subsequent_tab>) independently.  Please note: 
+C<$initial_tab> and C<$subsequent_tab> are the literal strings that will
+be used: it is unlikley you would want to pass in a number.
+
+Text::Wrap::fill() is a simple multi-paragraph formatter.  It formats
+each paragraph separately and then joins them together when it's done.  It
+will destory any whitespace in the original text.  It breaks text into
+paragraphs by looking for whitespace after a newline.  In other respects
+it acts like wrap().
+
+=head1 OVERRIDES
+
+C<Text::Wrap::wrap()> has a number of variables that control its behavior.
+Because other modules might be using C<Text::Wrap::wrap()> it is suggested
+that you leave these variables alone!  If you can't do that, then 
+use C<local($Text::Wrap::VARIABLE) = YOURVALUE> when you change the
+values so that the original value is restored.  This C<local()> trick
+will not work if you import the variable into your own namespace.
+
+Lines are wrapped at C<$Text::Wrap::columns> columns.  C<$Text::Wrap::columns>
+should be set to the full width of your output device.  In fact,
+every resulting line will have length of no more than C<$columns - 1>.  
+
+It is possible to control which characters terminate words by
+modifying C<$Text::Wrap::break>. Set this to a string such as
+C<'[\s:]'> (to break before spaces or colons) or a pre-compiled regexp
+such as C<qr/[\s']/> (to break before spaces or apostrophes). The
+default is simply C<'\s'>; that is, words are terminated by spaces.
+(This means, among other things, that trailing punctuation  such as
+full stops or commas stay with the word they are "attached" to.)
+
+Beginner note: In example 2, above C<$columns> is imported into
+the local namespace, and set locally.  In example 3,
+C<$Text::Wrap::columns> is set in its own namespace without importing it.
+
+C<Text::Wrap::wrap()> starts its work by expanding all the tabs in its
+input into spaces.  The last thing it does it to turn spaces back
+into tabs.  If you do not want tabs in your results, set 
+C<$Text::Wrap::unexapand> to a false value.  Likewise if you do not
+want to use 8-character tabstops, set C<$Text::Wrap::tabstop> to
+the number of characters you do want for your tabstops.
+
+If you want to separate your lines with something other than C<\n>
+then set C<$Text::Wrap::seporator> to your preference.
+
+When words that are longer than C<$columns> are encountered, they
+are broken up.  C<wrap()> adds a C<"\n"> at column C<$columns>.
+This behavior can be overridden by setting C<$huge> to
+'die' or to 'overflow'.  When set to 'die', large words will cause
+C<die()> to be called.  When set to 'overflow', large words will be
+left intact.  
+
+Historical notes: 'die' used to be the default value of
+C<$huge>.  Now, 'wrap' is the default value.
+
+=head1 EXAMPLE
+
+	print wrap("\t","","This is a bit of text that forms 
+		a normal book-style paragraph");
+
+=head1 AUTHOR
+
+David Muir Sharnoff <muir@idiom.com> with help from Tim Pierce and
+many many others.  
+
--- a/testsuite/tests-to-run/parallel-freebsd.sh
+++ b/testsuite/tests-to-run/parallel-freebsd.sh
@ -5,7 +5,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later

 echo "### These tests requires VirtualBox running with the following images"
-SERVER1=freebsd11
+SERVER1=freebsd12
 SSHUSER1=vagrant
 SSHLOGIN1=$SSHUSER1@$SERVER1
 echo $SSHUSER1@$SERVER1
--- a/testsuite/tests-to-run/parallel-manual.sh
+++ b/testsuite/tests-to-run/parallel-manual.sh
@ -6,6 +6,18 @@

 # These fail regularly

+par_ll_asian() {
+    echo '### --ll with Asian wide chars mess up display'
+    echo 'bug #63878: Wide East Asian chars in --latest-line'
+    p="parallel --ll --color --tag"
+    COLUMNS=80 $p echo tag fits, line fits a{}b{}c \
+	   ::: ヌー平行
+    COLUMNS=80 $p echo tag fits, line too long a{}b{}c \
+	   ::: ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー
+    COLUMNS=80 $p echo tag too long a{}b{}c \
+	   ::: ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行ヌー平行a
+}
+
 par_ll_tag() {
 	parallel --tag --ll -q printf "a\n{}\n" ::: should-be-tagged-A
 	parallel --tag --ll -q printf "a\n\r{}\n" ::: should-be-tagged-B
--- a/testsuite/tests-to-run/parallel-remote1.sh
+++ b/testsuite/tests-to-run/parallel-remote1.sh
@ -5,7 +5,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later

 SERVER1=parallel-server1
-SERVER2=parallel-server2
+SERVER2=parallel-server4
 SERVER3=parallel-server3
 SSHUSER1=vagrant
 SSHUSER2=vagrant
--- a/testsuite/tests-to-run/parallel-tutorial.sh
+++ b/testsuite/tests-to-run/parallel-tutorial.sh
@ -92,8 +92,10 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' "$testsuit
 	      s/^[A-Z][A-Z0-9_]*\s$//;
 	      # Fails often due to race
 	      s/cat: input_file: No such file or directory\n//;
-	      s{rsync: link_stat ".*/home/parallel/input_file.out" .*\n}{};
+	      s{rsync: .* link_stat ".*/home/parallel/input_file.out" .*\n}{};
 	      s{rsync error: some files/attrs were not transferred .*\n}{};
+	      s{Give up after 2 secs\n}{};
+	      s{parallel: Warning: Semaphore timed out. Exiting.\n}{};
 	      s{.* GtkDialog .*\n}{};
 	      s{tried 1}{};
 	      s/^\s*\n//;
--- a/testsuite/tests-to-run/test17.sh
+++ b/testsuite/tests-to-run/test17.sh
@ -5,7 +5,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later

 SERVER1=parallel-server1
-SERVER2=parallel-server2
+SERVER2=parallel-server3
 SSHUSER1=vagrant
 SSHUSER2=vagrant
 SSHLOGIN1=$SSHUSER1@$SERVER1
--- a/testsuite/tests-to-run/test18.sh
+++ b/testsuite/tests-to-run/test18.sh
@ -5,7 +5,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later

 SERVER1=parallel-server1
-SERVER2=parallel-server2
+SERVER2=parallel-server3
 SSHUSER1=vagrant
 SSHUSER2=vagrant
 SSHLOGIN1=$SSHUSER1@$SERVER1
--- a/testsuite/tests-to-run/test19.sh
+++ b/testsuite/tests-to-run/test19.sh
@ -7,7 +7,7 @@
 # TODO return multiple

 SERVER1=parallel-server1
-SERVER2=parallel-server2
+SERVER2=parallel-server3
 SSHUSER1=vagrant
 SSHUSER2=vagrant
 #SSHLOGIN1=parallel@$SERVER1
--- a/testsuite/tests-to-run/test23.sh
+++ b/testsuite/tests-to-run/test23.sh
@ -5,7 +5,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later

 SERVER1=parallel-server1
-SERVER2=parallel-server2
+SERVER2=parallel-server3
 SSHUSER1=vagrant
 SSHUSER2=vagrant
 #SSHLOGIN1=parallel@$SERVER1
--- a/testsuite/tests-to-run/test37.sh
+++ b/testsuite/tests-to-run/test37.sh
@ -5,7 +5,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later

 SERVER1=parallel-server1
-SERVER2=parallel-server2
+SERVER2=parallel-server3
 SSHUSER1=vagrant
 SSHUSER2=vagrant
 SSHLOGIN1=$SSHUSER1@$SERVER1
--- a/testsuite/tests-to-run/test60.sh
+++ b/testsuite/tests-to-run/test60.sh
@ -5,7 +5,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later

 SERVER1=parallel-server1
-SERVER2=parallel-server2
+SERVER2=parallel-server3
 SSHUSER1=vagrant
 SSHUSER2=vagrant
 export SSHLOGIN1=$SSHUSER1@$SERVER1
--- a/testsuite/wanted-results/parallel-freebsd
+++ b/testsuite/wanted-results/parallel-freebsd
@ -1,5 +1,5 @@
 ### These tests requires VirtualBox running with the following images
-vagrant@freebsd11
+vagrant@freebsd12
 par_compress_pipe	par_compress_pipe 2>&1
 par_compress_pipe	Test --compress --pipe
 par_compress_pipe	    1000    1000    3893
--- a/testsuite/wanted-results/parallel-local-0.3s
+++ b/testsuite/wanted-results/parallel-local-0.3s
@ -336,9 +336,9 @@ par_link_files_as_only_arg	bug #50685: single ::::+ does not work
 par_link_files_as_only_arg	1 1 1
 par_link_files_as_only_arg	2 2 2
 par_link_files_as_only_arg	3 3 3
-par_ll_long_followed_by_short	
par_ll_long_followed_by_short	[KA very long line 
-par_ll_long_followed_by_short	[A
par_ll_long_followed_by_short	[KA very long line 
-par_ll_long_followed_by_short	[A
par_ll_long_followed_by_short	[KOK 
+par_ll_long_followed_by_short	
par_ll_long_followed_by_short	[KA very long line        
+par_ll_long_followed_by_short	[A
par_ll_long_followed_by_short	[KA very long line        
+par_ll_long_followed_by_short	[A
par_ll_long_followed_by_short	[KOK      
 par_locale_quoting	### quoting in different locales
 par_locale_quoting	£`/tmp/test£`
 par_locale_quoting	$LC_ALL £`/tmp/test£`
--- a/testsuite/wanted-results/parallel-local-ssh4
+++ b/testsuite/wanted-results/parallel-local-ssh4
@ -153,22 +153,22 @@ par_--tmux_different_shells	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_different_shells	0
 par_--tmux_different_shells	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_different_shells	4
-par_--tmux_different_shells	See output with: tmux -S /tmp/tmsXXXXX attach
+par_--tmux_different_shells	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_different_shells	0
-par_--tmux_different_shells	See output with: tmux -S /tmp/tmsXXXXX attach
+par_--tmux_different_shells	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_different_shells	4
-par_--tmux_different_shells	See output with: tmux -S /tmp/tmsXXXXX attach
+par_--tmux_different_shells	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_different_shells	0
-par_--tmux_different_shells	See output with: tmux -S /tmp/tmsXXXXX attach
+par_--tmux_different_shells	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_different_shells	4
-par_--tmux_different_shells	See output with: tmux -S /tmp/tmsXXXXX attach
+par_--tmux_different_shells	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_different_shells	0
-par_--tmux_different_shells	See output with: tmux -S /tmp/tmsXXXXX attach
+par_--tmux_different_shells	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_different_shells	4
 par_--tmux_different_shells	# command is currently too long for csh. Maybe it can be fixed?
-par_--tmux_different_shells	See output with: tmux -S /tmp/tmsXXXXX attach
+par_--tmux_different_shells	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_different_shells	0
-par_--tmux_different_shells	See output with: tmux -S /tmp/tmsXXXXX attach
+par_--tmux_different_shells	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_different_shells	4
 par_--tmux_length	### works
 par_--tmux_length	See output with: tmux -S /TMP/tmsXXXXX attach
@ -179,6 +179,8 @@ par_--tmux_length	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_length	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_length	See output with: tmux -S /TMP/tmsXXXXX attach
 par_--tmux_length	See output with: tmux -S /TMP/tmsXXXXX attach
+par__test_different_rsync_versions	### different versions of rsync need fixups
+par__test_different_rsync_versions	### no output is good
 par_bigvar_csh	### csh
 par_bigvar_csh	3 big vars run remotely - length(base64) > 1000
 par_bigvar_csh	      1     200     692
--- a/testsuite/wanted-results/parallel-local-ssh5
+++ b/testsuite/wanted-results/parallel-local-ssh5
@ -67,7 +67,7 @@ par_--ssh_autossh	    AUTOSSH_DEBUG       - turn logging to maximum verbosity an
 par_--ssh_autossh	                          stderr
 par_--ssh_autossh	
 par_--ssh_autossh	rsync: connection unexpectedly closed (0 bytes received so far) [Receiver]
-par_--ssh_autossh	rsync error: error in rsync protocol data stream (code 12) at io.c(228) [Receiver=3.2.3]
+par_--ssh_autossh	rsync error: error in rsync protocol data stream (code 12) at io.c(231) [Receiver=3.2.7]
 par_--ssh_autossh	foo_autossh
 par_--shellquote_command_len	### test quoting will not cause a crash if too long
 par_--shellquote_command_len	-Slo -j10	" 1 1	      1       1       4
--- a/testsuite/wanted-results/parallel-local10
+++ b/testsuite/wanted-results/parallel-local10
@ -2,3 +2,5 @@
 ### See if we get compile error
 perl
 ### See if we read modules outside perllib
+### Test make .deb package
+To install the GNU Parallel Debian package, run:
--- a/testsuite/wanted-results/parallel-local12
+++ b/testsuite/wanted-results/parallel-local12
@ -91,8 +91,6 @@ https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html
 https://www.gnu.org/software/parallel/parallel_design.html#citation-notice
 https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt

-mentioned in the release notes of next version of GNU Parallel.
-

 Type: 'will cite' and press enter.
 > 
--- a/testsuite/wanted-results/parallel-local150
+++ b/testsuite/wanted-results/parallel-local150
@ -141,9 +141,6 @@ echo "### bug #35268: shell_quote doesn't treats [] brackets correctly"
 ### bug #35268: shell_quote doesn't treats [] brackets correctly
  touch /tmp/foo1;   stdout parallel echo ::: '/tmp/foo[123]';   rm /tmp/foo1
 /tmp/foo[123]
-echo '### Test make .deb package';   cd ~/privat/parallel/packager/debian;   stdout make | grep 'To install the GNU Parallel Debian package, run:'
-### Test make .deb package
-To install the GNU Parallel Debian package, run:
 echo '### Test basic --arg-sep'
 ### Test basic --arg-sep
  parallel -k echo ::: a b
--- a/testsuite/wanted-results/parallel-macos
+++ b/testsuite/wanted-results/parallel-macos
@ -1,7 +1,7 @@
-par_big_func	       1    3XXX   90XXX
-par_big_func	       1    3XXX   90XXX
-par_big_func	       1    3XXX   90XXX
-par_big_func	       1    1XXX   47XXX
+par_big_func	       1    3XXX   91XXX
+par_big_func	       1    3XXX   91XXX
+par_big_func	       1    3XXX   91XXX
+par_big_func	       1    1XXX   46XXX
 par_big_func_name	   18XXX
 par_big_func_name	   18XXX
 par_big_func_name	   18XXX
@ -65,8 +65,7 @@ par_many_func	       1     196    4XXX
 par_many_func	       1     196    4XXX
 par_many_func	       1     196    4XXX
 par_many_func	       1     196    4XXX
-par_many_func	       1     196    4XXX
-par_many_func	       1       4      68
+par_many_func	       1     200    4XXX
 par_many_var	   22XXX
 par_many_var	   22XXX
 par_many_var	   22XXX
--- a/testsuite/wanted-results/parallel-remote1
+++ b/testsuite/wanted-results/parallel-remote1
@ -6,8 +6,8 @@ par_filter_hosts_different_errors	### --filter-hosts - OK, non-such-user, connec
 par_filter_hosts_different_errors	aspire
 par_filter_hosts_no_ssh_nxserver	### test --filter-hosts with server w/o ssh, non-existing server
 par_filter_hosts_no_ssh_nxserver	vagrant@parallel-server1	
-par_filter_hosts_no_ssh_nxserver	vagrant@parallel-server2	
 par_filter_hosts_no_ssh_nxserver	vagrant@parallel-server3	
+par_filter_hosts_no_ssh_nxserver	vagrant@parallel-server4	
 par_special_ssh	### Test use special ssh
 par_special_ssh	TODO test ssh with > 9 simultaneous
 par_special_ssh	1
@ -119,7 +119,7 @@ par_timeout_retries	parallel: Warning: ssh 8.8.8.8 echo 8.8.8.8
 par_timeout_retries	parallel: Warning: This job was killed because it timed out:
 par_timeout_retries	parallel: Warning: ssh 8.8.8.8 echo 8.8.8.8
 par_timeout_retries	vagrant@parallel-server1
-par_timeout_retries	vagrant@parallel-server2
+par_timeout_retries	vagrant@parallel-server4
 par_timeout_retries	vagrant@parallel-server3
 par_workdir_in_HOME	### test --workdir . in $HOME
 par_workdir_in_HOME	OK
--- a/testsuite/wanted-results/parallel-tutorial
+++ b/testsuite/wanted-results/parallel-tutorial
@ -149,6 +149,8 @@ num128
 num30000
 num8
 outdir
+tmsFPAdA
+tmsNGdem
 tsv-file.tsv
 foo
 /TMP
@ -387,6 +389,16 @@ pre-A-post
 /usr/bin/bash: -c: line 1: syntax error near unexpected token `)'
 /usr/bin/bash: -c: line 1: `set a="outdir"; if( { test -d "$a" } ) echo "$a is a dir"'
 /usr/bin/bash: -c: line 1: syntax error near unexpected token `)'
+/usr/bin/bash: -c: line 1: `set a="tempfile"; if( { test -d "$a" } ) echo "$a is a dir"'
+/usr/bin/bash: -c: line 1: syntax error near unexpected token `)'
+/usr/bin/bash: -c: line 1: `set a="tempfile"; if( { test -d "$a" } ) echo "$a is a dir"'
+/usr/bin/bash: -c: line 1: syntax error near unexpected token `)'
+/usr/bin/bash: -c: line 1: `set a="tempfile"; if( { test -d "$a" } ) echo "$a is a dir"'
+/usr/bin/bash: -c: line 1: syntax error near unexpected token `)'
+/usr/bin/bash: -c: line 1: `set a="tmsFPAdA"; if( { test -d "$a" } ) echo "$a is a dir"'
+/usr/bin/bash: -c: line 1: syntax error near unexpected token `)'
+/usr/bin/bash: -c: line 1: `set a="tmsNGdem"; if( { test -d "$a" } ) echo "$a is a dir"'
+/usr/bin/bash: -c: line 1: syntax error near unexpected token `)'
 /usr/bin/bash: -c: line 1: `set a="tsv-file.tsv"; if( { test -d "$a" } ) echo "$a is a dir"'
  parallel --tag echo foo-{} ::: A B C
 A	foo-A
@ -871,7 +883,7 @@ For details: see  man env_parallel
  export -f my_func3
  parallel -vv --workdir ... --nice 17 --env _ --trc {}.out \
    -S $SERVER1 my_func3 {} ::: abc-file
-ssh -l parallel lo -- exec mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -e'ssh -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -l parallel lo -- exec perl -X -e GNU_Parallel_worker,eval+pack+q/H10000000/,join+q//,@ARGV BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 -rlDzR -e'ssh -l parallel' --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -- lo:./abc-file.out ./.;ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file.out 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec rm -rf .TMPWORKDIR;exit $_EXIT_status;
+ssh -l parallel lo -- exec mkdir -p ./.TMPWORKDIR && rsync --protocol 30 --old-args -rlDzR -e'ssh -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -l parallel lo -- exec perl -X -e GNU_Parallel_worker,eval+pack+q/H10000000/,join+q//,@ARGV BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 --old-args -rlDzR -e'ssh -l parallel' --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -- lo:./abc-file.out ./.;ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec 'sh -c '"'"'rm -f ./.TMPWORKDIR/abc-file.out 2>/dev/null;rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- exec rm -rf .TMPWORKDIR;exit $_EXIT_status;
  parset myvar1,myvar2 echo ::: a b
  echo $myvar1
  echo $myvar2
@ -1260,7 +1272,6 @@ Slow started
 Forced running after 1 sec
 Slow ended
 parallel: Warning: Semaphore timed out. Stealing the semaphore.
-parallel: Warning: Semaphore timed out. Exiting.
  parallel --help
 Usage:
 parallel [options] [command [arguments]] < list_of_arguments
@ -1325,8 +1336,6 @@ More about funding GNU Parallel and the citation notice:
 https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html
 https://www.gnu.org/software/tempfileallel_design.html#citation-notice
 https:BASE64-notice-faq.txt
-If you send a copy of your published article to tange@gnu.org, it will be
-mentioned in the release notes of next version of GNU Parallel.
  parallel --number-of-cpus
  parallel --number-of-cores
 9
--- a/testsuite/wanted-results/test37
+++ b/testsuite/wanted-results/test37
@ -73,6 +73,6 @@ perl -pe "\$a=1; print \$a" <(echo foo)
 1foo
 ### Test merging of profiles - sort needed because -k only works on the single machine
 vagrant@parallel-server1	a
-vagrant@parallel-server2	a
+vagrant@parallel-server3	a
 ### Test merging of profiles - sort needed because -k only works on the single machine --plain
 a
--- a/testsuite/wanted-results/test60
+++ b/testsuite/wanted-results/test60
@ -1,6 +1,6 @@
 par_nonall	### Test --nonall
 par_nonall	centos8.localdomain
-par_nonall	freebsd11.localdomain
+par_nonall	freebsd12.localdomain
 par_nonall_basefile	### Test --nonall --basefile
 par_nonall_basefile	/tmp/nonall--basefile
 par_nonall_basefile	/tmp/nonall--basefile
@ -10,8 +10,8 @@ par_nonall_sshloginfile_stdin	centos8.localdomain
 par_nonall_u	### Test --nonall -u - should be interleaved x y x y
 par_nonall_u	      1 centos8.localdomain
 par_nonall_u	      1 centos8.localdomain
-par_nonall_u	      1 freebsd11.localdomain
-par_nonall_u	      1 freebsd11.localdomain
+par_nonall_u	      1 freebsd12.localdomain
+par_nonall_u	      1 freebsd12.localdomain
 par_onall	### Test --onall
 par_onall	1
 par_onall	2