diff --git a/Makefile.in b/Makefile.in index 4f2a0486..b6643e67 100644 --- a/Makefile.in +++ b/Makefile.in @@ -207,6 +207,7 @@ INSTALL_SCRIPT = @INSTALL_SCRIPT@ INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ LIBOBJS = @LIBOBJS@ LIBS = @LIBS@ +LN_S = @LN_S@ LTLIBOBJS = @LTLIBOBJS@ MAKEINFO = @MAKEINFO@ MKDIR_P = @MKDIR_P@ diff --git a/configure b/configure index 8a7d70d1..606bc5ec 100755 --- a/configure +++ b/configure @@ -586,6 +586,7 @@ PACKAGE_URL='' ac_subst_vars='LTLIBOBJS LIBOBJS +LN_S DOCUMENTATION_FALSE DOCUMENTATION_TRUE AM_BACKSLASH @@ -2336,6 +2337,16 @@ else DOCUMENTATION_FALSE= fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ln -s works" >&5 +$as_echo_n "checking whether ln -s works... " >&6; } +LN_S=$as_ln_s +if test "$LN_S" = "ln -s"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no, using $LN_S" >&5 +$as_echo "no, using $LN_S" >&6; } +fi cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure diff --git a/configure.ac b/configure.ac index dd105f6d..4cec91aa 100644 --- a/configure.ac +++ b/configure.ac @@ -11,5 +11,5 @@ AC_ARG_ENABLE(documentation, [Omit building and installing the documentation. (default=no)]),, [enable_documentation=yes]) AM_CONDITIONAL([DOCUMENTATION], [test x$enable_documentation = xyes]) - +AC_PROG_LN_S AC_OUTPUT diff --git a/doc/release_new_version b/doc/release_new_version index fd244be7..9866bb88 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -127,7 +127,7 @@ pushd /tmp rm -rf parallel-${YYYYMMDD}* # This can take 7 minutes while ! wget http://ftp.gnu.org/gnu/parallel/parallel-$YYYYMMDD.tar.bz2 ; do sleep 2; done -#wget http://alpha.gnu.org/gnu/parallel/parallel-$YYYYMMDD.tar.bz2 +#while ! wget http://alpha.gnu.org/gnu/parallel/parallel-$YYYYMMDD.tar.bz2 ; do sleep 2; done tar xjvf parallel-$YYYYMMDD.tar.bz2 cd parallel-$YYYYMMDD ./configure @@ -217,9 +217,9 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20140822 ('Argentina/Gaza/Williams') released +Subject: GNU Parallel 20140822 ('Williams') released -GNU Parallel 20140822 ('Argentina/Gaza') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20140822 ('Williams') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ Haiku of the month: @@ -232,14 +232,18 @@ New in this release: * GNU Parallel now uses the same shell it was started from as the command shell for local jobs. So if GNU Parallel is started from tcsh it will use tcsh as its shell even if the login $SHELL is different. For remote jobs the login $SHELL will be used. -* The whole current environment in bash can be copied by using a shell wrapper function (env_parallel). +* The whole current environment in bash can be copied by using a shell wrapper function (Search manual for env_parallel). * GNU Parallel was cited in: A Web Service for Scholarly Big Data Information Extraction http://patshih.ist.psu.edu/publications/Williams-CiteSeerExtractor-ICWS14.pdf * --plus adds the replacement strings {+/} {+.} {+..} {+...} {..} {...} {/..} {/...}. The idea being that '+foo' matches the opposite of 'foo' and {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...} +* Comparison of the speed of different GNU Parallel versions: http://lists.gnu.org/archive/html/parallel/2014-08/msg00030.html + * GNU Parallel was covered in the webcast 2014-08-20: Data Science at the Command Line http://www.oreilly.com/pub/e/3115 +* Distributed processing with GNU parallel http://kazjote.eu/2014/08/11/distributed-processing-with-gnu-parallel + * A Peek into GNU Parallel http://blog.dataweave.in/post/94238943763/a-peek-into-gnu-parallel * Сборка GNU parallel для CentOS/RHEL http://www.stableit.ru/2014/07/gnu-parallel-centosrhel.html diff --git a/src/Makefile.am b/src/Makefile.am index 053a8de9..52dcc7f2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,4 +1,8 @@ -bin_SCRIPTS = parallel sem sql niceload +bin_SCRIPTS = parallel sql niceload + +install-exec-hook: + rm $(DESTDIR)$(bindir)/sem || true + $(LN_S) $(DESTDIR)$(bindir)/parallel $(DESTDIR)$(bindir)/sem if DOCUMENTATION man_MANS = parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.1 diff --git a/src/Makefile.in b/src/Makefile.in index c83b0da5..dc45a1e7 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -161,6 +161,7 @@ INSTALL_SCRIPT = @INSTALL_SCRIPT@ INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ LIBOBJS = @LIBOBJS@ LIBS = @LIBS@ +LN_S = @LN_S@ LTLIBOBJS = @LTLIBOBJS@ MAKEINFO = @MAKEINFO@ MKDIR_P = @MKDIR_P@ @@ -215,7 +216,7 @@ target_alias = @target_alias@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ -bin_SCRIPTS = parallel sem sql niceload +bin_SCRIPTS = parallel sql niceload @DOCUMENTATION_TRUE@man_MANS = parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.1 @DOCUMENTATION_TRUE@doc_DATA = parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf DISTCLEANFILES = parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.1 \ @@ -464,7 +465,8 @@ install-dvi: install-dvi-am install-dvi-am: install-exec-am: install-binSCRIPTS - + @$(NORMAL_INSTALL) + $(MAKE) $(AM_MAKEFLAGS) install-exec-hook install-html: install-html-am install-html-am: @@ -505,15 +507,15 @@ uninstall-am: uninstall-binSCRIPTS uninstall-docDATA uninstall-man uninstall-man: uninstall-man1 -.MAKE: install-am install-strip +.MAKE: install-am install-exec-am install-strip .PHONY: all all-am check check-am clean clean-generic cscopelist-am \ ctags-am distclean distclean-generic distdir dvi dvi-am html \ html-am info info-am install install-am install-binSCRIPTS \ install-data install-data-am install-docDATA install-dvi \ - install-dvi-am install-exec install-exec-am install-html \ - install-html-am install-info install-info-am install-man \ - install-man1 install-pdf install-pdf-am install-ps \ + install-dvi-am install-exec install-exec-am install-exec-hook \ + install-html install-html-am install-info install-info-am \ + install-man install-man1 install-pdf install-pdf-am install-ps \ install-ps-am install-strip installcheck installcheck-am \ installdirs maintainer-clean maintainer-clean-generic \ mostlyclean mostlyclean-generic pdf pdf-am ps ps-am tags-am \ @@ -521,6 +523,10 @@ uninstall-man: uninstall-man1 uninstall-man uninstall-man1 +install-exec-hook: + rm $(DESTDIR)$(bindir)/sem || true + $(LN_S) $(DESTDIR)$(bindir)/parallel $(DESTDIR)$(bindir)/sem + # Build documentation file if the tool to build exists. # Otherwise: Use the distributed version parallel.1: parallel.pod diff --git a/src/parallel b/src/parallel index db7ab951..0c17e284 100755 --- a/src/parallel +++ b/src/parallel @@ -65,6 +65,7 @@ if($opt::pipepart) { @fhlist = (*STDIN); } } + if($opt::skip_first_line) { # Skip the first line for the first file handle my $fh = $fhlist[0]; @@ -1605,6 +1606,10 @@ sub init_run_jobs { if(@opt::basefile) { setup_basefile(); } } +{ + my $last_time; + my %last_mtime; + sub start_more_jobs { # Run start_another_job() but only if: # * not $Global::start_no_new_jobs set @@ -1619,14 +1624,43 @@ sub start_more_jobs { if($Global::start_no_new_jobs) { return $jobs_started; } - if($Global::max_procs_file) { - # --jobs filename - my $mtime = (stat($Global::max_procs_file))[9]; - if($mtime > $Global::max_procs_file_last_mod) { - # file changed: Force re-computing max_jobs_running - $Global::max_procs_file_last_mod = $mtime; - for my $sshlogin (values %Global::host) { - $sshlogin->set_max_jobs_running(undef); + if(time - ($last_time||0) > 1) { + # At most do this every second + $last_time = time; + if($Global::max_procs_file) { + # --jobs filename + my $mtime = (stat($Global::max_procs_file))[9]; + if($mtime > $Global::max_procs_file_last_mod) { + # file changed: Force re-computing max_jobs_running + $Global::max_procs_file_last_mod = $mtime; + for my $sshlogin (values %Global::host) { + $sshlogin->set_max_jobs_running(undef); + } + } + } + if(@opt::sshloginfile) { + # Is --sshloginfile changed? + for my $slf (@opt::sshloginfile) { + my $actual_file = expand_slf_shorthand($slf); + my $mtime = (stat($actual_file))[9]; + $last_mtime{$actual_file} ||= $mtime; + if($mtime - $last_mtime{$actual_file} > 1) { + ::debug("run","--sshloginfile $actual_file changed. reload\n"); + $last_mtime{$actual_file} = $mtime; + # TODO reload $slf + my %sshlogin_before = %Global::host; + read_sshloginfile($actual_file); + parse_sshlogin(); + $opt::filter_hosts and filter_hosts(); + setup_basefile(); + my @added = grep { not $sshlogin_before{$_} } keys %Global::host; +# my @removed = grep { not $sshlogin_before{$_} } keys sshlogin_before %Global::host; +#wrong my @back = grep { $Global::host{$_}->jobslots() == 0 } + keys %Global::host; +# map { $Global::host{$_}->reset_jobslots() } @back; + + + } } } } @@ -1679,6 +1713,7 @@ sub start_more_jobs { return $jobs_started; } +} sub start_another_job { # If there are enough filehandles @@ -1813,6 +1848,9 @@ sub drain_job_queue { # * because of too little time between each ssh login. start_more_jobs(); $sleep = ::reap_usleep($sleep); + if($Global::max_jobs_running == 0) { + ::warning("There are no job slots available. Increase --jobs.\n"); + } } } while ($Global::total_running > 0 or @@ -2064,8 +2102,9 @@ sub get_job_with_sshlogin { $job->failed_here()) { # This command with these args failed for this sshlogin my ($no_of_failed_sshlogins,$min_failures) = $job->min_failed(); - if($no_of_failed_sshlogins == keys %Global::host and - $job->failed_here() == $min_failures) { + # Only look at the Global::host that have > 0 jobslots + if($no_of_failed_sshlogins == grep { $_->max_jobs_running() > 0 } values %Global::host + and $job->failed_here() == $min_failures) { # It failed the same or more times on another host: # run it on this host } else { @@ -2090,32 +2129,44 @@ sub __REMOTE_SSH__ {} sub read_sshloginfiles { # Returns: N/A for my $s (@_) { - read_sshloginfile($s); + read_sshloginfile(expand_slf_shorthand($s)); } } +sub expand_slf_shorthand { + my $file = shift; + if($file eq "-") { + # skip: It is stdin + } elsif($file eq "..") { + $file = $ENV{'HOME'}."/.parallel/sshloginfile"; + } elsif($file eq ".") { + $file = "/etc/parallel/sshloginfile"; + } elsif(not -r $file) { + if(not -r $ENV{'HOME'}."/.parallel/".$file) { + # Try prepending ~/.parallel + ::error("Cannot open $file.\n"); + ::wait_and_exit(255); + } else { + $file = $ENV{'HOME'}."/.parallel/".$file; + } + } + return $file; +} + sub read_sshloginfile { # Returns: N/A my $file = shift; my $close = 1; my $in_fh; - if($file eq "..") { - $file = $ENV{'HOME'}."/.parallel/sshloginfile"; - } - if($file eq ".") { - $file = "/etc/parallel/sshloginfile"; - } + ::debug("init","--slf ",$file); if($file eq "-") { $in_fh = *STDIN; $close = 0; } else { if(not open($in_fh, "<", $file)) { # Try the filename - if(not open($in_fh, "<", $ENV{'HOME'}."/.parallel/".$file)) { - # Try prepending ~/.parallel - ::error("Cannot open $file.\n"); - ::wait_and_exit(255); - } + ::error("Cannot open $file.\n"); + ::wait_and_exit(255); } } while(<$in_fh>) { @@ -2137,7 +2188,7 @@ sub parse_sshlogin { # Split up -S sshlogin,sshlogin for my $s (split /,/, $sshlogin) { if ($s eq ".." or $s eq "-") { - read_sshloginfile($s); + read_sshloginfile(expand_slf_shorthand($s)); } else { push (@login, $s); } @@ -2145,19 +2196,21 @@ sub parse_sshlogin { } $Global::minimal_command_line_length = 8_000_000; for my $sshlogin_string (@login) { + if($Global::host{$sshlogin_string}) { + # This sshlogin has already been added: next + next; + } my $sshlogin = SSHLogin->new($sshlogin_string); if($sshlogin_string eq ":") { $sshlogin->set_maxlength(Limits::Command::max_length()); } else { # If all chars needs to be quoted, every other character will be \ - $sshlogin->set_maxlength(Limits::Command::max_length()/2); + $sshlogin->set_maxlength(int(Limits::Command::max_length()/2)); } $Global::minimal_command_line_length = ::min($Global::minimal_command_line_length, $sshlogin->maxlength()); $Global::host{$sshlogin->string()} = $sshlogin; } - - # debug("start", "sshlogin: ", my_dump(%Global::host),"\n"); if($opt::transfer or @opt::return or $opt::cleanup or @opt::basefile) { if(not remote_hosts()) { @@ -3145,6 +3198,7 @@ sub new { 'jobs_completed' => 0, 'maxlength' => undef, 'max_jobs_running' => undef, + 'orig_max_jobs_running' => undef, 'ncpus' => $ncpus, 'sshcommand' => undef, 'serverlogin' => undef, @@ -3159,6 +3213,8 @@ sub new { 'swap_activity_file' => $ENV{'HOME'} . "/.parallel/tmp/swap_activity-" . $no_slash_string, 'swap_activity' => undef, + # What time was this object created? + 'added' => time, }, ref($class) || $class; } @@ -3190,11 +3246,6 @@ sub dec_jobs_running { $self->{'jobs_running'}--; } -#sub set_jobs_running { -# my $self = shift; -# $self->{'jobs_running'} = shift; -#} - sub set_maxlength { my $self = shift; $self->{'maxlength'} = shift; @@ -3225,6 +3276,8 @@ sub set_max_jobs_running { # max_jobs_running could be resat if -j is a changed file $Global::max_jobs_running += $self->{'max_jobs_running'}; } + # Initialize orig to the first value that comes around + $self->{'orig_max_jobs_running'} ||= $self->{'max_jobs_running'}; } sub swapping { @@ -3754,7 +3807,8 @@ sub processes_available_by_system_limit { } if(not $more_filehandles) { ::warning("Only enough file handles to run ", $system_limit, " jobs in parallel.\n", - "Raising ulimit -n or /etc/security/limits.conf may help.\n"); + "Running 'parallel -j0 -N", $system_limit, " --pipe parallel -j0' or ", + "raising ulimit -n or /etc/security/limits.conf may help.\n"); } if($max_system_proc_reached) { ::warning("Only enough available processes to run ", $system_limit, @@ -4255,7 +4309,7 @@ sub no_of_cores_hpux { # Number of CPU cores on HP-UX # undef if not HP-UX my $no_of_cores = - (`/usr/bin/mpsched -s 2>&1 | grep 'Processor Count' | awk '{ print \$4 }'`); + (`/usr/bin/mpsched -s 2>&1 | grep 'Processor Count' | awk '{ print \$3 }'`); return $no_of_cores; } @@ -5093,8 +5147,7 @@ sub min_failed { # the minimal number of times this command has failed my $self = shift; my $min_failures = - ::min(map { $self->{'failed'}{$_} } - keys %{$self->{'failed'}}); + ::min(map { $self->{'failed'}{$_} } keys %{$self->{'failed'}}); my $number_of_sshlogins_failed_on = scalar keys %{$self->{'failed'}}; return ($number_of_sshlogins_failed_on,$min_failures); } @@ -5466,9 +5519,6 @@ sub start { # Exit value: # empty input = true # some input = exit val from command - # Bug: - # If the command does not read the first char, the temp file - # is not deleted. my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".chr"); $command = qq{ sh -c 'dd bs=1 count=1 of=$tmpfile 2>/dev/null'; @@ -5491,10 +5541,10 @@ sub start { # Give STDIN to the first job if using -a (but only if running # locally - otherwise CTRL-C does not work for other jobs Bug#36585) *IN = *STDIN; - # The eval is needed to catch exception from open3 if($opt::tmux) { $command = $job->tmux_wrap($command); } + # The eval is needed to catch exception from open3 eval { $pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) || ::die_bug("open3-a"); @@ -5507,10 +5557,10 @@ sub start { open(my $devtty_fh, "<", "/dev/tty")) { # Give /dev/tty to the command if no one else is using it *IN = $devtty_fh; - # The eval is needed to catch exception from open3 if($opt::tmux) { $command = $job->tmux_wrap($command); } + # The eval is needed to catch exception from open3 eval { $pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) || ::die_bug("open3-/dev/tty"); @@ -5522,6 +5572,7 @@ sub start { if($opt::tmux) { $command = $job->tmux_wrap($command); } + # The eval is needed to catch exception from open3 eval { $pid = ::open3(::gensym, ">&OUT", ">&ERR", $Global::shell, "-c", $command) || ::die_bug("open3-gensym"); @@ -5964,8 +6015,7 @@ sub populate { # Returns: N/A my $self = shift; my $next_arg; -# my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length(); - my $max_len = Limits::Command::max_length(); + my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length(); if($opt::cat or $opt::fifo) { # Get a tempfile name @@ -7123,7 +7173,6 @@ sub replace { $perlexpr =~ s/^-?\d+ //; # Positional replace treated as normal replace if(not defined $self->{"rpl",0,$perlexpr}) { local $_; - # TODO disable warnings if($Global::trim eq "n") { $_ = $self->{'orig'}; } else { @@ -7321,18 +7370,18 @@ sub acquire { my $start_time = time; while(1) { $self->atomic_link_if_count_less_than() and last; - ::debug("run", "Remove dead locks"); + ::debug("sem", "Remove dead locks"); my $lockdir = $self->{'lockdir'}; for my $d (glob "$lockdir/*") { - ::debug("run", "Lock $d $lockdir\n"); + ::debug("sem", "Lock $d $lockdir\n"); $d =~ m:$lockdir/([0-9]+)\@([-\._a-z0-9]+)$:o or next; my ($pid, $host) = ($1, $2); if($host eq ::hostname()) { if(not kill 0, $1) { - ::debug("run", "Dead: $d"); + ::debug("sem", "Dead: $d"); unlink $d; } else { - ::debug("run", "Alive: $d"); + ::debug("sem", "Alive: $d"); } } } @@ -7354,7 +7403,7 @@ sub acquire { last; } } - ::debug("run", "acquired $self->{'pid'}\n"); + ::debug("sem", "acquired $self->{'pid'}\n"); } sub release { @@ -7372,6 +7421,27 @@ sub release { ::debug("run", "released $self->{'pid'}\n"); } +sub _release { + my $self = shift; + + unlink $self->{'pidfile'}; + $self->lock(); + my $nlinks = $self->nlinks(); + ::debug("sem", $nlinks, "<", $self->{'count'}); + if($nlinks-- > 1) { + unlink $self->{'idfile'}; + open (my $fh, ">", $self->{'idfile'}) or + ::die_bug("write_idfile: $self->{'idfile'}"); + print $fh "#"x$nlinks; + close $fh; + } else { + unlink $self->{'idfile'}; + rmdir $self->{'lockdir'}; + } + $self->unlock(); + ::debug("sem", "released $self->{'pid'}\n"); +} + sub atomic_link_if_count_less_than { # Link $file1 to $file2 if nlinks to $file1 < $count my $self = shift; @@ -7392,10 +7462,35 @@ sub atomic_link_if_count_less_than { return $retval; } +sub _atomic_link_if_count_less_than { + # Link $file1 to $file2 if nlinks to $file1 < $count + my $self = shift; + my $retval = 0; + $self->lock(); + my $nlinks = $self->nlinks(); + ::debug("sem", $nlinks, "<", $self->{'count'}); + if($nlinks++ < $self->{'count'}) { + -d $self->{'lockdir'} or mkdir_or_die($self->{'lockdir'}); + if(not -e $self->{'idfile'}) { + open (my $fh, ">", $self->{'idfile'}) or + ::die_bug("write_idfile: $self->{'idfile'}"); + close $fh; + } + open (my $fh, ">", $self->{'idfile'}) or + ::die_bug("write_idfile: $self->{'idfile'}"); + print $fh "#"x$nlinks; + close $fh; + $retval = link $self->{'idfile'}, $self->{'pidfile'}; + } + $self->unlock(); + ::debug("sem", "atomic $retval"); + return $retval; +} + sub nlinks { my $self = shift; if(-e $self->{'idfile'}) { - ::debug("run", "nlinks", (stat(_))[3], "\n"); + ::debug("sem", "nlinks", (stat(_))[3], "size", (stat(_))[7], "\n"); return (stat(_))[3]; } else { return 0; @@ -7481,4 +7576,3 @@ sub mkdir_or_die { # Keep perl -w happy $opt::x = $Semaphore::timeout = $Semaphore::wait = $Global::no_more_file_handles_warned = $Job::file_descriptor_warning_printed = $Global::max_slot_number = 0; - diff --git a/src/parallel_tutorial.1 b/src/parallel_tutorial.1 index 65a7c955..5eee0547 100644 --- a/src/parallel_tutorial.1 +++ b/src/parallel_tutorial.1 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "PARALLEL_TUTORIAL 1" -.TH PARALLEL_TUTORIAL 1 "2014-07-26" "20140722" "parallel" +.TH PARALLEL_TUTORIAL 1 "2014-07-26" "20140815" "parallel" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html index 206bed23..d894106d 100644 --- a/src/parallel_tutorial.html +++ b/src/parallel_tutorial.html @@ -2,7 +2,7 @@ - +GNU Parallel tutorial diff --git a/testsuite/tests-to-run/parallel-local2.sh b/testsuite/tests-to-run/parallel-local2.sh index fc684d80..a12a9236 100644 --- a/testsuite/tests-to-run/parallel-local2.sh +++ b/testsuite/tests-to-run/parallel-local2.sh @@ -1,20 +1,19 @@ #!/bin/bash -highload () -{ - # Force load > #cpus - CPUS=$(parallel --number-of-cores) - seq 0 0.1 $CPUS | parallel -j0 timeout 50 burnP6 2>/dev/null & +forceload () { + # Force load + LOAD=$1 + # Start 10 times as many burnP6 + seq 0 0.1 $1 | parallel -j0 timeout 20 burnP6 2>/dev/null & PID=$! - sleep 20 - perl -e 'do{$a=`uptime`} while($a=~/average: *(\S+)/ and $1 < '$CPUS')' + # Give GNU Parallel 1 second to startup + sleep 1 + perl -e 'do{$a=`uptime`} while($a=~/average: *(\S+)/ and $1 < '$LOAD')' # Load is now > $CPUS - # Kill off burnP6 and the parent parallel -# kill $PID; sleep 0.1; kill $PID; killall burnP6; sleep 0.3; kill -9 $PID 2>/dev/null } -highload 2>/dev/null & -sleep 1 +# Force load avg > number of cpu cores +forceload $(parallel --number-of-cores) cat <<'EOF' | parallel -vj0 -k -L1 echo "bug #38441: CPU usage goes to 100% if load is higher than --load at first job" @@ -26,14 +25,17 @@ echo "bug #38441: CPU usage goes to 100% if load is higher than --load at first echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834' seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo - -echo '### Test too slow spawning' -# Let the commands below run during high load -seq `parallel --number-of-cores` | parallel -j200% -N0 timeout -k 25 26 burnP6 & - sleep 1; - seq 1 1000 | stdout nice nice parallel -s 100 -uj0 true | - perl -pe '/parallel: Warning: Starting \d+ processes took/ and do {close STDIN; `killall -9 burnP6`; print "OK\n"; exit }' EOF # Make sure we got all the burnP6 killed killall -9 burnP6 2>/dev/null + +echo '### Test too slow spawning' +# Let the commands below run during high load +seq 1000 | timeout 20 parallel -j400% -N0 burnP6 2>/dev/null & +PID=$! +seq 1 1000 | stdout nice nice parallel --halt 1 -uj0 -N0 kill $PID | + perl -pe '/parallel: Warning: Starting \d+ processes took/ and do {close STDIN; `kill '$PID';killall -9 burnP6`; print "OK\n"; exit }'; + +# Make sure we got all the burnP6 killed +killall -9 burnP6 2>/dev/null diff --git a/testsuite/tests-to-run/parallel-local23.sh b/testsuite/tests-to-run/parallel-local23.sh index 782c1a07..5adc5e29 100755 --- a/testsuite/tests-to-run/parallel-local23.sh +++ b/testsuite/tests-to-run/parallel-local23.sh @@ -3,18 +3,21 @@ rm -rf tmp 2>/dev/null cp -a input-files/testdir2 tmp +NICEPAR="nice nice parallel" +export NICEPAR + cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k -L1 echo '### bug #42329: --line-buffer gives wrong output'; - parallel --line-buffer --tag seq ::: 10000000 | wc -c; - parallel --line-buffer seq ::: 10000000 | wc -c + $NICEPAR --line-buffer --tag seq ::: 10000000 | wc -c; + $NICEPAR --line-buffer seq ::: 10000000 | wc -c echo '### Test \0 as recend'; - printf "a\0b\0c\0" | parallel --recend '\0' -k -N1 --pipe cat -v \; echo; - printf "\0a\0b\0c" | parallel --recstart '\0' -k -N1 --pipe cat -v \; echo + printf "a\0b\0c\0" | $NICEPAR --recend '\0' -k -N1 --pipe cat -v \; echo; + printf "\0a\0b\0c" | $NICEPAR --recstart '\0' -k -N1 --pipe cat -v \; echo echo '### Test filenames containing UTF-8'; cd tmp; - find . -name '*.jpg' | nice nice parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}; + find . -name '*.jpg' | $NICEPAR -j +0 convert -geometry 120 {} {//}/thumb_{/}; find |grep -v CVS | sort; echo '### bug #39554: Feature request: line buffered output'; @@ -26,35 +29,35 @@ echo '### bug #39554: Feature request: line buffered output --tag'; echo echo '### test round-robin'; - nice seq 1000 | parallel --block 1k --pipe --round-robin wc | sort + nice seq 1000 | $NICEPAR --block 1k --pipe --round-robin wc | sort echo '### --version must have higher priority than retired options' - parallel --version -g -Y -U -W -T | tail + $NICEPAR --version -g -Y -U -W -T | tail echo '### bug #39787: --xargs broken' - nice perl -e 'for(1..30000){print "$_\n"}' | nice parallel --xargs -k echo | perl -ne 'print length $_,"\n"' + nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"' -echo '### --delay should grow by 2 sec per arg' -stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 | perl -ne '$_ >= 2 and $_ <= 5 and print "OK\n"' -stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 3 | perl -ne '$_ >= 4 and $_ <= 7 and print "OK\n"' +echo '### --delay should grow by 3 sec per arg' + stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"' + stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 3 | perl -ne '$_ >= 6 and $_ <= 11 and print "OK\n"' echo '### Exit value should not be affected if an earlier job times out' - parallel -j2 --timeout 1 --joblog - -k ::: "sleep 10" "exit 255" | field 7 + $NICEPAR -j2 --timeout 1 --joblog - -k ::: "sleep 10" "exit 255" | field 7 echo '### --header regexp' - (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --header '(%.*\n)*' echo JOB{#}\;cat + (echo %head1; echo %head2; seq 5) | $NICEPAR -kj2 --pipe -N2 --header '(%.*\n)*' echo JOB{#}\;cat echo '### --header num' - (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --header 2 echo JOB{#}\;cat + (echo %head1; echo %head2; seq 5) | $NICEPAR -kj2 --pipe -N2 --header 2 echo JOB{#}\;cat echo '### --header regexp --round-robin' - (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header '(%.*\n)*' echo JOB\;wc | sort + (echo %head1; echo %head2; seq 5) | $NICEPAR -kj2 --pipe -N2 --round --header '(%.*\n)*' echo JOB\;wc | sort echo '### --header num --round-robin' - (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header 2 echo JOB{#}\;wc | sort + (echo %head1; echo %head2; seq 5) | $NICEPAR -kj2 --pipe -N2 --round --header 2 echo JOB{#}\;wc | sort echo '### shebang-wrap' - nice nice parallel -k {} {} A B C ::: ./input-files/shebang/shebangwrap.*[^~] + $NICEPAR -k {} {} A B C ::: ./input-files/shebang/shebangwrap.*[^~] EOF diff --git a/testsuite/tests-to-run/parallel-local9.sh b/testsuite/tests-to-run/parallel-local9.sh index 9af68329..6eabdcf3 100644 --- a/testsuite/tests-to-run/parallel-local9.sh +++ b/testsuite/tests-to-run/parallel-local9.sh @@ -4,18 +4,20 @@ PAR="nice nice parallel -j2 --pipe --keeporder --block 150000 --tmpdir=/dev/shm" export PAR XAP="nice nice parallel --xapply" export XAP +NICEPAR="nice nice parallel" +export NICEPAR cat <<'EOF' | sed -e 's/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k -L1 echo 'bug #41613: --compress --line-buffer no newline'; - perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo + perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo echo 'bug #41613: --compress --line-buffer no --tagstring'; diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| - parallel -N10 -L1 --pipe -j6 --block 20M --compress + $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| - parallel -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer + $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' @@ -23,92 +25,92 @@ echo 'bug #41613: --compress --line-buffer no --tagstring'; echo 'bug #41613: --compress --line-buffer with --tagstring'; diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| - parallel -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} + $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| - parallel -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer + $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' echo 'bug #41613: --compress --line-buffer - no newline'; - echo 'pipe compress tagstring' - perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat; echo "K" - echo 'pipe compress notagstring' - perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat; echo "K" - echo 'pipe nocompress tagstring' - perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat; echo "K" - echo 'pipe nocompress notagstring' - perl -e 'print "O"'| parallel --pipe --line-buffer cat; echo "K" - echo 'nopipe compress tagstring' - parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" - echo 'nopipe compress notagstring' - parallel --compress --line-buffer echo {} O ::: -n; echo "K" - echo 'nopipe nocompress tagstring' - parallel --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" - echo 'nopipe nocompress notagstring' - parallel --line-buffer echo {} O ::: -n; echo "K" + echo 'pipe compress tagstring' + perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat; echo "K" + echo 'pipe compress notagstring' + perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat; echo "K" + echo 'pipe nocompress tagstring' + perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat; echo "K" + echo 'pipe nocompress notagstring' + perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat; echo "K" + echo 'nopipe compress tagstring' + $NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" + echo 'nopipe compress notagstring' + $NICEPAR --compress --line-buffer echo {} O ::: -n; echo "K" + echo 'nopipe nocompress tagstring' + $NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" + echo 'nopipe nocompress notagstring' + $NICEPAR --line-buffer echo {} O ::: -n; echo "K" echo 'bug #41412: --timeout + --delay causes deadlock'; seq 10 | parallel -j10 --timeout 1 --delay .3 echo; parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3; - parallel -j10 --timeout 2.2 --delay 3 "sleep {}; echo {}" ::: 1 2 4 5 6 + parallel -j10 --timeout 2.2 --delay 3 "sleep {}; echo {}" ::: 1 2 7 8 9 echo '### Test --spreadstdin - more procs than args'; rm -f /tmp/parallel.ss.*; - seq 1 5 | stdout parallel -j 10 --spreadstdin 'cat >/tmp/parallel.ss.$PARALLEL_SEQ' >/dev/null; + seq 1 5 | stdout $NICEPAR -j 10 --spreadstdin 'cat >/tmp/parallel.ss.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss.*; echo '### Test --spreadstdin - more args than procs'; rm -f /tmp/parallel.ss2.*; - seq 1 10 | stdout parallel -j 5 --spreadstdin 'cat >/tmp/parallel.ss2.$PARALLEL_SEQ' >/dev/null; + seq 1 10 | stdout $NICEPAR -j 5 --spreadstdin 'cat >/tmp/parallel.ss2.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss2.* -nice nice seq 1 1000 | nice nice parallel -j1 --spreadstdin cat "|cat "|wc -c -nice nice seq 1 10000 | nice nice parallel -j10 --spreadstdin cat "|cat "|wc -c -nice nice seq 1 100000 | nice nice parallel -j1 --spreadstdin cat "|cat "|wc -c -nice nice seq 1 1000000 | nice nice parallel -j10 --spreadstdin cat "|cat "|wc -c +nice nice seq 1 1000 | $NICEPAR -j1 --spreadstdin cat "|cat "|wc -c +nice nice seq 1 10000 | $NICEPAR -j10 --spreadstdin cat "|cat "|wc -c +nice nice seq 1 100000 | $NICEPAR -j1 --spreadstdin cat "|cat "|wc -c +nice nice seq 1 1000000 | $NICEPAR -j10 --spreadstdin cat "|cat "|wc -c -seq 1 10 | parallel --recend "\n" -j1 --spreadstdin gzip -9 >/tmp/foo.gz +seq 1 10 | $NICEPAR --recend "\n" -j1 --spreadstdin gzip -9 >/tmp/foo.gz echo '### Test --spreadstdin - similar to the failing below'; - nice seq 1 100000 | nice nice parallel --recend "\n" -j10 --spreadstdin gzip -9 >/tmp/foo2.gz; + nice seq 1 100000 | $NICEPAR --recend "\n" -j10 --spreadstdin gzip -9 >/tmp/foo2.gz; diff <(nice seq 1 100000) <(zcat /tmp/foo2.gz |sort -n); diff <(nice seq 1 100000|wc -c) <(zcat /tmp/foo2.gz |wc -c) echo '### Test --spreadstdin - this failed during devel'; nice seq 1 1000000 | md5sum; - nice seq 1 1000000 | nice nice parallel --recend "\n" -j10 --spreadstdin gzip -9 | zcat | sort -n | md5sum + nice seq 1 1000000 | $NICEPAR --recend "\n" -j10 --spreadstdin gzip -9 | zcat | sort -n | md5sum echo '### Test --spreadstdin -k'; - nice seq 1 1000000 | nice nice parallel -k --recend "\n" -j10 --spreadstdin gzip -9 | zcat | md5sum + nice seq 1 1000000 | $NICEPAR -k --recend "\n" -j10 --spreadstdin gzip -9 | zcat | md5sum echo '### Test --spreadstdin --files'; - nice seq 1 1000000 | shuf | parallel --files --recend "\n" -j10 --spreadstdin sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum + nice seq 1 1000000 | shuf | $NICEPAR --files --recend "\n" -j10 --spreadstdin sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum echo '### Test --number-of-cpus'; - stdout parallel --number-of-cpus + stdout $NICEPAR --number-of-cpus echo '### Test --number-of-cores'; - stdout parallel --number-of-cores + stdout $NICEPAR --number-of-cores echo '### Test --use-cpus-instead-of-cores'; - (seq 1 4 | stdout parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done & - (seq 1 4 | stdout parallel -j100% sleep) && echo cores done & - echo 'Cores should complete first on machines with less than 4 physical CPUs'; + (seq 1 8 | stdout parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done & + (seq 1 8 | stdout parallel -j100% sleep) && echo cores done & + echo 'Cores should complete first on machines with less than 8 physical CPUs'; wait echo '### Test --tag ::: a ::: b'; - stdout parallel -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a ::: b + stdout $NICEPAR -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a ::: b echo '### Test --tag ::: a b'; - stdout parallel -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b + stdout $NICEPAR -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b echo '### Test --tag -X ::: a b'; - stdout parallel -k --tag -X -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b + stdout $NICEPAR -k --tag -X -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b echo '### Test bash redirection <()'; - parallel 'cat <(echo {}); echo b' ::: a + $NICEPAR 'cat <(echo {}); echo b' ::: a echo '### Test bug https://savannah.gnu.org/bugs/index.php?33352' @@ -132,7 +134,7 @@ echo '### Test bug https://savannah.gnu.org/bugs/index.php?33352' # child. echo "# md5sum - directly" - perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | md5sum + nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | md5sum echo "# parallel | md5sum" nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR cat | md5sum @@ -214,54 +216,54 @@ echo "### Tests that failed for OO-rewrite" parallel -u --semaphore seq 1 10 '|' pv -qL 20; sem --wait; echo done echo a | parallel echo {1} echo "echo a" | parallel -parallel -j1 -I :: -X echo 'a::b::^c::[.}c' ::: 1 +nice parallel -j1 -I :: -X echo 'a::b::^c::[.}c' ::: 1 echo "### BUG: The length for -X is not close to max (131072)" -seq 1 4000 | parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc +seq 1 4000 | nice parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc echo "### BUG: empty lines with --show-limit" -echo | parallel --show-limits +echo | $NICEPAR --show-limits echo '### Test -N' -seq 1 5 | parallel -kN3 echo {1} {2} {3} +seq 1 5 | $NICEPAR -kN3 echo {1} {2} {3} echo '### Test --arg-file-sep with files of different lengths' -parallel --xapply --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 1) <(seq 3 4) +$XAP --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 1) <(seq 3 4) echo '### Test respect -s' -parallel -kvm -IARG -s15 echo ARG ::: 1 22 333 4444 55555 666666 7777777 88888888 999999999 +$NICEPAR -kvm -IARG -s15 echo ARG ::: 1 22 333 4444 55555 666666 7777777 88888888 999999999 echo '### Test eof string after :::' -parallel -k -E ole echo ::: foo ole bar +$NICEPAR -k -E ole echo ::: foo ole bar echo '### Test -C and --trim rl' -parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' +$NICEPAR -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' echo '### Test empty input' -/dev/null 2>&1 ping -c 1 freebsd7.tange.dk >/dev/null 2>&1 echo "### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error" +rm -rf /tmp/parallel cp `which parallel` /tmp/parallel stdout parallel -kj10 --argsep == --basefile /tmp/parallel --tag --nonall -S redhat9.tange.dk,centos3.tange.dk,centos5.tange.dk,freebsd7.tange.dk /tmp/parallel --no-notice echo ::: OK_if_no_perl_warnings | sort diff --git a/testsuite/tests-to-run/test32.sh b/testsuite/tests-to-run/test32.sh index 1def9f34..77c66994 100644 --- a/testsuite/tests-to-run/test32.sh +++ b/testsuite/tests-to-run/test32.sh @@ -23,7 +23,8 @@ seq 1 2 | parallel --retries 2 --sshlogin 8/localhost,8/: -j-1 "hostname; false" seq 1 1 | parallel --retries 2 --sshlogin 1/localhost,1/: -j1 "hostname; false" | wc -l seq 1 1 | parallel --retries 2 --sshlogin 1/localhost,1/: -j9 "hostname; false" | wc -l seq 1 1 | parallel --retries 2 --sshlogin 1/localhost,1/: -j0 "hostname; false" | wc -l -seq 1 1 | parallel --retries 2 --sshlogin 1/localhost,1/: -j-1 "hostname; false" | wc -l +# Fails due to 0 jobslots +# seq 1 1 | parallel --retries 2 --sshlogin 1/localhost,1/: -j-1 "hostname; false" | wc -l echo '### These were not affected by the bug' seq 1 8 | parallel --retries 2 --sshlogin 1/localhost,9/: -j-1 "hostname; false" | wc -l diff --git a/testsuite/wanted-results/parallel-install b/testsuite/wanted-results/parallel-install index 33c8875c..69291a42 100644 --- a/testsuite/wanted-results/parallel-install +++ b/testsuite/wanted-results/parallel-install @@ -1,5 +1,7 @@ ### Test installation missing pod2* make[0]: Entering directory `/home/tange/privat/parallel' +make dist-gzip am__post_remove_distdir='@:' +make[0]: Entering directory `/home/tange/privat/parallel' if test -d "parallel-00000000"; then find "parallel-00000000" -type d ! -perm -200 -exec chmod u+w {} ';' && rm -rf "parallel-00000000" || { sleep 5 && rm -rf "parallel-00000000"; }; else :; fi test -d "parallel-00000000" || mkdir "parallel-00000000" (cd src && make top_distdir=../parallel-00000000 distdir=../parallel-00000000/src \ @@ -14,6 +16,7 @@ test -n "" \ ! -type d ! -perm -444 -exec /bin/bash /home/tange/privat/parallel/install-sh -c -m a+r {} {} \; \ || chmod -R a+r "parallel-00000000" tardir=parallel-00000000 && ${TAR-tar} chof - "$tardir" | GZIP=--best gzip -c >parallel-00000000.tar.gz +make[0]: Leaving directory `/home/tange/privat/parallel' if test -d "parallel-00000000"; then find "parallel-00000000" -type d ! -perm -200 -exec chmod u+w {} ';' && rm -rf "parallel-00000000" || { sleep 5 && rm -rf "parallel-00000000"; }; else :; fi make[0]: Leaving directory `/home/tange/privat/parallel' checking for a BSD-compatible install... /usr/bin/install -c @@ -21,6 +24,9 @@ checking whether build environment is sane... yes checking for a thread-safe mkdir -p... /bin/mkdir -p checking for gawk... gawk checking whether make sets $(MAKE)... yes +checking whether make supports nested variables... yes +checking whether ln -s works... yes +checking that generated files are newer than configure... done configure: creating ./config.status config.status: creating Makefile config.status: creating src/Makefile @@ -57,31 +63,31 @@ pod2man --release='00000000' --center='parallel' \ || echo "Warning: pod2man not found. Using old parallel_tutorial.1" /bin/bash: pod2man: command not found Warning: pod2man not found. Using old parallel_tutorial.1 -pod2html ./parallel.pod > ./parallel.htmln \ +pod2html --title "GNU Parallel" ./parallel.pod > ./parallel.htmln \ && mv ./parallel.htmln ./parallel.html \ || echo "Warning: pod2html not found. Using old parallel.html" /bin/bash: pod2html: command not found Warning: pod2html not found. Using old parallel.html rm -f ./pod2htm* -pod2html ./parallel_tutorial.pod > ./parallel_tutorial.htmln \ +pod2html --title "GNU Parallel tutorial" ./parallel_tutorial.pod > ./parallel_tutorial.htmln \ && mv ./parallel_tutorial.htmln ./parallel_tutorial.html \ || echo "Warning: pod2html not found. Using old parallel_tutorial.html" /bin/bash: pod2html: command not found Warning: pod2html not found. Using old parallel_tutorial.html rm -f ./pod2htm* -pod2html ./sem.pod > ./sem.htmln \ +pod2html --title "sem (GNU Parallel)" ./sem.pod > ./sem.htmln \ && mv ./sem.htmln ./sem.html \ || echo "Warning: pod2html not found. Using old sem.html" /bin/bash: pod2html: command not found Warning: pod2html not found. Using old sem.html rm -f ./pod2htm* -pod2html ./sql > ./sql.htmln \ +pod2html --title "GNU SQL" ./sql > ./sql.htmln \ && mv ./sql.htmln ./sql.html \ || echo "Warning: pod2html not found. Using old sql.html" /bin/bash: pod2html: command not found Warning: pod2html not found. Using old sql.html rm -f ./pod2htm* -pod2html ./niceload.pod > ./niceload.htmln \ +pod2html --title "GNU niceload" ./niceload.pod > ./niceload.htmln \ && mv ./niceload.htmln ./niceload.html \ || echo "Warning: pod2html not found. Using old niceload.html" /bin/bash: pod2html: command not found @@ -129,32 +135,37 @@ pod2pdf --output-file ./parallel_tutorial.pdf ./parallel_tutorial.pod --title "G Warning: pod2pdf not found. Using old parallel_tutorial.pdf make[0]: Entering directory `/tmp/parallel-00000000/src' /bin/mkdir -p '/usr/local/bin' - /usr/bin/install -c parallel sem sql niceload '/usr/local/bin' -pod2html ./parallel.pod > ./parallel.htmln \ + /usr/bin/install -c parallel sql niceload '/usr/local/bin' +make install-exec-hook +make[0]: Entering directory `/tmp/parallel-00000000/src' +rm /usr/local/bin/sem || true +ln -s /usr/local/bin/parallel /usr/local/bin/sem +make[0]: Leaving directory `/tmp/parallel-00000000/src' +pod2html --title "GNU Parallel" ./parallel.pod > ./parallel.htmln \ && mv ./parallel.htmln ./parallel.html \ || echo "Warning: pod2html not found. Using old parallel.html" /bin/bash: pod2html: command not found Warning: pod2html not found. Using old parallel.html rm -f ./pod2htm* -pod2html ./parallel_tutorial.pod > ./parallel_tutorial.htmln \ +pod2html --title "GNU Parallel tutorial" ./parallel_tutorial.pod > ./parallel_tutorial.htmln \ && mv ./parallel_tutorial.htmln ./parallel_tutorial.html \ || echo "Warning: pod2html not found. Using old parallel_tutorial.html" /bin/bash: pod2html: command not found Warning: pod2html not found. Using old parallel_tutorial.html rm -f ./pod2htm* -pod2html ./sem.pod > ./sem.htmln \ +pod2html --title "sem (GNU Parallel)" ./sem.pod > ./sem.htmln \ && mv ./sem.htmln ./sem.html \ || echo "Warning: pod2html not found. Using old sem.html" /bin/bash: pod2html: command not found Warning: pod2html not found. Using old sem.html rm -f ./pod2htm* -pod2html ./sql > ./sql.htmln \ +pod2html --title "GNU SQL" ./sql > ./sql.htmln \ && mv ./sql.htmln ./sql.html \ || echo "Warning: pod2html not found. Using old sql.html" /bin/bash: pod2html: command not found Warning: pod2html not found. Using old sql.html rm -f ./pod2htm* -pod2html ./niceload.pod > ./niceload.htmln \ +pod2html --title "GNU niceload" ./niceload.pod > ./niceload.htmln \ && mv ./niceload.htmln ./niceload.html \ || echo "Warning: pod2html not found. Using old niceload.html" /bin/bash: pod2html: command not found diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index 863ad290..677ef635 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -9,7 +9,7 @@ echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error" ### Test Force outside the file handle limit, 2009-02-17 Gave fork error (echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | stdout parallel -uj 0 | egrep -v 'processes took|adjusting' parallel: Warning: Only enough file handles to run 252 jobs in parallel. -Raising ulimit -n or /etc/security/limits.conf may help. +Running 'parallel -j0 -N252 --pipe parallel -j0' or raising ulimit -n or /etc/security/limits.conf may help. Start end echo '**' diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 844ea10e..795134ac 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -64,7 +64,6 @@ bug #40137: SHELL not bash: Warning when exporting funcs no_warning . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning warning -parallel: Warning: Shell functions may not be supported in /bin/sh echo '### bug #40002: --files and --nonall seem not to work together:' ### bug #40002: --files and --nonall seem not to work together: parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l diff --git a/testsuite/wanted-results/parallel-local2 b/testsuite/wanted-results/parallel-local2 index 46d42128..1ad46426 100644 --- a/testsuite/wanted-results/parallel-local2 +++ b/testsuite/wanted-results/parallel-local2 @@ -10,8 +10,5 @@ echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834' 1 2 3 -echo '### Test too slow spawning' ### Test too slow spawning -# Let the commands below run during high load -seq `parallel --number-of-cores` | parallel -j200% -N0 timeout -k 25 26 burnP6 & sleep 1; seq 1 1000 | stdout nice nice parallel -s 100 -uj0 true | perl -pe '/parallel: Warning: Starting \d+ processes took/ and do {close STDIN; `killall -9 burnP6`; print "OK\n"; exit }' OK diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index 122664da..2c223fca 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -1,8 +1,8 @@ -echo '### bug #42329: --line-buffer gives wrong output'; parallel --line-buffer --tag seq ::: 10000000 | wc -c; parallel --line-buffer seq ::: 10000000 | wc -c +echo '### bug #42329: --line-buffer gives wrong output'; $NICEPAR --line-buffer --tag seq ::: 10000000 | wc -c; $NICEPAR --line-buffer seq ::: 10000000 | wc -c ### bug #42329: --line-buffer gives wrong output 168888897 78888897 -echo '### Test \0 as recend'; printf "a\0b\0c\0" | parallel --recend '\0' -k -N1 --pipe cat -v \; echo; printf "\0a\0b\0c" | parallel --recstart '\0' -k -N1 --pipe cat -v \; echo +echo '### Test \0 as recend'; printf "a\0b\0c\0" | $NICEPAR --recend '\0' -k -N1 --pipe cat -v \; echo; printf "\0a\0b\0c" | $NICEPAR --recstart '\0' -k -N1 --pipe cat -v \; echo ### Test \0 as recend a^@ b^@ @@ -10,7 +10,7 @@ c^@ ^@a ^@b ^@c -echo '### Test filenames containing UTF-8'; cd tmp; find . -name '*.jpg' | nice nice parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}; find |grep -v CVS | sort; echo '### bug #39554: Feature request: line buffered output'; parallel -j0 --linebuffer 'echo -n start {};sleep 0.{#};echo middle -n {};sleep 1.{#}5;echo next to last {};sleep 1.{#};echo -n last {}' ::: A B C +echo '### Test filenames containing UTF-8'; cd tmp; find . -name '*.jpg' | $NICEPAR -j +0 convert -geometry 120 {} {//}/thumb_{/}; find |grep -v CVS | sort; echo '### bug #39554: Feature request: line buffered output'; parallel -j0 --linebuffer 'echo -n start {};sleep 0.{#};echo middle -n {};sleep 1.{#}5;echo next to last {};sleep 1.{#};echo -n last {}' ::: A B C ### Test filenames containing UTF-8 . ./1-col.txt @@ -44,7 +44,7 @@ B next to last B C next to last C A last AB last BC last Cecho -echo '### test round-robin'; nice seq 1000 | parallel --block 1k --pipe --round-robin wc | sort +echo '### test round-robin'; nice seq 1000 | $NICEPAR --block 1k --pipe --round-robin wc | sort ### test round-robin 223 223 893 250 250 1000 @@ -52,7 +52,7 @@ echo '### test round-robin'; nice seq 1000 | parallel --block 1k --pipe --roun 277 277 1000 echo '### --version must have higher priority than retired options' ### --version must have higher priority than retired options - parallel --version -g -Y -U -W -T | tail + $NICEPAR --version -g -Y -U -W -T | tail License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. GNU parallel comes with no warranty. @@ -65,24 +65,24 @@ O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. echo '### bug #39787: --xargs broken' ### bug #39787: --xargs broken - nice perl -e 'for(1..30000){print "$_\n"}' | nice parallel --xargs -k echo | perl -ne 'print length $_,"\n"' + nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"' 131064 37830 -echo '### --delay should grow by 2 sec per arg' -### --delay should grow by 2 sec per arg -stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 | perl -ne '$_ >= 2 and $_ <= 5 and print "OK\n"' +echo '### --delay should grow by 3 sec per arg' +### --delay should grow by 3 sec per arg + stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"' OK -stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 3 | perl -ne '$_ >= 4 and $_ <= 7 and print "OK\n"' + stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 3 | perl -ne '$_ >= 6 and $_ <= 11 and print "OK\n"' OK echo '### Exit value should not be affected if an earlier job times out' ### Exit value should not be affected if an earlier job times out - parallel -j2 --timeout 1 --joblog - -k ::: "sleep 10" "exit 255" | field 7 + $NICEPAR -j2 --timeout 1 --joblog - -k ::: "sleep 10" "exit 255" | field 7 Exitval -1 255 echo '### --header regexp' ### --header regexp - (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --header '(%.*\n)*' echo JOB{#}\;cat + (echo %head1; echo %head2; seq 5) | $NICEPAR -kj2 --pipe -N2 --header '(%.*\n)*' echo JOB{#}\;cat JOB1 %head1 %head2 @@ -99,7 +99,7 @@ JOB3 5 echo '### --header num' ### --header num - (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --header 2 echo JOB{#}\;cat + (echo %head1; echo %head2; seq 5) | $NICEPAR -kj2 --pipe -N2 --header 2 echo JOB{#}\;cat JOB1 %head1 %head2 @@ -116,21 +116,21 @@ JOB3 5 echo '### --header regexp --round-robin' ### --header regexp --round-robin - (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header '(%.*\n)*' echo JOB\;wc | sort + (echo %head1; echo %head2; seq 5) | $NICEPAR -kj2 --pipe -N2 --round --header '(%.*\n)*' echo JOB\;wc | sort 4 4 18 5 5 20 JOB JOB echo '### --header num --round-robin' ### --header num --round-robin - (echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header 2 echo JOB{#}\;wc | sort + (echo %head1; echo %head2; seq 5) | $NICEPAR -kj2 --pipe -N2 --round --header 2 echo JOB{#}\;wc | sort 4 4 18 5 5 20 JOB1 JOB2 echo '### shebang-wrap' ### shebang-wrap - nice nice parallel -k {} {} A B C ::: ./input-files/shebang/shebangwrap.*[^~] + $NICEPAR -k {} {} A B C ::: ./input-files/shebang/shebangwrap.*[^~] ./input-files/shebang/shebangwrap.oct A B diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 095d004b..1714bfdb 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -1,48 +1,48 @@ echo 'bug #41613: --compress --line-buffer no newline'; bug #41613: --compress --line-buffer no newline - perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo + perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo It worked echo 'bug #41613: --compress --line-buffer no --tagstring'; bug #41613: --compress --line-buffer no --tagstring - diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| parallel -N10 -L1 --pipe -j6 --block 20M --compress pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| parallel -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' + diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' Good: --line-buffer matters echo 'bug #41613: --compress --line-buffer with --tagstring'; bug #41613: --compress --line-buffer with --tagstring - diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| parallel -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| parallel -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' + diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' Good: --line-buffer matters echo 'bug #41613: --compress --line-buffer - no newline'; bug #41613: --compress --line-buffer - no newline - echo 'pipe compress tagstring' + echo 'pipe compress tagstring' pipe compress tagstring - perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat; echo "K" + perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat; echo "K" 1 OK - echo 'pipe compress notagstring' + echo 'pipe compress notagstring' pipe compress notagstring - perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat; echo "K" + perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat; echo "K" OK - echo 'pipe nocompress tagstring' + echo 'pipe nocompress tagstring' pipe nocompress tagstring - perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat; echo "K" + perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat; echo "K" 1 OK - echo 'pipe nocompress notagstring' + echo 'pipe nocompress notagstring' pipe nocompress notagstring - perl -e 'print "O"'| parallel --pipe --line-buffer cat; echo "K" + perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat; echo "K" OK - echo 'nopipe compress tagstring' + echo 'nopipe compress tagstring' nopipe compress tagstring - parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" + $NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" 1 OK - echo 'nopipe compress notagstring' + echo 'nopipe compress notagstring' nopipe compress notagstring - parallel --compress --line-buffer echo {} O ::: -n; echo "K" + $NICEPAR --compress --line-buffer echo {} O ::: -n; echo "K" OK - echo 'nopipe nocompress tagstring' + echo 'nopipe nocompress tagstring' nopipe nocompress tagstring - parallel --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" + $NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" 1 OK - echo 'nopipe nocompress notagstring' + echo 'nopipe nocompress notagstring' nopipe nocompress notagstring - parallel --line-buffer echo {} O ::: -n; echo "K" + $NICEPAR --line-buffer echo {} O ::: -n; echo "K" OK echo 'bug #41412: --timeout + --delay causes deadlock'; bug #41412: --timeout + --delay causes deadlock @@ -61,17 +61,17 @@ bug #41412: --timeout + --delay causes deadlock 1 2 3 - parallel -j10 --timeout 2.2 --delay 3 "sleep {}; echo {}" ::: 1 2 4 5 6 + parallel -j10 --timeout 2.2 --delay 3 "sleep {}; echo {}" ::: 1 2 7 8 9 1 2 -echo '### Test --spreadstdin - more procs than args'; rm -f /tmp/parallel.ss.*; seq 1 5 | stdout parallel -j 10 --spreadstdin 'cat >/tmp/parallel.ss.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss.*; +echo '### Test --spreadstdin - more procs than args'; rm -f /tmp/parallel.ss.*; seq 1 5 | stdout $NICEPAR -j 10 --spreadstdin 'cat >/tmp/parallel.ss.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss.*; ### Test --spreadstdin - more procs than args 1 2 3 4 5 -echo '### Test --spreadstdin - more args than procs'; rm -f /tmp/parallel.ss2.*; seq 1 10 | stdout parallel -j 5 --spreadstdin 'cat >/tmp/parallel.ss2.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss2.* +echo '### Test --spreadstdin - more args than procs'; rm -f /tmp/parallel.ss2.*; seq 1 10 | stdout $NICEPAR -j 5 --spreadstdin 'cat >/tmp/parallel.ss2.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss2.* ### Test --spreadstdin - more args than procs 1 2 @@ -83,55 +83,55 @@ echo '### Test --spreadstdin - more args than procs'; rm -f /tmp/parallel.ss2. 8 9 10 -nice nice seq 1 1000 | nice nice parallel -j1 --spreadstdin cat "|cat "|wc -c +nice nice seq 1 1000 | $NICEPAR -j1 --spreadstdin cat "|cat "|wc -c 3893 -nice nice seq 1 10000 | nice nice parallel -j10 --spreadstdin cat "|cat "|wc -c +nice nice seq 1 10000 | $NICEPAR -j10 --spreadstdin cat "|cat "|wc -c 48894 -nice nice seq 1 100000 | nice nice parallel -j1 --spreadstdin cat "|cat "|wc -c +nice nice seq 1 100000 | $NICEPAR -j1 --spreadstdin cat "|cat "|wc -c 588895 -nice nice seq 1 1000000 | nice nice parallel -j10 --spreadstdin cat "|cat "|wc -c +nice nice seq 1 1000000 | $NICEPAR -j10 --spreadstdin cat "|cat "|wc -c 6888896 -seq 1 10 | parallel --recend "\n" -j1 --spreadstdin gzip -9 >/tmp/foo.gz -echo '### Test --spreadstdin - similar to the failing below'; nice seq 1 100000 | nice nice parallel --recend "\n" -j10 --spreadstdin gzip -9 >/tmp/foo2.gz; diff <(nice seq 1 100000) <(zcat /tmp/foo2.gz |sort -n); diff <(nice seq 1 100000|wc -c) <(zcat /tmp/foo2.gz |wc -c) +seq 1 10 | $NICEPAR --recend "\n" -j1 --spreadstdin gzip -9 >/tmp/foo.gz +echo '### Test --spreadstdin - similar to the failing below'; nice seq 1 100000 | $NICEPAR --recend "\n" -j10 --spreadstdin gzip -9 >/tmp/foo2.gz; diff <(nice seq 1 100000) <(zcat /tmp/foo2.gz |sort -n); diff <(nice seq 1 100000|wc -c) <(zcat /tmp/foo2.gz |wc -c) ### Test --spreadstdin - similar to the failing below -echo '### Test --spreadstdin - this failed during devel'; nice seq 1 1000000 | md5sum; nice seq 1 1000000 | nice nice parallel --recend "\n" -j10 --spreadstdin gzip -9 | zcat | sort -n | md5sum +echo '### Test --spreadstdin - this failed during devel'; nice seq 1 1000000 | md5sum; nice seq 1 1000000 | $NICEPAR --recend "\n" -j10 --spreadstdin gzip -9 | zcat | sort -n | md5sum ### Test --spreadstdin - this failed during devel 8a7095c1c23bfadc311fe6b16d950582 - 8a7095c1c23bfadc311fe6b16d950582 - -echo '### Test --spreadstdin -k'; nice seq 1 1000000 | nice nice parallel -k --recend "\n" -j10 --spreadstdin gzip -9 | zcat | md5sum +echo '### Test --spreadstdin -k'; nice seq 1 1000000 | $NICEPAR -k --recend "\n" -j10 --spreadstdin gzip -9 | zcat | md5sum ### Test --spreadstdin -k 8a7095c1c23bfadc311fe6b16d950582 - -echo '### Test --spreadstdin --files'; nice seq 1 1000000 | shuf | parallel --files --recend "\n" -j10 --spreadstdin sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum +echo '### Test --spreadstdin --files'; nice seq 1 1000000 | shuf | $NICEPAR --files --recend "\n" -j10 --spreadstdin sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum ### Test --spreadstdin --files 8a7095c1c23bfadc311fe6b16d950582 - -echo '### Test --number-of-cpus'; stdout parallel --number-of-cpus +echo '### Test --number-of-cpus'; stdout $NICEPAR --number-of-cpus ### Test --number-of-cpus 1 -echo '### Test --number-of-cores'; stdout parallel --number-of-cores +echo '### Test --number-of-cores'; stdout $NICEPAR --number-of-cores ### Test --number-of-cores 8 -echo '### Test --use-cpus-instead-of-cores'; (seq 1 4 | stdout parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done & (seq 1 4 | stdout parallel -j100% sleep) && echo cores done & echo 'Cores should complete first on machines with less than 4 physical CPUs'; wait +echo '### Test --use-cpus-instead-of-cores'; (seq 1 8 | stdout parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done & (seq 1 8 | stdout parallel -j100% sleep) && echo cores done & echo 'Cores should complete first on machines with less than 8 physical CPUs'; wait ### Test --use-cpus-instead-of-cores -Cores should complete first on machines with less than 4 physical CPUs +Cores should complete first on machines with less than 8 physical CPUs cores done CPUs done -echo '### Test --tag ::: a ::: b'; stdout parallel -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a ::: b +echo '### Test --tag ::: a ::: b'; stdout $NICEPAR -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a ::: b ### Test --tag ::: a ::: b a b stdout-a b a b stderr-a b -echo '### Test --tag ::: a b'; stdout parallel -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b +echo '### Test --tag ::: a b'; stdout $NICEPAR -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b ### Test --tag ::: a b a stdout-a a stderr-a b stdout-b b stderr-b -echo '### Test --tag -X ::: a b'; stdout parallel -k --tag -X -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b +echo '### Test --tag -X ::: a b'; stdout $NICEPAR -k --tag -X -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b ### Test --tag -X ::: a b a b stdout-a stdout-b a b stderr-a stderr-b echo '### Test bash redirection <()'; ### Test bash redirection <() - parallel 'cat <(echo {}); echo b' ::: a + $NICEPAR 'cat <(echo {}); echo b' ::: a a b echo '### Test bug https://savannah.gnu.org/bugs/index.php?33352' @@ -155,7 +155,7 @@ echo '### Test bug https://savannah.gnu.org/bugs/index.php?33352' # child. echo "# md5sum - directly" # md5sum - directly - perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | md5sum + nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# parallel | md5sum" # parallel | md5sum @@ -430,15 +430,15 @@ echo a | parallel echo {1} a echo "echo a" | parallel a -parallel -j1 -I :: -X echo 'a::b::^c::[.}c' ::: 1 +nice parallel -j1 -I :: -X echo 'a::b::^c::[.}c' ::: 1 a1b1^c1[.}c echo "### BUG: The length for -X is not close to max (131072)" ### BUG: The length for -X is not close to max (131072) -seq 1 4000 | parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc +seq 1 4000 | nice parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc 1 12821 131060 echo "### BUG: empty lines with --show-limit" ### BUG: empty lines with --show-limit -echo | parallel --show-limits +echo | $NICEPAR --show-limits Maximal size of command: 131071 Maximal used size of command: 131071 @@ -447,17 +447,17 @@ and run commands; if this is not what you wanted to happen, please press CTRL-D or CTRL-C echo '### Test -N' ### Test -N -seq 1 5 | parallel -kN3 echo {1} {2} {3} +seq 1 5 | $NICEPAR -kN3 echo {1} {2} {3} 1 2 3 4 5 echo '### Test --arg-file-sep with files of different lengths' ### Test --arg-file-sep with files of different lengths -parallel --xapply --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 1) <(seq 3 4) +$XAP --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 1) <(seq 3 4) 1 3 1 4 echo '### Test respect -s' ### Test respect -s -parallel -kvm -IARG -s15 echo ARG ::: 1 22 333 4444 55555 666666 7777777 88888888 999999999 +$NICEPAR -kvm -IARG -s15 echo ARG ::: 1 22 333 4444 55555 666666 7777777 88888888 999999999 echo 1 22 333 1 22 333 echo 4444 @@ -474,51 +474,51 @@ echo 999999999 999999999 echo '### Test eof string after :::' ### Test eof string after ::: -parallel -k -E ole echo ::: foo ole bar +$NICEPAR -k -E ole echo ::: foo ole bar foo echo '### Test -C and --trim rl' ### Test -C and --trim rl -parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' +$NICEPAR -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' a_b_c_ a_b_c_d echo '### Test empty input' ### Test empty input - /dev/null && echo setenv PARALLEL_SEQ 1\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=1\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.' -'newline1 > tmp/parallel.file.' -'newline1.out;cat tmp/parallel.file.' -'newline1 > tmp/parallel.file.' -'newline1.out2 --l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.' -'newline1.out --l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.' -'newline1.out2 -parallel@parallel-server1 (rm -f ./tmp/parallel.file.' -'newline1; rmdir ./tmp/ ./ 2>/dev/null;) -parallel@parallel-server1 (rm -f ./tmp/parallel.file.' -'newline1.out; rmdir ./tmp/ ./ 2>/dev/null;) -parallel@parallel-server1 (rm -f ./tmp/parallel.file.' -'newline1.out2; rmdir ./tmp/ ./ 2>/dev/null;) -parallel@parallel-server2 mkdir -p ./. --l parallel parallel-server2 rsync --server -lDrRze.iLs . ./. --tt -oLogLevel=quiet parallel@parallel-server2 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ 2\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=2\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.' +-tt -oLogLevel=quiet parallel@parallel-server1 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ 2\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=2\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.' 'newline2 > tmp/parallel.file.' 'newline2.out;cat tmp/parallel.file.' 'newline2 > tmp/parallel.file.' 'newline2.out2 --l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.' +-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.' 'newline2.out --l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.' +-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.' 'newline2.out2 -parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +parallel@parallel-server1 (rm -f ./tmp/parallel.file.' 'newline2; rmdir ./tmp/ ./ 2>/dev/null;) -parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +parallel@parallel-server1 (rm -f ./tmp/parallel.file.' 'newline2.out; rmdir ./tmp/ ./ 2>/dev/null;) -parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +parallel@parallel-server1 (rm -f ./tmp/parallel.file.' 'newline2.out2; rmdir ./tmp/ ./ 2>/dev/null;) +parallel@parallel-server2 mkdir -p ./. +-l parallel parallel-server2 rsync --server -lDrRze.iLs . ./. +-tt -oLogLevel=quiet parallel@parallel-server2 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ 1\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=1\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.' +'newline1 > tmp/parallel.file.' +'newline1.out;cat tmp/parallel.file.' +'newline1 > tmp/parallel.file.' +'newline1.out2 +-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.' +'newline1.out +-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.' +'newline1.out2 +parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +'newline1; rmdir ./tmp/ ./ 2>/dev/null;) +parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +'newline1.out; rmdir ./tmp/ ./ 2>/dev/null;) +parallel@parallel-server2 (rm -f ./tmp/parallel.file.' +'newline1.out2; rmdir ./tmp/ ./ 2>/dev/null;) diff --git a/testsuite/wanted-results/test32 b/testsuite/wanted-results/test32 index bf42e7fb..f841e195 100644 --- a/testsuite/wanted-results/test32 +++ b/testsuite/wanted-results/test32 @@ -11,7 +11,6 @@ 1 1 1 -1 ### These were not affected by the bug 8 8