From 09343427ec8bf403d6edda24f32a89fa8ecb4ed0 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 26 Dec 2014 11:14:27 +0100 Subject: [PATCH 1/4] parallel: Bug for --retries on two hosts. --- src/parallel | 1 + 1 file changed, 1 insertion(+) diff --git a/src/parallel b/src/parallel index 2359c7e8..fe03670c 100755 --- a/src/parallel +++ b/src/parallel @@ -4273,6 +4273,7 @@ sub compute_number_of_processes { # Cleanup: Unget the command_lines or the @args $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->unget(@args); $Global::JobQueue->unget(@jobs); + @jobs = undef; } sub processes_available_by_system_limit { From dd9d647e8106d23e7ebe79bce6a19b661e25258d Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 26 Dec 2014 11:44:02 +0100 Subject: [PATCH 2/4] parallel: quoting of ^ has changed. --- src/Makefile.in | 2 +- src/parallel | 8 +++++--- testsuite/wanted-results/parallel-local23 | 12 ++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Makefile.in b/src/Makefile.in index 16f9df9d..ba2715f3 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -78,7 +78,7 @@ NORMAL_UNINSTALL = : PRE_UNINSTALL = : POST_UNINSTALL = : subdir = src -DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am +DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am README ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/configure.ac am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ diff --git a/src/parallel b/src/parallel index fe03670c..ddd5a8b0 100755 --- a/src/parallel +++ b/src/parallel @@ -3375,6 +3375,9 @@ sub multiply_binary_prefix { # Returns: # $value = int with prefixes multiplied my $s = shift; + if(not $s) { + return $s; + } $s =~ s/ki/*1024/gi; $s =~ s/mi/*1024*1024/gi; $s =~ s/gi/*1024*1024*1024/gi; @@ -3994,7 +3997,6 @@ sub loadavg_too_high { }; $ps =~ s/[ \t\n]+/ /g; $cmd = "perl -e ".::shell_quote_scalar($ps); - $cmd =~ s/\^/\\^/g; } return $cmd; } @@ -4057,7 +4059,6 @@ sub loadavg { } else { $cmd .= loadavg_cmd(); } -# $cmd .= "ps ax -o state,command"; # As the command can take long to run if run remote # save it to a tmp file before moving it to the correct file ::debug("load", "Cmd: ", $cmd); @@ -7229,6 +7230,7 @@ sub new { my $return_files = shift; my @unget = (); my ($count,$posrpl,$perlexpr); + my ($replacecount_ref, $len_ref); my @command = @$commandref; # If the first command start with '-' it is probably an option if($command[0] =~ /^\s*(-\S+)/) { @@ -7283,7 +7285,7 @@ sub new { } } # Add {} if no replacement strings in @command - my($replacecount_ref, $len_ref, @command) = + ($replacecount_ref, $len_ref, @command) = replacement_counts_and_lengths(@command); if("@command" =~ /^\S*\257 +This is free software: you are free to change and redistribute it. GNU parallel comes with no warranty. Web site: http://www.gnu.org/software/parallel -When using programs that use GNU Parallel to process data for publication please cite: - -O. Tange (2011): GNU Parallel - The Command-Line Power Tool, -;login: The USENIX Magazine, February 2011:42-47. - -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +When using programs that use GNU Parallel to process data for publication +please cite as described in 'parallel --bibtex'. echo '### bug #39787: --xargs broken' ### bug #39787: --xargs broken nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"' From 8bcb05813bd893911091e134f4d4dd4136eb40e2 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 26 Dec 2014 12:02:31 +0100 Subject: [PATCH 3/4] parallel: quoting of replacement string. --- src/parallel | 9 ++++++--- testsuite/wanted-results/parallel-local12 | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/parallel b/src/parallel index ddd5a8b0..ef3d7a31 100755 --- a/src/parallel +++ b/src/parallel @@ -1481,7 +1481,7 @@ sub shell_quote { # @shell_quoted_strings = string quoted with \ as needed by the shell my @strings = (@_); for my $a (@strings) { - $a =~ s/([\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\*\>\<\~\|\; \"\!\$\&\'\202-\377])/\\$1/g; + $a =~ s/([\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\>\<\~\|\; \"\!\$\&\'\202-\377])/\\$1/g; $a =~ s/[\n]/'\n'/g; # filenames with '\n' is quoted using \' } return wantarray ? @strings : "@strings"; @@ -7210,6 +7210,8 @@ sub replaced { my $regexp = join('|', map { my $s = $_; $s =~ s/(\W)/\\$1/g; $s } sort { length $b <=> length $a } keys %replace); for(@target) { + # ::debug("replace","Replace in ",::my_dump($_)); + # TODO can this be /o ? s/($regexp)/join(" ",@{$replace{$1}})/ge; } } @@ -7287,8 +7289,9 @@ sub new { # Add {} if no replacement strings in @command ($replacecount_ref, $len_ref, @command) = replacement_counts_and_lengths(@command); - if("@command" =~ /^\S*\257 Date: Fri, 26 Dec 2014 18:31:06 +0100 Subject: [PATCH 4/4] parallel: Avoid using Tie::RefHash. Passes test-suite. --- src/niceload | 2 +- src/parallel | 18 +- testsuite/tests-to-run/parallel-local1.sh | 4 +- testsuite/wanted-results/parallel-local-3s | 20 +- testsuite/wanted-results/parallel-local1 | 8 +- testsuite/wanted-results/parallel-local104 | Bin 370 -> 370 bytes testsuite/wanted-results/parallel-local105 | 6 +- testsuite/wanted-results/parallel-local15 | 242 ++++++++++----------- testsuite/wanted-results/parallel-local23 | 4 +- testsuite/wanted-results/parallel-local9 | 4 +- testsuite/wanted-results/test13 | 11 +- testsuite/wanted-results/test15 | 18 +- 12 files changed, 171 insertions(+), 166 deletions(-) diff --git a/src/niceload b/src/niceload index 534479c6..1785914e 100755 --- a/src/niceload +++ b/src/niceload @@ -3,7 +3,7 @@ # Copyright (C) 2004,2005,2006,2006,2008,2009,2010 Ole Tange, # http://ole.tange.dk # -# Copyright (C) 2010,2011,2012,2013,2014 Ole Tange, +# Copyright (C) 2010,2011,2012,2013,2014,2015 Ole Tange, # http://ole.tange.dk and Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify diff --git a/src/parallel b/src/parallel index ef3d7a31..36dfcf60 100755 --- a/src/parallel +++ b/src/parallel @@ -1,7 +1,7 @@ #!/usr/bin/env perl -# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014 Ole Tange and -# Free Software Foundation, Inc. +# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange +# and Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -951,7 +951,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20141212; + $Global::version = 20141225; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -3036,7 +3036,8 @@ sub version { } print join("\n", "GNU $Global::progname $Global::version", - "Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014 Ole Tange and Free Software Foundation, Inc.", + "Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange", + "and Free Software Foundation, Inc.", "License GPLv3+: GNU GPL version 3 or later ", "This is free software: you are free to change and redistribute it.", "GNU $Global::progname comes with no warranty.", @@ -3341,13 +3342,15 @@ sub kill_youngster_if_not_enough_mem { # If less than 50% enough free mem: kill off the youngest child # Put the child back in the queue. my %jobs_of; - use Tie::RefHash; - tie %jobs_of, 'Tie::RefHash'; + my @sshlogins; for my $job (values %Global::running) { + if(not $jobs_of{$job->sshlogin()}) { + push @sshlogins, $job->sshlogin(); + } push @{$jobs_of{$job->sshlogin()}}, $job; } - for my $sshlogin (keys %jobs_of) { + for my $sshlogin (@sshlogins) { for my $job (sort { $b->seq() <=> $a->seq() } @{$jobs_of{$sshlogin}}) { if($sshlogin->memfree() < $opt::memfree * 0.5) { ::debug("mem","\n",map { $_->seq()." " } (sort { $b->seq() <=> $a->seq() } @{$jobs_of{$sshlogin}})); @@ -7211,7 +7214,6 @@ sub replaced { sort { length $b <=> length $a } keys %replace); for(@target) { # ::debug("replace","Replace in ",::my_dump($_)); - # TODO can this be /o ? s/($regexp)/join(" ",@{$replace{$1}})/ge; } } diff --git a/testsuite/tests-to-run/parallel-local1.sh b/testsuite/tests-to-run/parallel-local1.sh index 4643e555..6c9ed7ff 100644 --- a/testsuite/tests-to-run/parallel-local1.sh +++ b/testsuite/tests-to-run/parallel-local1.sh @@ -1,8 +1,8 @@ #!/bin/bash cat <<'EOF' | parallel -vj0 -k -echo "bug #43654: --bar with command not using {}" - COLUMNS=80 stdout parallel --bar true {.} ::: 1 +echo "bug #43654: --bar with command not using {} - only last output line " + COLUMNS=80 stdout parallel --bar true {.} ::: 1 | perl -pe 's/.*\r/\r/' echo "### Test --basenamereplace" parallel -j1 -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index 515199c4..60ee916c 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -19,9 +19,9 @@ echo '### Test --halt-on-error 1'; (echo "sleep 1;true"; echo "sleep 2;false"; 127 parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: sleep 2;false -/bin/bash: non_exist: command not found parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: sleep 2;false +/bin/bash: non_exist: command not found parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: sleep 4; non_exist echo '**' @@ -41,31 +41,31 @@ echo '### Test last dying print --halt-on-error 1'; (seq 0 8;echo 0; echo 9) | exit code 9 0 1 -2 -3 -4 -5 -6 -7 -8 -0 -9 parallel: Starting no more jobs. Waiting for 9 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 +2 parallel: Starting no more jobs. Waiting for 8 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2 +3 parallel: Starting no more jobs. Waiting for 7 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3 +4 parallel: Starting no more jobs. Waiting for 6 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4 +5 parallel: Starting no more jobs. Waiting for 5 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5 +6 parallel: Starting no more jobs. Waiting for 4 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6 +7 parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7 +8 +0 parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8 +9 parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9 echo '### Test last dying print --halt-on-error 2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; echo exit code $? diff --git a/testsuite/wanted-results/parallel-local1 b/testsuite/wanted-results/parallel-local1 index 268964f4..d14a0b37 100644 --- a/testsuite/wanted-results/parallel-local1 +++ b/testsuite/wanted-results/parallel-local1 @@ -1,7 +1,7 @@ -echo "bug #43654: --bar with command not using {}" -bug #43654: --bar with command not using {} - COLUMNS=80 stdout parallel --bar true {.} ::: 1 - # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 100 100% 1:0=0s 1  +echo "bug #43654: --bar with command not using {} - only last output line " +bug #43654: --bar with command not using {} - only last output line + COLUMNS=80 stdout parallel --bar true {.} ::: 1 | perl -pe 's/.*\r/\r/' + 100% 1:0=0s 1  echo "### Test --basenamereplace" ### Test --basenamereplace parallel -j1 -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b diff --git a/testsuite/wanted-results/parallel-local104 b/testsuite/wanted-results/parallel-local104 index 4a369e13ea15d189778cc7f590b6a32bb9a08ba2..8938e877d1e12e93c3cc484c14373e65e9866235 100644 GIT binary patch delta 117 zcmWN}u@S={3nxs};5M<&2TLnXk2gsJ3pzSeh7R11bvi%F=kUh8vu>!ZkfXv6wl Ef1Btf2mk;8 delta 117 zcmV~$OAW&?3; This is free software: you are free to change and redistribute it. GNU parallel comes with no warranty. diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 1d50b9c6..66a55420 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -206,12 +206,14 @@ With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = See 'man parallel' for details +Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. + parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed. echo '### Test of -j filename'; echo 3 >/tmp/jobs_to_run1; parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4; # Should give 6 8 10 5 4 ### Test of -j filename diff --git a/testsuite/wanted-results/test13 b/testsuite/wanted-results/test13 index 6c952f4c..c32da0a3 100644 --- a/testsuite/wanted-results/test13 +++ b/testsuite/wanted-results/test13 @@ -51,8 +51,6 @@ job2 14 15 16 -17 -18 2 3 4 @@ -61,10 +59,10 @@ job2 7 8 9 -Running 'parallel -j0 -N9 --pipe parallel -j0' or raising ulimit -n or /etc/security/limits.conf may help. +Running 'parallel -j0 -N8 --pipe parallel -j0' or raising ulimit -n or /etc/security/limits.conf may help. parallel: SIGTERM received. No new jobs will be started. -parallel: Waiting for these 9 jobs to finish. Send SIGTERM again to stop now. -parallel: Warning: Only enough file handles to run 9 jobs in parallel. +parallel: Waiting for these 8 jobs to finish. Send SIGTERM again to stop now. +parallel: Warning: Only enough file handles to run 8 jobs in parallel. parallel: sleep 3; echo 10 parallel: sleep 3; echo 11 parallel: sleep 3; echo 12 @@ -72,8 +70,7 @@ parallel: sleep 3; echo 13 parallel: sleep 3; echo 14 parallel: sleep 3; echo 15 parallel: sleep 3; echo 16 -parallel: sleep 3; echo 17 -parallel: sleep 3; echo 18 +parallel: sleep 3; echo 9 ### Test bug: empty line for | sh with -k a b diff --git a/testsuite/wanted-results/test15 b/testsuite/wanted-results/test15 index e2d07d6a..aecb222f 100644 --- a/testsuite/wanted-results/test15 +++ b/testsuite/wanted-results/test15 @@ -1,12 +1,13 @@ ### Test -p --interactive spawn /tmp/parallel-script-for-expect +Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. This helps funding further development; and it won't cost you a cent. -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'. @@ -15,13 +16,14 @@ sleep 0.1; echo opt-p 2 ?...n sleep 0.1; echo opt-p 3 ?...y opt-p 1 opt-p 3 +Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. This helps funding further development; and it won't cost you a cent. -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'. @@ -256,13 +258,14 @@ xargs Expect: 3 1 2 1 2 parallel Expect: 3 1 via psedotty 2 +Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. This helps funding further development; and it won't cost you a cent. -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'. @@ -275,13 +278,14 @@ xargs Expect: 1 3 2 3 2 parallel Expect: 1 3 2 via pseudotty +Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. This helps funding further development; and it won't cost you a cent. -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'. @@ -387,10 +391,10 @@ line 2 ### Test --no-run-if-empty and -r: This should give no output ### Test --help and -h: Help output (just check we get the same amount of lines) Output from -h and --help -33 -33 +35 +35 ### Test --version: Version output (just check we get the same amount of lines) -14 +11 ### Test --verbose and -t echo bar echo car