From af6e638719bea4738040ff0fe091ea6aeadd0962 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 26 Dec 2014 18:31:06 +0100 Subject: [PATCH] parallel: Avoid using Tie::RefHash. Passes test-suite. --- src/niceload | 2 +- src/parallel | 18 +- testsuite/tests-to-run/parallel-local1.sh | 4 +- testsuite/wanted-results/parallel-local-3s | 20 +- testsuite/wanted-results/parallel-local1 | 8 +- testsuite/wanted-results/parallel-local104 | Bin 370 -> 370 bytes testsuite/wanted-results/parallel-local105 | 6 +- testsuite/wanted-results/parallel-local15 | 242 ++++++++++----------- testsuite/wanted-results/parallel-local23 | 4 +- testsuite/wanted-results/parallel-local9 | 4 +- testsuite/wanted-results/test13 | 11 +- testsuite/wanted-results/test15 | 18 +- 12 files changed, 171 insertions(+), 166 deletions(-) diff --git a/src/niceload b/src/niceload index 534479c6..1785914e 100755 --- a/src/niceload +++ b/src/niceload @@ -3,7 +3,7 @@ # Copyright (C) 2004,2005,2006,2006,2008,2009,2010 Ole Tange, # http://ole.tange.dk # -# Copyright (C) 2010,2011,2012,2013,2014 Ole Tange, +# Copyright (C) 2010,2011,2012,2013,2014,2015 Ole Tange, # http://ole.tange.dk and Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify diff --git a/src/parallel b/src/parallel index ef3d7a31..36dfcf60 100755 --- a/src/parallel +++ b/src/parallel @@ -1,7 +1,7 @@ #!/usr/bin/env perl -# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014 Ole Tange and -# Free Software Foundation, Inc. +# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange +# and Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -951,7 +951,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20141212; + $Global::version = 20141225; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -3036,7 +3036,8 @@ sub version { } print join("\n", "GNU $Global::progname $Global::version", - "Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014 Ole Tange and Free Software Foundation, Inc.", + "Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange", + "and Free Software Foundation, Inc.", "License GPLv3+: GNU GPL version 3 or later ", "This is free software: you are free to change and redistribute it.", "GNU $Global::progname comes with no warranty.", @@ -3341,13 +3342,15 @@ sub kill_youngster_if_not_enough_mem { # If less than 50% enough free mem: kill off the youngest child # Put the child back in the queue. my %jobs_of; - use Tie::RefHash; - tie %jobs_of, 'Tie::RefHash'; + my @sshlogins; for my $job (values %Global::running) { + if(not $jobs_of{$job->sshlogin()}) { + push @sshlogins, $job->sshlogin(); + } push @{$jobs_of{$job->sshlogin()}}, $job; } - for my $sshlogin (keys %jobs_of) { + for my $sshlogin (@sshlogins) { for my $job (sort { $b->seq() <=> $a->seq() } @{$jobs_of{$sshlogin}}) { if($sshlogin->memfree() < $opt::memfree * 0.5) { ::debug("mem","\n",map { $_->seq()." " } (sort { $b->seq() <=> $a->seq() } @{$jobs_of{$sshlogin}})); @@ -7211,7 +7214,6 @@ sub replaced { sort { length $b <=> length $a } keys %replace); for(@target) { # ::debug("replace","Replace in ",::my_dump($_)); - # TODO can this be /o ? s/($regexp)/join(" ",@{$replace{$1}})/ge; } } diff --git a/testsuite/tests-to-run/parallel-local1.sh b/testsuite/tests-to-run/parallel-local1.sh index 4643e555..6c9ed7ff 100644 --- a/testsuite/tests-to-run/parallel-local1.sh +++ b/testsuite/tests-to-run/parallel-local1.sh @@ -1,8 +1,8 @@ #!/bin/bash cat <<'EOF' | parallel -vj0 -k -echo "bug #43654: --bar with command not using {}" - COLUMNS=80 stdout parallel --bar true {.} ::: 1 +echo "bug #43654: --bar with command not using {} - only last output line " + COLUMNS=80 stdout parallel --bar true {.} ::: 1 | perl -pe 's/.*\r/\r/' echo "### Test --basenamereplace" parallel -j1 -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index 515199c4..60ee916c 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -19,9 +19,9 @@ echo '### Test --halt-on-error 1'; (echo "sleep 1;true"; echo "sleep 2;false"; 127 parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: sleep 2;false -/bin/bash: non_exist: command not found parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: sleep 2;false +/bin/bash: non_exist: command not found parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: sleep 4; non_exist echo '**' @@ -41,31 +41,31 @@ echo '### Test last dying print --halt-on-error 1'; (seq 0 8;echo 0; echo 9) | exit code 9 0 1 -2 -3 -4 -5 -6 -7 -8 -0 -9 parallel: Starting no more jobs. Waiting for 9 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 +2 parallel: Starting no more jobs. Waiting for 8 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2 +3 parallel: Starting no more jobs. Waiting for 7 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3 +4 parallel: Starting no more jobs. Waiting for 6 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4 +5 parallel: Starting no more jobs. Waiting for 5 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5 +6 parallel: Starting no more jobs. Waiting for 4 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6 +7 parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7 +8 +0 parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8 +9 parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9 echo '### Test last dying print --halt-on-error 2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; echo exit code $? diff --git a/testsuite/wanted-results/parallel-local1 b/testsuite/wanted-results/parallel-local1 index 268964f4..d14a0b37 100644 --- a/testsuite/wanted-results/parallel-local1 +++ b/testsuite/wanted-results/parallel-local1 @@ -1,7 +1,7 @@ -echo "bug #43654: --bar with command not using {}" -bug #43654: --bar with command not using {} - COLUMNS=80 stdout parallel --bar true {.} ::: 1 - # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 0 0% 0:1=0s 1  # 0 sec 1 100 100% 1:0=0s 1  +echo "bug #43654: --bar with command not using {} - only last output line " +bug #43654: --bar with command not using {} - only last output line + COLUMNS=80 stdout parallel --bar true {.} ::: 1 | perl -pe 's/.*\r/\r/' + 100% 1:0=0s 1  echo "### Test --basenamereplace" ### Test --basenamereplace parallel -j1 -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b diff --git a/testsuite/wanted-results/parallel-local104 b/testsuite/wanted-results/parallel-local104 index 4a369e13ea15d189778cc7f590b6a32bb9a08ba2..8938e877d1e12e93c3cc484c14373e65e9866235 100644 GIT binary patch delta 117 zcmWN}u@S={3nxs};5M<&2TLnXk2gsJ3pzSeh7R11bvi%F=kUh8vu>!ZkfXv6wl Ef1Btf2mk;8 delta 117 zcmV~$OAW&?3; This is free software: you are free to change and redistribute it. GNU parallel comes with no warranty. diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 1d50b9c6..66a55420 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -206,12 +206,14 @@ With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = See 'man parallel' for details +Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. + parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed. echo '### Test of -j filename'; echo 3 >/tmp/jobs_to_run1; parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4; # Should give 6 8 10 5 4 ### Test of -j filename diff --git a/testsuite/wanted-results/test13 b/testsuite/wanted-results/test13 index 6c952f4c..c32da0a3 100644 --- a/testsuite/wanted-results/test13 +++ b/testsuite/wanted-results/test13 @@ -51,8 +51,6 @@ job2 14 15 16 -17 -18 2 3 4 @@ -61,10 +59,10 @@ job2 7 8 9 -Running 'parallel -j0 -N9 --pipe parallel -j0' or raising ulimit -n or /etc/security/limits.conf may help. +Running 'parallel -j0 -N8 --pipe parallel -j0' or raising ulimit -n or /etc/security/limits.conf may help. parallel: SIGTERM received. No new jobs will be started. -parallel: Waiting for these 9 jobs to finish. Send SIGTERM again to stop now. -parallel: Warning: Only enough file handles to run 9 jobs in parallel. +parallel: Waiting for these 8 jobs to finish. Send SIGTERM again to stop now. +parallel: Warning: Only enough file handles to run 8 jobs in parallel. parallel: sleep 3; echo 10 parallel: sleep 3; echo 11 parallel: sleep 3; echo 12 @@ -72,8 +70,7 @@ parallel: sleep 3; echo 13 parallel: sleep 3; echo 14 parallel: sleep 3; echo 15 parallel: sleep 3; echo 16 -parallel: sleep 3; echo 17 -parallel: sleep 3; echo 18 +parallel: sleep 3; echo 9 ### Test bug: empty line for | sh with -k a b diff --git a/testsuite/wanted-results/test15 b/testsuite/wanted-results/test15 index e2d07d6a..aecb222f 100644 --- a/testsuite/wanted-results/test15 +++ b/testsuite/wanted-results/test15 @@ -1,12 +1,13 @@ ### Test -p --interactive spawn /tmp/parallel-script-for-expect +Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. This helps funding further development; and it won't cost you a cent. -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'. @@ -15,13 +16,14 @@ sleep 0.1; echo opt-p 2 ?...n sleep 0.1; echo opt-p 3 ?...y opt-p 1 opt-p 3 +Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. This helps funding further development; and it won't cost you a cent. -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'. @@ -256,13 +258,14 @@ xargs Expect: 3 1 2 1 2 parallel Expect: 3 1 via psedotty 2 +Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. This helps funding further development; and it won't cost you a cent. -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'. @@ -275,13 +278,14 @@ xargs Expect: 1 3 2 3 2 parallel Expect: 1 3 2 via pseudotty +Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. This helps funding further development; and it won't cost you a cent. -Or you can get GNU Parallel without this requirement by paying 10000 EUR. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. To silence this citation notice run 'parallel --bibtex' once or use '--no-notice'. @@ -387,10 +391,10 @@ line 2 ### Test --no-run-if-empty and -r: This should give no output ### Test --help and -h: Help output (just check we get the same amount of lines) Output from -h and --help -33 -33 +35 +35 ### Test --version: Version output (just check we get the same amount of lines) -14 +11 ### Test --verbose and -t echo bar echo car