diff --git a/doc/release_new_version b/doc/release_new_version index c31fe174..f5e2d74f 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -226,34 +226,17 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20141122 ('Rosetta') released +Subject: GNU Parallel 20141222 ('') released -GNU Parallel 20141122 ('Rosetta') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20141222 ('') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ Haiku of the month: -Hadoop bit too much? -Want a simpler syntax now? -Use GNU Parallel. - -- Ole Tange - -A central piece of command generation was rewritten making this release beta quality. As always it passes the testsuite, so most functionality clearly works. +<<>> New in this release: -* Remote systems can be divided into hostgroups (e.g. web and db) by prepending '@groupname/' to the sshlogin. Multiple groups can be given by separating groups with '+'. E.g. @web/www1 @web+db/www2 @db/mariadb - -* Remote execution can be restricted to servers that are part of one or more groups by '@groupname' as an sshlogin. Multiple groups can be given by separating groups with '+'. E.g. -S @web or -S @db+web - -* With --hostgroup you can restrict arguments to certain hostgroups by appending '@groupname' to the argument. Multiple groups can be given by separating groups with '+'. E.g. my_web_arg@web db-or-web-arg@db+web db-only-arg@db Thanks to Michel Courtine for developing a prototype for this. - -* GNU Parallel was cited in: HTSeq-Hadoop: Extending HTSeq for Massively Parallel Sequencing Data Analysis using Hadoop http://essenceofescience.se/wp-content/uploads/2014/11/Siretskiy.pdf - -* GNU Parallel was cited in: SlideToolkit: An Assistive Toolset for the Histological Quantification of Whole Slide Images http://www.plosone.org/article/info%3Adoi%2F10.1371%2Fjournal.pone.0110289#close - -* GNU Parallel was cited in: Exploring a multiprocessor design space to analyze the impact of using STT-RAM in the memory hierarchy http://conservancy.umn.edu/bitstream/handle/11299/167286/Borse_umn_0130M_15431.pdf - -* Command-Line OCR with Tesseract on Mac OS X https://ryanfb.github.io/etc/2014/11/13/command_line_ocr_on_mac_os_x.html +* GNU Parallel was cited in: Parallel post-processing with MPI-Bash http://dl.acm.org/citation.cfm?id=2691137 * Bug fixes and man page updates. diff --git a/src/parallel b/src/parallel index 4b449573..25b62f81 100755 --- a/src/parallel +++ b/src/parallel @@ -194,7 +194,6 @@ sub pipe_part_files { # Remote exec should look like: # ssh -oLogLevel=quiet lo 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ FOO\ /tmp/foo\ \|\|\ export\ FOO=/tmp/foo\; \(wc\ -\ \$FOO\) # ssh -tt not allowed. Remote will die due to broken pipe anyway. - # TODO test remote with --fifo / --cat return @cat_partials; } @@ -3134,15 +3133,15 @@ sub which { ( 'aix' => $sysv, 'cygwin' => $sysv, - 'msys' => $sysv, - 'dec_osf' => $sysv, 'darwin' => $bsd, + 'dec_osf' => $sysv, 'dragonfly' => $bsd, 'freebsd' => $bsd, 'gnu' => $sysv, 'hpux' => $sysv, 'linux' => $sysv, 'mirbsd' => $bsd, + 'msys' => $sysv, 'netbsd' => $bsd, 'nto' => $sysv, 'openbsd' => $bsd, @@ -3724,6 +3723,66 @@ sub loadavg_too_high { $loadavg > $self->max_loadavg()); } +{ + my $cmd; + sub loadavg_cmd { + if(not $cmd) { + # aix => "ps -ae -o state,command" # state wrong + # bsd => "ps ax -o state,command" + # sysv => "ps -ef -o s -o comm" + # dec_osf => bsd + # dragonfly => bsd + # freebsd => bsd + # gnu => bsd + # hpux => ps -el|awk '{print $2,$14,$15}' + # irix => ps -ef -o state -o comm + # linux => bsd + # minix => ps el|awk '{print \$1,\$11}' + # mirbsd => bsd + # netbsd => bsd + # openbsd => bsd + # solaris => sysv + # svr5 => sysv + # ultrix => ps -ax | awk '{print $3,$5}' + # unixware => ps -el|awk '{print $2,$14,$15}' + my $ps = q{ + $sysv="ps -ef -o s -o comm"; + $sysv2="ps -ef -o state -o comm"; + $bsd="ps ax -o state,command"; + $psel="ps -el|awk '{ print \$2,\$14,\$15 }'"; + $dummy="echo S COMMAND;echo R dummy"; + %ps=( + 'aix' => "uptime", + 'cygwin' => $sysv, + 'darwin' => $bsd, + 'dec_osf' => $sysv2, + 'dragonfly' => $bsd, + 'freebsd' => $bsd, + 'gnu' => $bsd, + 'hpux' => $psel, + 'irix' => 'ps -ef -o state -o comm', + 'linux' => $bsd, + 'minix' => "ps el|awk '{print \$1,\$11}'", + 'mirbsd' => $bsd, + 'msys' => $sysv, + 'netbsd' => $bsd, + 'nto' => $dummy, + 'openbsd' => $bsd, + 'solaris' => $sysv, + 'svr5' => $psel, + 'ultrix' => "ps -ax | awk '{print \$3,\$5}'", + ); + print `$ps{$^O}`; + }; + $ps =~ s/[ \t\n]+/ /g; + $cmd = "perl -e ".::shell_quote_scalar($ps); + $cmd =~ s/\^/\\^/g; + } + return $cmd; + } +} + + sub loadavg { # If the currently know loadavg is too old: # Recompute a new one in the background @@ -3741,21 +3800,28 @@ sub loadavg { local $/ = undef; my $load_out = <$load_fh>; close $load_fh; - my $load =()= ($load_out=~/(^[DR]....[^\[])/gm); + my $load =()= ($load_out=~/(^\s?[DOR]\S* [^\[])/gm); if($load > 0) { # load is overestimated by 1 $self->{'loadavg'} = $load - 1; - ::debug("load", "New loadavg: ", $self->{'loadavg'}); - } else { - ::die_bug("loadavg_invalid_content: $load_out"); + ::debug("load", "New loadavg: ", $self->{'loadavg'},"\n"); + } elsif ($load_out=~/average: (\d+.\d+)/) { + # Aix does not support instant load average + # 04:11AM up 21 days, 12:55, 1 user, load average: 1.85, 1.57, 1.55 + $self->{'loadavg'} = $1; + } else { + ::die_bug("loadavg_invalid_content: " . + $self->{'loadavg_file'} . "\n$load_out"); } - ::debug("load", "Last update: ", $self->{'last_loadavg_update'}); - if(time - $self->{'last_loadavg_update'} > 10) { - # last loadavg was started 10 seconds ago - ::debug("load", time - $self->{'last_loadavg_update'}, " secs old: ", - $self->{'loadavg_file'}); + # Because of instant load average, it should not be delayed 10 secs + # The instant load does not give 2 R if there is only 1 cpu. +# ::debug("load", "Last update: ", $self->{'last_loadavg_update'}); +# if(time - $self->{'last_loadavg_update'} > 10) { +# # last loadavg was started 10 seconds ago +# ::debug("load", time - $self->{'last_loadavg_update'}, " secs old: ", +# $self->{'loadavg_file'}); $update_loadavg_file = 1; - } +# } } else { ::debug("load", "No loadavg file: ", $self->{'loadavg_file'}); $self->{'loadavg'} = undef; @@ -3768,12 +3834,15 @@ sub loadavg { -e $ENV{'HOME'}."/.parallel/tmp" or mkdir $ENV{'HOME'}."/.parallel/tmp"; my $cmd = ""; if($self->{'string'} ne ":") { - $cmd = $self->sshcommand() . " " . $self->serverlogin() . " "; + $cmd = $self->sshcommand() . " " . $self->serverlogin() . " " . + ::shell_quote_scalar(loadavg_cmd()); + } else { + $cmd .= loadavg_cmd(); } - # TODO Is is called 'ps ax -o state,command' on other platforms? - $cmd .= "ps ax -o state,command"; +# $cmd .= "ps ax -o state,command"; # As the command can take long to run if run remote # save it to a tmp file before moving it to the correct file + ::debug("load", "Cmd: ", $cmd); my $file = $self->{'loadavg_file'}; my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".loa"); qx{ ($cmd > $tmpfile && mv $tmpfile $file || rm $tmpfile) & };