From fe4173927fae9d617ae347b6b9f5ecf324ac1db1 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 17 Mar 2019 23:07:32 +0100 Subject: [PATCH] parallel: Fixed: --null conflicts with --resume --joblog. --- configure | 20 +- configure.ac | 2 +- doc/haikus | 7 +- doc/release_new_version | 7 +- src/Makefile.am | 2 +- src/Makefile.in | 2 +- src/env_parallel.ash | 1 + src/env_parallel.bash | 5 +- src/env_parallel.dash | 1 + src/parallel | 55 +++--- src/parallel.pod | 96 ++++++---- src/parallel_alternatives.pod | 125 +++++++++++- src/parallel_cheat.fodt | 180 +++++++++++------- src/parallel_tutorial.pod | 8 +- src/parcat.pod | 2 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 5 + testsuite/tests-to-run/parallel-local-1s.sh | 13 ++ testsuite/tests-to-run/parallel-local-30s.sh | 6 +- testsuite/tests-to-run/parallel-local-3s.sh | 17 +- testsuite/wanted-results/parallel-local-0.3s | 2 + testsuite/wanted-results/parallel-local-100s | 4 +- testsuite/wanted-results/parallel-local-10s | 4 +- testsuite/wanted-results/parallel-local-1s | 7 + testsuite/wanted-results/parallel-local12 | 2 + 24 files changed, 396 insertions(+), 177 deletions(-) diff --git a/configure b/configure index 0b18c92c..c711473a 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20190222. +# Generated by GNU Autoconf 2.69 for parallel 20190310. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20190222' -PACKAGE_STRING='parallel 20190222' +PACKAGE_VERSION='20190310' +PACKAGE_STRING='parallel 20190310' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20190222 to adapt to many kinds of systems. +\`configure' configures parallel 20190310 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1281,7 +1281,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20190222:";; + short | recursive ) echo "Configuration of parallel 20190310:";; esac cat <<\_ACEOF @@ -1357,7 +1357,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20190222 +parallel configure 20190310 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20190222, which was +It was created by parallel $as_me 20190310, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2237,7 +2237,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20190222' + VERSION='20190310' cat >>confdefs.h <<_ACEOF @@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20190222, which was +This file was extended by parallel $as_me 20190310, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2942,7 +2942,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20190222 +parallel config.status 20190310 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 71224308..24c9acc0 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20190222], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20190310], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/haikus b/doc/haikus index 8364cdac..d15e8141 100644 --- a/doc/haikus +++ b/doc/haikus @@ -1,9 +1,10 @@ Quote of the month: +There are so many things to love about GNU parallel. You could honestly teach a whole parallel computing course with it and never have to leave it for a real language. +-- Aubrey Bailey @DNAvinci@twitter - - - +Parallel is a life send, what an awesome piece of software. +-- Blaze9@reddit.com Y'all need some GNU parallel in your lives -- ChaKu @ChaiLovesChai@twitter diff --git a/doc/release_new_version b/doc/release_new_version index b9d3c325..d4f6f54d 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -207,9 +207,10 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20190222 ('indien pakistan Kashmir') released <<[stable]>> +Subject: GNU Parallel 20190322 (climatestrike FridayforFuture +Christchurch 'Max 8' fly i etiopien 737Max 8 indien pakistan Kashmir') released <<[stable]>> -GNU Parallel 20190222 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20190322 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> @@ -220,7 +221,7 @@ Quote of the month: New in this release: -* GNU Parallel now includes a cheat sheet: +* GNU Parallel now includes a cheat sheet: parallel_cheat.pdf https://calendar.colorado.edu/event/high_throughput_computing_on_rmacc_summit_and_beyond#.XH2NBhB7mV4 diff --git a/src/Makefile.am b/src/Makefile.am index 327d6e02..49c7287b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -288,4 +288,4 @@ EXTRA_DIST = parallel sem sql niceload parcat parset env_parallel \ env_parallel.tcsh env_parallel.zsh parcat.pod parset.pod \ sem.pod parallel.pod env_parallel.pod niceload.pod \ parallel_tutorial.pod parallel_book.pod parallel_design.pod \ - parallel_alternatives.pod $(DISTCLEANFILES) + parallel_alternatives.pod parallel_cheat.fodt $(DISTCLEANFILES) diff --git a/src/Makefile.in b/src/Makefile.in index 87f69c9b..cfdcf81a 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -275,7 +275,7 @@ EXTRA_DIST = parallel sem sql niceload parcat parset env_parallel \ env_parallel.tcsh env_parallel.zsh parcat.pod parset.pod \ sem.pod parallel.pod env_parallel.pod niceload.pod \ parallel_tutorial.pod parallel_book.pod parallel_design.pod \ - parallel_alternatives.pod $(DISTCLEANFILES) + parallel_alternatives.pod parallel_cheat.fodt $(DISTCLEANFILES) all: all-am diff --git a/src/env_parallel.ash b/src/env_parallel.ash index 3ad677e5..b842d7f9 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -186,6 +186,7 @@ env_parallel() { # myfunc is a shell function (in zsh) # which is /usr/bin/which (in sh, bash) # which is hashed (/usr/bin/which) + # gi is aliased to `grep -i' (in bash) # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' # Return 0 if found, 1 otherwise LANG=C type "$@" | diff --git a/src/env_parallel.bash b/src/env_parallel.bash index cd28d1e6..1accf187 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -173,11 +173,12 @@ env_parallel() { # type returns: # ll is an alias for ls -l (in ash) # bash is a tracked alias for /bin/bash - # true is a shell builtin + # true is a shell builtin (in bash) # myfunc is a function (in bash) # myfunc is a shell function (in zsh) - # which is /usr/bin/which + # which is /usr/bin/which (in sh, bash) # which is hashed (/usr/bin/which) + # gi is aliased to `grep -i' (in bash) # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' # Return 0 if found, 1 otherwise LANG=C type "$@" | diff --git a/src/env_parallel.dash b/src/env_parallel.dash index 0083b0b0..20a0f833 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -186,6 +186,7 @@ env_parallel() { # myfunc is a shell function (in zsh) # which is /usr/bin/which (in sh, bash) # which is hashed (/usr/bin/which) + # gi is aliased to `grep -i' (in bash) # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' # Return 0 if found, 1 otherwise LANG=C type "$@" | diff --git a/src/parallel b/src/parallel index 79161e76..5c21003f 100755 --- a/src/parallel +++ b/src/parallel @@ -2009,6 +2009,8 @@ sub open_joblog() { if($opt::resume || $opt::resume_failed || $opt::retry_failed) { if(open(my $joblog_fh, "<", $opt::joblog)) { # Read the joblog + # Override $/ with \n because -d might be set + local $/ = "\n"; # If there is a header: Open as append later $append = <$joblog_fh>; my $joblog_regexp; @@ -2017,22 +2019,19 @@ sub open_joblog() { # 4 host 1360490623.067 3.445 1023 1222 0 0 command $joblog_regexp='^(\d+)(?:\t[^\t]+){5}\t0\t0\t'; my @group; - { - local $/ = "\n"; - while(<$joblog_fh>) { - if(/$joblog_regexp/o) { - # This is 30% faster than set_job_already_run($1); - vec($Global::job_already_run,($1||0),1) = 1; - $Global::total_completed++; - $group[$1-1] = "true"; - } elsif(/(\d+)\s+\S+(\s+[-0-9.]+){6}\s+(.*)$/) { - # Grab out the command - $group[$1-1] = $3; - } else { - chomp; - ::error("Format of '$opt::joblog' is wrong: $_"); - ::wait_and_exit(255); - } + while(<$joblog_fh>) { + if(/$joblog_regexp/o) { + # This is 30% faster than set_job_already_run($1); + vec($Global::job_already_run,($1||0),1) = 1; + $Global::total_completed++; + $group[$1-1] = "true"; + } elsif(/(\d+)\s+\S+(\s+[-0-9.]+){6}\s+(.*)$/) { + # Grab out the command + $group[$1-1] = $3; + } else { + chomp; + ::error("Format of '$opt::joblog' is wrong: $_"); + ::wait_and_exit(255); } } if(@group) { @@ -2077,6 +2076,8 @@ sub open_joblog() { } close $joblog_fh; } + # $opt::null may be set if the commands contain \n + if($opt::null) { $/ = "\0"; } } if($opt::dryrun) { # Do not write to joblog in a dry-run @@ -2426,9 +2427,7 @@ sub shell_quote(@) { # @strings = strings to be quoted # Returns: # @shell_quoted_strings = string quoted as needed by the shell - return wantarray ? - (map { Q($_) } @_) - : (join" ",map { Q($_) } @_); + return wantarray ? (map { Q($_) } @_) : (join" ",map { Q($_) } @_); } sub shell_quote_scalar_rc($) { @@ -4190,7 +4189,7 @@ sub start_no_new_jobs() { ::status ("$Global::progname: SIGTERM received. No new jobs will be started.", "$Global::progname: Waiting for these ".(keys %Global::running). - " jobs to finish. Send SIGTERM again to stop now."); + " jobs to finish. Send SIGTERM to stop now."); list_running_jobs(); $Global::start_no_new_jobs ||= 1; } @@ -4569,11 +4568,14 @@ sub citation() { "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice + # https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt "This helps funding further development; AND IT WON'T COST YOU A CENT.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", "", "More about funding GNU Parallel and the citation notice:", + "https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html", "https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice", + "https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt", "", "If you send a copy of your published article to tange\@gnu.org, it will be", "mentioned in the release notes of next version of GNU Parallel.", @@ -4591,8 +4593,13 @@ sub citation() { close $fh; ::status( "", - "Thank you for your support. It is much appreciated. The citation", - "notice is now silenced.", + "Thank you for your support: You are the reason why there is funding to", + "continue maintaining GNU Parallel. On behalf of future versions of", + "GNU Parallel, which would not exist without your support:", + "", + " THANK YOU SO MUCH", + "", + "It is really appreciated. The citation notice is now silenced.", ""); } else { ::status( @@ -11166,8 +11173,8 @@ sub new($$) { my $fhs = shift; for my $fh (@$fhs) { if(-t $fh and -t ($Global::status_fd || *STDERR)) { - ::warning("Input is read from the terminal. You either know what you", - "are doing (in which case: YOU ARE AWESOME!) or you forgot", + ::warning("Input is read from the terminal. You are either an expert", + "(in which case: YOU ARE AWESOME!) or maybe you forgot", "::: or :::: or -a or to pipe data into parallel. If so", "consider going through the tutorial: man parallel_tutorial", "Press CTRL-D to exit."); diff --git a/src/parallel.pod b/src/parallel.pod index 4cb88f1e..b2bcdad7 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -30,22 +30,23 @@ STOP! Read the B below if you are new to GNU B. GNU B is a shell tool for executing jobs in parallel using -one or more computers. A job can be a single command or a small -script that has to be run for each of the lines in the input. The -typical input is a list of files, a list of hosts, a list of users, a -list of URLs, or a list of tables. A job can also be a command that -reads from a pipe. GNU B can then split the input into -blocks and pipe a block into each command in parallel. +one or more computers. A job can be a single command or a small script +that has to be run for each of the lines in the input. The typical +input is a list of files, a list of hosts, a list of users, a list of +URLs, or a list of tables. A job can also be a command that reads from +a pipe. GNU B can then split the input into blocks and pipe +a block into each command in parallel. -If you use xargs and tee today you will find GNU B very easy to -use as GNU B is written to have the same options as xargs. If -you write loops in shell, you will find GNU B may be able to -replace most of the loops and make them run faster by running several -jobs in parallel. +If you use xargs and tee today you will find GNU B very easy +to use as GNU B is written to have the same options as +xargs. If you write loops in shell, you will find GNU B may +be able to replace most of the loops and make them run faster by +running several jobs in parallel. -GNU B makes sure output from the commands is the same output as -you would get had you run the commands sequentially. This makes it -possible to use output from GNU B as input for other programs. +GNU B makes sure output from the commands is the same output +as you would get had you run the commands sequentially. This makes it +possible to use output from GNU B as input for other +programs. For each line of input GNU B will execute I with the line as arguments. If no I is given, the line of input is @@ -121,8 +122,8 @@ replacement string; then the string is not quoted. Input line without extension. This replacement string will be replaced by the input with the extension removed. If the input line contains -B<.> after the last B, the last B<.> until the end of the string will -be removed and B<{.}> will be replaced with the +B<.> after the last B, the last B<.> until the end of the string +will be removed and B<{.}> will be replaced with the remaining. E.g. I becomes I, I becomes I, I becomes I, I remains I. If the input line does not @@ -522,7 +523,7 @@ this will run 10*5 = 50 jobs in total: parallel --pipepart -a myfile --block -10 -j5 wc -This is an efficient alternative to B<--round-robin> because data is +This is an efficient alternative to B<--roundrobin> because data is never read by GNU B, but you can still have very few jobslots process a large amount of data. @@ -1892,14 +1893,14 @@ with B<--sshlogin>. =item B<--round> Normally B<--pipe> will give a single block to each instance of the -command. With B<--round-robin> all blocks will at random be written to +command. With B<--roundrobin> all blocks will at random be written to commands already running. This is useful if the command takes a long time to initialize. -B<--keep-order> will not work with B<--round-robin> as it is +B<--keep-order> will not work with B<--roundrobin> as it is impossible to track which input block corresponds to which output. -B<--round-robin> implies B<--pipe>, except if B<--pipepart> is given. +B<--roundrobin> implies B<--pipe>, except if B<--pipepart> is given. =item B<--rpl> 'I I' @@ -2741,6 +2742,17 @@ FUBAR in all files in this dir and subdirs: Note B<-q> is needed because of the space in 'FOO BAR'. +=head1 EXAMPLE: Simple network scanner + +B can generate IP-addresses from CIDR notation. With GNU +B you can build a simple network scanner to see which +addresses respond to B: + + prips 130.229.16.0/20 | \ + parallel --timeout 2 -j0 \ + 'ping -c 1 {} >/dev/null && echo {}' 2>/dev/null + + =head1 EXAMPLE: Reading arguments from command line GNU B can take the arguments from command line instead of @@ -3069,11 +3081,11 @@ B is an excellent tool to download videos. It can, however, not download videos in parallel. This takes a playlist and downloads 10 videos in parallel. - url='https://youtu.be/watch?v=0wOf2Fgi3DE&list=UU_cznB5YZZmvAmeq7Y3EriQ' + url='youtu.be/watch?v=0wOf2Fgi3DE&list=UU_cznB5YZZmvAmeq7Y3EriQ' export url - youtube-dl --flat-playlist "$url" | + youtube-dl --flat-playlist "https://$url" | parallel --tagstring {#} --lb -j10 \ - youtube-dl --playlist-start {#} --playlist-end {#} '"$url"' + youtube-dl --playlist-start {#} --playlist-end {#} '"https://$url"' =head1 EXAMPLE: Prepend last modified date (ISO8601) to file name @@ -3312,7 +3324,7 @@ repeat the list and use replacement string B<{choose_k}>: parallel --plus echo {choose_k} ::: A B C D ::: A B C D - parallel --plus echo arg2 {2choose_k} arg1 {1choose_k} ::: A B C D ::: A B C D + parallel --plus echo 2{2choose_k} 1{1choose_k} ::: A B C D ::: A B C D B<{choose_k}> works for any number of input sources: @@ -3342,7 +3354,8 @@ If the input is in the file in.txt: If the input is in the array $a here are two solutions: - seq $((${#a[@]}-1)) | env_parallel --env a echo '${a[{=$_--=}]} - ${a[{}]}' + seq $((${#a[@]}-1)) | \ + env_parallel --env a echo '${a[{=$_--=}]} - ${a[{}]}' parallel echo {1} - {2} ::: "${a[@]::${#a[@]}-1}" :::+ "${a[@]:1}" @@ -3370,7 +3383,7 @@ significant. Compare the speed of these: If your program cannot take multiple arguments, then you can use GNU B to spawn multiple GNU Bs: - seq -w 0 9999999 | + seq -w 0 9999999 | \ parallel -j10 -q -I,, --pipe parallel -j0 touch pict{}.jpg If B<-j0> normally spawns 252 jobs, then the above will try to spawn @@ -3465,7 +3478,8 @@ Check the uptime of the servers in I<~/.parallel/sshloginfile>: Give each job a new color. Most terminals support ANSI colors with the escape code "\033[30;3Xm" where 0 <= X <= 7: - parallel --tagstring '\033[30;3{=$_=++$::color%8=}m' seq {} ::: {1..10} + seq 10 | \ + parallel --tagstring '\033[30;3{=$_=++$::color%8=}m' seq {} parallel --rpl '{color} $_="\033[30;3".(++$::color%8)."m"' \ --tagstring {color} seq {} ::: {1..10} @@ -3587,7 +3601,7 @@ is much faster. If it still does not fit in memory you can do this: - parallel --pipepart -a regexps.txt --block 1M grep -Ff - -n bigfile | + parallel --pipepart -a regexps.txt --block 1M grep -Ff - -n bigfile | \ sort -un | perl -pe 's/^\d+://' The 1M should be your free memory divided by the number of CPU threads and @@ -3599,7 +3613,7 @@ GNU/Linux you can do: percpu=$((free / 200 / $(parallel --number-of-threads)))k parallel --pipepart -a regexps.txt --block $percpu --compress \ - grep -F -f - -n bigfile | + grep -F -f - -n bigfile | \ sort -un | perl -pe 's/^\d+://' If you can live with duplicated lines and wrong order, it is faster to do: @@ -3612,8 +3626,8 @@ If you can live with duplicated lines and wrong order, it is faster to do: If the CPU is the limiting factor parallelization should be done on the regexps: - cat regexp.txt | parallel --pipe -L1000 --round-robin --compress \ - grep -f - -n bigfile | + cat regexp.txt | parallel --pipe -L1000 --roundrobin --compress \ + grep -f - -n bigfile | \ sort -un | perl -pe 's/^\d+://' The command will start one B per CPU and read I one @@ -3633,7 +3647,7 @@ these chunks. To parallelize both reading of I and I combine the two using B<--fifo>: parallel --pipepart --block 100M -a bigfile --fifo cat regexp.txt \ - \| parallel --pipe -L1000 --round-robin grep -f - {} + \| parallel --pipe -L1000 --roundrobin grep -f - {} If a line matches multiple regexps, the line may be duplicated. @@ -3672,7 +3686,7 @@ To run B on B: To run commands on more than one remote computer run: - seq 10 | parallel --sshlogin server.example.com,server2.example.net echo + seq 10 | parallel --sshlogin s1.example.com,s2.example.net echo Or: @@ -3985,7 +3999,8 @@ is even faster: DBURL=pg://user:pass@host/mydb DBTABLEURL=$DBURL/mytable parallel --sqlandworker $DBTABLEURL seq ::: {1..10} - sql $DBURL "COPY (SELECT * FROM mytable) TO stdout DELIMITER ',' CSV HEADER;" + sql $DBURL \ + "COPY (SELECT * FROM mytable) TO stdout DELIMITER ',' CSV HEADER;" Or MySQL: @@ -4015,11 +4030,11 @@ can read into R or LibreCalc, then you can use B<--results>: The show Aflyttet on Radio 24syv publishes an RSS feed with their audio podcasts on: http://arkiv.radio24syv.dk/audiopodcast/channel/4466232 -Using B you can extract the URLs for 2016 and download them +Using B you can extract the URLs for 2019 and download them using GNU B: - wget -O - http://arkiv.radio24syv.dk/audiopodcast/channel/4466232 | - xpath -e "//ancestor::pubDate[contains(text(),'2016')]/../enclosure/@url" | + wget -O - http://arkiv.radio24syv.dk/audiopodcast/channel/4466232 | \ + xpath -e "//pubDate[contains(text(),'2019')]/../enclosure/@url" | \ parallel -u wget '{= s/ url="//; s/"//; =}' @@ -4124,7 +4139,8 @@ separator before each customer (column 2 = $F[1]). Here we first make a 50 character random string, which we then use as the separator: sep=`perl -e 'print map { ("a".."z","A".."Z")[rand(52)] } (1..50);'` - cat my.csv | perl -ape '$F[1] ne $l and print "'$sep'"; $l = $F[1]' | + cat my.csv | \ + perl -ape '$F[1] ne $l and print "'$sep'"; $l = $F[1]' | \ parallel --recend $sep --rrs --pipe -N1 wc If your program can process multiple customers replace B<-N1> with a @@ -4141,13 +4157,13 @@ process. That means that if you just spawn more GNU Bs then each of them can run 250 jobs. This will spawn up to 2500 jobs: cat myinput |\ - parallel --pipe -N 50 --round-robin -j50 parallel -j50 your_prg + parallel --pipe -N 50 --roundrobin -j50 parallel -j50 your_prg This will spawn up to 62500 jobs (use with caution - you need 64 GB RAM to do this, and you may need to increase /proc/sys/kernel/pid_max): cat myinput |\ - parallel --pipe -N 250 --round-robin -j250 parallel -j250 your_prg + parallel --pipe -N 250 --roundrobin -j250 parallel -j250 your_prg =head1 EXAMPLE: Working as mutex and counting semaphore diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 832902f8..56b521ad 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -2050,7 +2050,8 @@ be done by the user. Stdout (standard output) is prepended with o:. Stderr (standard error) is sendt to stdout (standard output) and prepended with e:. -For short jobs with little output B is 20% faster than GNU B. +For short jobs with little output B is 20% faster than GNU +B and 60% slower than B. http://savannah.nongnu.org/projects/par (Last checked: 2019-02) @@ -2060,13 +2061,131 @@ http://savannah.nongnu.org/projects/par (Last checked: 2019-02) B does not support composed commands, so commands must be wrapped in B. -It buffers output in RAM. +It buffers output in RAM. It only takes file names from the filesystem as input (similar to B). https://github.com/sharkdp/fd (Last checked: 2019-02) +=head2 DIFFERENCES BETWEEN lateral AND GNU Parallel + +B is very similar to B: It takes a single command and +runs it in the background. The design means that output from parallel +running jobs may mix. If it dies unexpectly it leaves a socket in +~/.lateral/socket.PID. + +B deals badly with too long command lines. This makes the +B server crash: + + lateral run echo `seq 100000| head -c 1000k` + +Any options will be read by B so this does not work +(B interprets the B<-l>): + + lateral run ls -l + +Composed commands do not work: + + lateral run pwd ';' ls + +Functions do not work: + + myfunc() { echo a; } + export -f myfunc + lateral run myfunc + +Running B in the terminal causes the parent shell to die: + + echo '#!/bin/bash' > mycmd + echo emacs -nw >> mycmd + chmod +x mycmd + lateral start + lateral run ./mycmd + +Here are the examples from https://github.com/akramer/lateral with the +corresponding GNU B and GNU B commands: + + 1$ lateral start + 1$ for i in $(cat /tmp/names); do + 1$ lateral run -- some_command $i + 1$ done + 1$ lateral wait + 1$ + 1$ for i in $(cat /tmp/names); do + 1$ sem some_command $i + 1$ done + 1$ sem --wait + 1$ + 1$ parallel some_command :::: /tmp/names + + 2$ lateral start + 2$ for i in $(seq 1 100); do + 2$ lateral run -- my_slow_command < workfile$i > /tmp/logfile$i + 2$ done + 2$ lateral wait + 2$ + 2$ for i in $(seq 1 100); do + 2$ sem my_slow_command < workfile$i > /tmp/logfile$i + 2$ done + 2$ sem --wait + 2$ + 2$ parallel 'my_slow_command < workfile{} > /tmp/logfile{}' \ + ::: {1..100} + + 3$ lateral start -p 0 # yup, it will just queue tasks + 3$ for i in $(seq 1 100); do + 3$ lateral run -- command_still_outputs_but_wont_spam inputfile$i + 3$ done + 3$ lateral config -p 10; lateral wait # command output spam can commence + 3$ + 3$ for i in $(seq 1 100); do + 3$ echo "command inputfile$i" >> joblist + 3$ done + 3$ parallel -j 10 :::: joblist + 3$ + 3$ echo 1 > /tmp/njobs + 3$ parallel -j /tmp/njobs command inputfile{} \ + ::: {1..100} & + 3$ echo 10 >/tmp/njobs + 3$ wait + +https://github.com/akramer/lateral (Last checked: 2019-03) + + +=head2 DIFFERENCES BETWEEN with-this AND GNU Parallel + +The examples from https://github.com/amritb/with-this.git and the +corresponding GNU B command: + + with -v "$(cat myurls.txt)" "curl -L this" + parallel curl -L ::: myurls.txt + + with -v "$(cat myregions.txt)" \ + "aws --region=this ec2 describe-instance-status" + parallel aws --region={} ec2 describe-instance-status \ + :::: myregions.txt + + with -v "$(ls)" "kubectl --kubeconfig=this get pods" + ls | parallel kubectl --kubeconfig={} get pods + + with -v "$(ls | grep config)" "kubectl --kubeconfig=this get pods" + ls | grep config | parallel kubectl --kubeconfig={} get pods + + with -v "$(echo {1..10})" "echo 123" + parallel -N0 echo 123 ::: {1..10} + +Stderr is merged with stdout. B buffers in RAM. It uses 3x +the output size, so you cannot have output larger than 1/3rd the +amount of RAM. The input values cannot contain spaces. Composed +commands do not work. + +B gives some additional information, so the output has to +be cleaned before piping it to the next command. + +https://github.com/amritb/with-this.git (Last checked: 2019-03) + + =head2 Todo Url for spread @@ -2079,8 +2198,6 @@ https://github.com/bahmutov/with-package https://github.com/xuchenCN/go-pssh -https://github.com/amritb/with-this.git - https://github.com/flesler/parallel https://github.com/Julian/Verge diff --git a/src/parallel_cheat.fodt b/src/parallel_cheat.fodt index fab12cce..fb3e80f5 100644 --- a/src/parallel_cheat.fodt +++ b/src/parallel_cheat.fodt @@ -1,24 +1,24 @@ - 2019-02-27T22:13:36.7819247182019-03-04T21:52:31.122787645P4DT21H57M39S25LibreOffice/6.0.6.2$Linux_X86_64 LibreOffice_project/00m0$Build-22019-03-04T02:37:46.666605685 + 2019-02-27T22:13:36.7819247182019-03-12T01:40:23.357999303P11DT5H49M15S40LibreOffice/6.0.6.2$Linux_X86_64 LibreOffice_project/00m0$Build-22019-03-08T23:06:56.779293422 - 471 + 15875 0 22287 - 10515 + 11042 true false view2 - 8511 - 2536 + 18724 + 25474 0 - 471 + 15875 22285 - 10984 + 26915 0 1 false @@ -41,7 +41,7 @@ true true false - lwH+/1hlcm94X1BoYXNlcl82MTQwRE4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ1VQUzpYZXJveF9QaGFzZXJfNjE0MEROAAAAAAAAAAAWAAMAuAAAAAAAAAAEAAhSAAAEdAAASm9iRGF0YSAxCnByaW50ZXI9WGVyb3hfUGhhc2VyXzYxNDBETgpvcmllbnRhdGlvbj1Qb3J0cmFpdApjb3BpZXM9MQpjb2xsYXRlPWZhbHNlCm1hcmdpbmRhanVzdG1lbnQ9MCwwLDAsMApjb2xvcmRlcHRoPTI0CnBzbGV2ZWw9MApwZGZkZXZpY2U9MQpjb2xvcmRldmljZT0wClBQRENvbnRleERhdGEKUGFnZVNpemU6QTQAABIAQ09NUEFUX0RVUExFWF9NT0RFDwBEdXBsZXhNb2RlOjpPZmY= + uAH+/1hlcm94X1BoYXNlcl82MTQwRE4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ1VQUzpYZXJveF9QaGFzZXJfNjE0MEROAAAAAAAAAAAWAAMA2QAAAAAAAAAEAAhSAAAEdAAASm9iRGF0YSAxCnByaW50ZXI9WGVyb3hfUGhhc2VyXzYxNDBETgpvcmllbnRhdGlvbj1Qb3J0cmFpdApjb3BpZXM9MQpjb2xsYXRlPWZhbHNlCm1hcmdpbmRhanVzdG1lbnQ9MCwwLDAsMApjb2xvcmRlcHRoPTI0CnBzbGV2ZWw9MApwZGZkZXZpY2U9MQpjb2xvcmRldmljZT0wClBQRENvbnRleERhdGEKRHVwbGV4Ok5vbmUAUGFnZVNpemU6QTQASW5wdXRTbG90OkF1dG9TZWxlY3QAABIAQ09NUEFUX0RVUExFWF9NT0RFDwBEdXBsZXhNb2RlOjpPZmY= true false @@ -92,7 +92,7 @@ false false true - 1506730 + 1662577 true false false @@ -366,10 +366,10 @@ - + - + @@ -611,30 +611,41 @@ - - + + + + + + - + - + - + - + + + + - + + + + + @@ -642,56 +653,83 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -720,12 +758,12 @@ - GNU Parallel Cheat Sheet + GNU Parallel Cheat Sheet - GNU Parallel is a replacement for xargs, and for loops. It can also split a file or a stream into blocks and pass those to commands running in parallel. + GNU Parallel is a replacement for xargs and for loops. It can also split a file or a stream into blocks and pass those to commands running in parallel. @@ -734,22 +772,22 @@ - Examples + Examples - Compress all *.html files in parallel – 2 per CPU threadparallel --jobs 200% gzip ::: *.html + Compress all *.html files in parallel – 2 jobs per CPU thread in parallelparallel --jobs 200% gzip ::: *.html - Convert all *.wav to *.mp3 using lameparallel lame {} -o {.}.mp3 ::: *.wav + Convert all *.wav to *.mp3 using lame – 1 job per CPU thread in parallel (default)parallel lame {} -o {.}.mp3 ::: *.wav - Chop bigfile into 1MB blocks and grep for the string foobarcat bigfile | parallel --pipe grep foobar + Chop bigfile into 1MB blocks and grep for the string foobarcat bigfile | parallel --pipe grep foobar @@ -763,32 +801,32 @@ - parallel echo ::: cmd line input source + parallel echo ::: cmd line input source - cat input_from_stdin | parallel echo + cat input_from_stdin | parallel echo - parallel echo ::: multiple input ::: source values + parallel echo ::: multiple input sources ::: with values - parallel -a input_from_file echo + parallel -a input_from_file echo - parallel echo :::: input_from_file + parallel echo :::: input_from_file - parallel echo :::: input_from_file ::: and command line + parallel echo :::: input_from_file ::: and command line @@ -801,7 +839,7 @@ Replacement string - Value + Value if input is mydir/mysubdir/myfile.myext @@ -822,10 +860,10 @@ - {/}, {//}, {/.} + {/}, {//}, {/.} - myfile.myext, mydir/mysubdir, myfile + myfile.myext, mydir/mysubdir, myfile @@ -833,7 +871,7 @@ {#} - The sequence number of the job + The sequence number of the job @@ -841,7 +879,7 @@ {%} - The job slot number + The job slot number @@ -849,20 +887,20 @@ {2} - Value from the second input source + Value from the second input source - {2.} {2/} {2//} {2/.} + {2.} {2/} {2//} {2/.} - Combination of {2} and {.} {/} {//} {/.} + Combination of {2} and {.} {/} {//} {/.} - {= perl expression =} + {= perl expression =} Change $_ with perl expression @@ -874,12 +912,12 @@ - Control the output – keep the order the same as the input + Control the output – keep the same order as the input, prepend with input value - parallel --keep-order "sleep {}; echo {}" ::: 5 4 3 2 1 + parallel --keep-order --tag "sleep {}; echo {}" ::: 5 4 3 2 1 @@ -893,14 +931,14 @@ - Run 2 jobs in parallel - parallel --jobs 2 "sleep {}; echo {}" ::: 5 4 3 2 1 + Run 2 jobs in parallel – command is a composed command + parallel --jobs 2 "sleep {}; echo {}" ::: 5 4 3 2 1 - See what will be run - parallel --dryrun echo ::: Red Green Blue ::: S M L + See what will be run + parallel --dryrun echo {2} {1} ::: bird flower fish ::: Red Green Blue @@ -909,12 +947,12 @@ - Remote execution + Remote execution - parallel -S server1 -S server2 "hostname; echo {}" ::: foo bar + parallel -S server1 -S server2 "hostname; echo {}" ::: foo bar @@ -923,17 +961,17 @@ - Pipe mode + Pipe mode - cat bigfile | parallel --pipe wc -l + cat bigfile | parallel --pipe wc -l - Chop bigfile into one block per CPU thread and grep for foobarparallel -a bigfile --pipepart --block -1 grep foobar + Chop bigfile into one block per CPU thread and grep for foobarparallel -a bigfile --pipepart --block -1 grep foobar @@ -942,23 +980,21 @@ - Read more – Your command line will love you for it + Read more – Your command line will love you for it - parallel --help; man parallel; man parallel_tutorial + parallel --help; man parallel; man parallel_tutorial; www.pi.dk/1 - GNU Parallel 2018 https://doi.org/10.5281/zenodo.1146014 + GNU Parallel 2018 https://doi.org/10.5281/zenodo.1146014 - - - + (CC-By-SA) 2019-03-11 Ole Tange diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index eaa46e4a..07efabd7 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -2395,13 +2395,13 @@ Output (the order may be different): GNU B treats each line as a record. If the order of records is unimportant (e.g. you need all lines processed, but you do not care -which is processed first), then you can use B<--round-robin>. Without -B<--round-robin> GNU B will start a command per block; with -B<--round-robin> only the requested number of jobs will be started +which is processed first), then you can use B<--roundrobin>. Without +B<--roundrobin> GNU B will start a command per block; with +B<--roundrobin> only the requested number of jobs will be started (B<--jobs>). The records will then be distributed between the running jobs: - cat num1000000 | parallel --pipe -j4 --round-robin wc + cat num1000000 | parallel --pipe -j4 --roundrobin wc Output will be similar to: diff --git a/src/parcat.pod b/src/parcat.pod index 7718306c..f2939b64 100644 --- a/src/parcat.pod +++ b/src/parcat.pod @@ -44,7 +44,7 @@ the files. =head2 Simple line buffered output -GNU Parallel saves output to tempfiles. If the amount of data is +GNU B saves output to tempfiles. If the amount of data is bigger than the free disk space, then you can use this technique to do line buffering without saving to disk: diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 0dfdc8c1..6fc3c830 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -941,6 +941,11 @@ par_shard() { perl -pe 's/(.*\d{5,}){3}/OK/' } +par_fish() { + echo '### https://github.com/fish-shell/fish-shell/issues/5582' + echo OK | stdout fish -c 'parallel --pipe cat' +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | LC_ALL=C sort | parallel --timeout 30 -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 64bea043..f095d738 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -2232,6 +2232,19 @@ CPU revision : 4 :::+ "2-8-8-8" "1-4-8-4" "1-2-4-2" "1-2-2-2" "2-24-48-24" "1-2-2-2" "1-8-8-8" } +par_null_resume() { + echo '### --null --resume --jl' + log=/tmp/null-resume-$$.log + + true > "$log" + printf "%s\n" a b c | parallel --resume -k --jl $log echo + printf "%s\n" a b c | parallel --resume -k --jl $log echo + true > "$log" + printf "%s\0" A B C | parallel --null --resume -k --jl $log echo + printf "%s\0" A B C | parallel --null --resume -k --jl $log echo + rm "$log" +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | LC_ALL=C sort | parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 6db0b890..1842b5c1 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -4,11 +4,11 @@ # Each should be taking 30-100s and be possible to run in parallel # I.e.: No race conditions, no logins -par_sigterm() { - echo '### Test SIGTERM' +par_sighup() { + echo '### Test SIGHUP' parallel -k -j5 sleep 15';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 & A=$! - sleep 29; kill -TERM $A + sleep 29; kill -HUP $A wait LC_ALL=C sort /tmp/parallel$$ rm /tmp/parallel$$ diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index 9b7adce3..047a948e 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -46,8 +46,8 @@ par_slow_args_generation() { seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo } -par_kill_term_twice() { - echo '### Are children killed if GNU Parallel receives TERM twice? There should be no sleep at the end' +par_kill_term() { + echo '### Are children killed if GNU Parallel receives TERM? There should be no sleep at the end' parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 & T=$! @@ -56,8 +56,17 @@ par_kill_term_twice() { kill -TERM $T sleep 1 pstree $$ - kill -TERM $T - sleep 1 +} + +par_kill_hup() { + echo '### Are children killed if GNU Parallel receives HUP? There should be no sleep at the end' + + parallel -j 2 -q bash -c 'sleep {} & pid=$!; wait $pid' ::: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 & + T=$! + sleep 2 + pstree $$ + kill -HUP $T + sleep 2 pstree $$ } diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 42f6c520..5ee9e0bb 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -412,6 +412,8 @@ par_file_ending_in_newline ### Hans found a bug giving unitialized variable par_file_ending_in_newline gzip /tmp/parallel_f1 par_file_ending_in_newline gzip '/tmp/parallel_f2 par_file_ending_in_newline ' +par_fish ### https://github.com/fish-shell/fish-shell/issues/5582 +par_fish OK par_halt_on_error_division_by_zero ### --halt-on-error soon,fail=100% with no input should not give division by zero par_halt_on_error_division_by_zero 0 par_halt_one_job # Halt soon if there is a single job diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index 817eb3dc..08e23620 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -409,7 +409,7 @@ par_test_build_and_install rm /tmp/parallel-install/bin/sem || true par_test_build_and_install ln -s parallel /tmp/parallel-install/bin/sem par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000/src' par_test_build_and_install /bin/mkdir -p '/tmp/parallel-install/share/doc/parallel' -par_test_build_and_install /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_book.html parallel_design.html parallel_alternatives.html parcat.html parset.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_book.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_book.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf '/tmp/parallel-install/share/doc/parallel' +par_test_build_and_install /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_book.html parallel_design.html parallel_alternatives.html parcat.html parset.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_book.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_book.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf parallel_cheat.pdf '/tmp/parallel-install/share/doc/parallel' par_test_build_and_install /bin/mkdir -p '/tmp/parallel-install/share/man/man1' par_test_build_and_install /usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parcat.1 parset.1 '/tmp/parallel-install/share/man/man1' par_test_build_and_install /bin/mkdir -p '/tmp/parallel-install/share/man/man7' @@ -823,7 +823,7 @@ par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parset par_test_build_and_install /bin/bash: pod2pdf: command not found par_test_build_and_install Warning: pod2pdf not found. Using old parset.pdf par_test_build_and_install /bin/mkdir -p '/tmp/parallel-install/share/doc/parallel' -par_test_build_and_install /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_book.html parallel_design.html parallel_alternatives.html parcat.html parset.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_book.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_book.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf '/tmp/parallel-install/share/doc/parallel' +par_test_build_and_install /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_book.html parallel_design.html parallel_alternatives.html parcat.html parset.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_book.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_book.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf parallel_cheat.pdf '/tmp/parallel-install/share/doc/parallel' par_test_build_and_install pod2man --release='00000000' --center='parallel' \ par_test_build_and_install --section=1 ./parallel.pod > ./parallel.1n \ par_test_build_and_install && mv ./parallel.1n ./parallel.1 \ diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 39c74a3d..bafc765a 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -27,8 +27,8 @@ par_interactive sleep 0.1; echo opt-p 2 ?...n par_interactive sleep 0.1; echo opt-p 3 ?...y par_interactive spawn /tmp/parallel-script-for-expect par_k ### Test -k -par_k parallel: Warning: Only enough file handles to run 9 jobs in parallel. -par_k parallel: Warning: Running 'parallel -j0 -N 9 --pipe parallel -j0' or +par_k parallel: Warning: Only enough file handles to run 8 jobs in parallel. +par_k parallel: Warning: Running 'parallel -j0 -N 8 --pipe parallel -j0' or par_k parallel: Warning: raising 'ulimit -n' or 'nofile' in /etc/security/limits.conf par_k parallel: Warning: or /proc/sys/fs/file-max may help. par_k begin diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index de83dd90..24b5d44e 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -172,6 +172,13 @@ par_incomplete_linebuffer 7 par_incomplete_linebuffer 8 par_incomplete_linebuffer 9 par_incomplete_linebuffer 10 +par_null_resume ### --null --resume --jl +par_null_resume a +par_null_resume b +par_null_resume c +par_null_resume A +par_null_resume B +par_null_resume C par_parset ### test parset par_parset Put output into $myarray par_parset 10 diff --git a/testsuite/wanted-results/parallel-local12 b/testsuite/wanted-results/parallel-local12 index 2c15ca0e..056b6c73 100644 --- a/testsuite/wanted-results/parallel-local12 +++ b/testsuite/wanted-results/parallel-local12 @@ -60,7 +60,9 @@ This helps funding further development; AND IT WON'T COST YOU A CENT. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. More about funding GNU Parallel and the citation notice: +https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice +https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt If you send a copy of your published article to tange@gnu.org, it will be mentioned in the release notes of next version of GNU Parallel.