Passes testsuite.

This commit is contained in:
Ole Tange 2016-07-22 18:10:28 +02:00
parent 805d924c16
commit 71ba3f1a8f
22 changed files with 128 additions and 115 deletions

View file

@ -221,28 +221,17 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
Subject: GNU Parallel 20160622 ('Orlando') released <<[stable]>>
GNU Parallel 20160622 ('Orlando') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
GNU Parallel 20160722 ('Brexit') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>>
Haiku of the month:
Does path on remote
not include GNU Parallel?
Try --env PATH.
-- Ole Tange
New in this release:
* $PATH can now be exported using --env PATH. Useful if GNU Parallel is not in your path on remote machines.
* If --block is left out, --pipepart will use a block size that will result in 10 jobs per jobslot.
* The cookie from 2016-01-04 was won by Morgan Rodgers on the 2016-06-06 after 5 months.
* Vote for GNU Parallel's community ad on https://meta.askubuntu.com/questions/14925/community-promotion-ads-2016/15046#15046
* <<Possibly http://link.springer.com/chapter/10.1007%2F978-3-319-22053-6_46>>
* <<Possibly http://link.springer.com/article/10.1007/s12021-015-9290-5>>
@ -269,22 +258,6 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
* <<link No citation: Next-generation TCP for ns-3 simulator http://www.sciencedirect.com/science/article/pii/S1569190X15300939>>
* GNU Parallel was cited in: Genotypic Resistance Tests Sequences Reveal the Role of Marginalized Populations in HIV-1 Transmission in Switzerland http://www.nature.com/articles/srep27580
* GNU Parallel was cited in: A Deep-Learning Approach for Operation of an Automated Realtime Flare Forecast https://arxiv.org/pdf/1606.01587.pdf
* Unix parallel: Populating all the USB sticks http://www.markhneedham.com/blog/2016/06/01/unix-parallel-populating-all-the-usb-sticks/
* GNU Parallel is a fantastic utility https://www.dray.be/parallel/
* Combining a large number of VCF files https://gtamazian.com/2016/06/16/combining-a-large-number-of-vcf-files/
* Useful Scraping Techniques http://blog.apps.npr.org/2016/06/17/scraping-tips.html
* Tools of the trade http://onox.com.br/2015/05/tools-of-the-trade/#more-198
* Armslist scraper https://github.com/nprapps/armslist-scraper/
* Bug fixes and man page updates.
GNU Parallel - For people who live life in the parallel lane.

View file

@ -596,7 +596,7 @@ sub parse_options {
"noheaders|no-headers|n" => \$::opt_n,
"r" => \$::opt_retry,
"retries=s" => \$::opt_retries,
"debug" => \$::opt_debug,
"debug|D" => \$::opt_debug,
# Shebang #!/usr/bin/parallel -Yotheroptions
"Y|shebang" => \$::opt_shebang,
"skip-first-line" => \$::opt_skip_first_line,
@ -1010,9 +1010,10 @@ sub parse_dburl {
)?
(?:
/
([^?]*)? # Database ($6)
([^?/]*)? # Database ($6)
)?
(?:
/?
\?
(.*)? # Query ($7)
)?
@ -1022,7 +1023,8 @@ sub parse_dburl {
$options{password} = undef_if_empty(uri_unescape($3));
$options{host} = undef_if_empty(uri_unescape($4));
$options{port} = undef_if_empty(uri_unescape($5));
$options{database} = undef_if_empty(uri_unescape($6));
$options{database} = undef_if_empty(uri_unescape($6))
|| $options{user};
$options{query} = undef_if_empty(uri_unescape($7));
debug("dburl $url\n");
debug("databasedriver ",$options{databasedriver}, " user ", $options{user},

View file

@ -8,7 +8,16 @@ sudo aptitude install imagemagick expect autossh
# DEBIAN package
sudo aptitude install dpkg-dev build-essential debhelper
# SHEBANG TOOLS
sudo aptitude install gnuplot octave ruby r-base-core
# SQL TOOLS
sudo aptitude install libdbd-pg-perl libdbd-sqlite3-perl
# Compression
sudo aptitude install pxz pixz
# To configure zsh
echo Configure zsh and exit
zsh
# DATABASES
sudo aptitude install postgresql mysql-server
@ -37,6 +46,7 @@ create_shell_user() {
sshpass -e ssh-copy-id $shell@lo &&
echo "ssh-keyscan" &&
ssh $shell@lo 'ssh-keyscan -t rsa lo >> .ssh/known_hosts' &&
ssh $shell@lo 'ssh-keyscan -t rsa server >> .ssh/known_hosts' &&
echo "chsh" &&
sudo chsh -s $(which $shell || which ${shell#"nopath"}) $shell &&
echo | ssh -t $shell@lo ssh-keygen &&
@ -62,7 +72,7 @@ ssh_copy_id() {
export -f ssh_copy_id
shellsplus="parallel $shells"
parallel --bar -j200% --timeout 3 --retries 10 --tag ssh_copy_id {1}@lo {2}@lo ::: $shellsplus ::: $shellsplus
parallel -j10 ssh {1}@lo ssh {2}@lo echo {} OK ::: $shellsplus ::: $shellsplus
parallel -j10 --retries 3 --tag ssh {1}@lo ssh {2}@lo echo OK ::: $shellsplus ::: $shellsplus
# change paths to no path
(
@ -86,12 +96,9 @@ lsh-keygen | lsh-writekey -c none
lsh-export-key --openssh < ~/.lsh/identity.pub | lsh localhost 'cat >>.ssh/authorized_keys'
lsh-export-key --openssh < ~/.lsh/identity.pub | ssh csh@localhost 'cat >>.ssh/authorized_keys'
# To configure zsh
zsh
parallel --record-env
# SHEBANG TOOLS
sudo aptitude install gnuplot octave ruby r-base-core libdbd-pg-perl libdbd-sqlite3-perl pxz pixz
sudo cp /usr/bin/tmux /usr/local/bin/tmux1.8

View file

@ -26,6 +26,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
' |
stdout parallel --joblog /tmp/jl-`basename $0` -j6 -vd'\n\n' |
perl -pe '$|=1;
# --pipe --roundrobin wc
s: \d{6} \d{6} \d{7}: 999999 999999 9999999:;
# --tmux
s:(/tmp\S+)(tms).....:$1$2XXXXX:;
# --files

View file

@ -9,7 +9,7 @@ sql $MYSQL_ADMIN_DBURL CREATE DATABASE sqlunittest;
sql $MYSQL_ADMIN_DBURL "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';"
sql $MYSQL_ADMIN_DBURL "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';"
MYSQL_TEST_DBURL=mysql://sqlunittest:CB5A1FFFA5A@
MYSQL_TEST_DBURL=mysql://sqlunittest:CB5A1FFFA5A@/
echo '### Test of #! -Y with file as input'
cat >/tmp/shebang <<EOF

View file

@ -9,7 +9,7 @@ sql $MYSQL_ADMIN_DBURL CREATE DATABASE sqlunittest;
sql $MYSQL_ADMIN_DBURL "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';"
sql $MYSQL_ADMIN_DBURL "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';"
MYSQL_TEST_DBURL=mysql://sqlunittest:CB5A1FFFA5A@
MYSQL_TEST_DBURL=mysql://sqlunittest:CB5A1FFFA5A@/sqlunittest
echo '### Test reading sql from url command line'
echo | sql "$MYSQL_TEST_DBURL/?SELECT 'Yes it works' as 'Test reading SQL from command line';"
@ -30,7 +30,7 @@ perl -i -ne '$seen{$_}++ || print' ~/.sql/aliases
stdout sql ":cyclic3?SELECT 'NO IT DID NOT' as 'Test if :cyclic is found works';"
echo "### Test alias with statement .sql/aliases"
echo ":testselect sqlite:////tmp/file.sqlite?SELECT 'It works' AS 'Test statement in alias';" >> ~/.sql/aliases
echo ":testselect sqlite:///%2Ftmp%2Ffile.sqlite?SELECT 'It works' AS 'Test statement in alias';" >> ~/.sql/aliases
perl -i -ne '$seen{$_}++ || print' ~/.sql/aliases
echo | stdout sql :testselect
echo ":testselectmysql mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest?SELECT 'It works' AS 'Test statement in alias';" >> ~/.sql/aliases

View file

@ -2,19 +2,18 @@
#cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | nice timeout -k 1 40 parallel -j0 -k -L1
echo '### Test of --retries - it should run 13 jobs in total';
seq 0 12 | stdout parallel --progress -kj100% --retries 1 -S 12/localhost,1/:,parallel@parallel-server1 -vq \
perl -e 'sleep 1;print "job{}\n";exit({})' |
perl -ne 'BEGIN{$/="\r";} @a=(split /\//,$_); END{print $a[1]+$a[4]+$a[7],"\n"}'
seq 0 12 | stdout parallel --retries 1 -S 12/localhost,1/:,parallel@parallel-server1 -uq \
perl -e 'print "job{}\n";exit({})' | wc -l
echo '### Test of --retries - it should run 25 jobs in total';
seq 0 12 | stdout parallel --progress -kj100% --retries 2 -S 12/localhost,1/:,parallel@parallel-server1 -vq \
perl -e 'sleep 1;print "job{}\n";exit({})' |
perl -ne 'BEGIN{$/="\r";} @a=(split /\//,$_); END{print $a[1]+$a[4]+$a[7],"\n"}'
seq 0 12 | stdout parallel --retries 2 -S 12/localhost,1/:,parallel@parallel-server1 -uq \
perl -e 'print "job{}\n";exit({})' | wc -l
echo '### Test of --retries - it should run 49 jobs in total';
seq 0 12 | stdout parallel --progress -kj100% --retries 4 -S 12/localhost,1/:,parallel@parallel-server1 -vq \
perl -e 'sleep 1;print "job{}\n";exit({})' |
perl -ne 'BEGIN{$/="\r";} @a=(split /\//,$_); END{print $a[1]+$a[4]+$a[7],"\n"}'
seq 0 12 | stdout parallel --retries 4 -S 12/localhost,1/:,parallel@parallel-server1 -uq \
perl -e 'print "job{}\n";exit({})' | wc -l
#EOF
echo '### Bug with --retries'
seq 1 8 | parallel --retries 2 --sshlogin 8/localhost,8/: -j+0 "hostname; false" | wc -l

View file

@ -306,7 +306,7 @@ OK
/home/tange/privat/parallel/testsuite
OK
parallel --wd ... 'pwd; echo $OLDPWD; echo' ::: OK | perl -pe 's/\d+/0/g'
/home/tange/.parallel/tmp/hp-0-0
/home/tange/.parallel/tmp/aspire-0-0
/home/tange/privat/parallel/testsuite
OK
parallel --wd . 'pwd; echo $OLDPWD; echo' ::: OK
@ -595,7 +595,7 @@ e
echo '### test too long args'
### test too long args
perl -e 'print "z"x1000000' | parallel echo 2>&1
parallel: Error: Command line too long (1000005 >= 65524) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
parallel: Error: Command line too long (1000005 >= 65528) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
perl -e 'print "z"x1000000' | xargs echo 2>&1
xargs: argument line too long
(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -km -s 10 echo
@ -862,7 +862,7 @@ echo far
### Test --show-limits
(echo b; echo c; echo f) | parallel -k --show-limits echo {}ar
Maximal size of command: 131049
Maximal used size of command: 65524
Maximal used size of command: 65528
Execution of will continue now, and it will try to read its input
and run commands; if this is not what you wanted to happen, please
@ -1243,17 +1243,26 @@ a b
seq 1 92 | parallel -j+0 -kX -s 100 echo
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
66 67 68 69 70 71 72 73 74 75 76 77 78 79
80 81 82 83 84 85 86 87 88 89 90 91 92
66 67 68 69
70 71 72 73
74 75 76 77
78 79 80 81
82 83 84 85
86 87 88 89
90 91 92
echo '### Test distribute arguments at EOF to 5 jobslots'
### Test distribute arguments at EOF to 5 jobslots
seq 1 92 | parallel -j+3 -kX -s 100 echo
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
66 67 68 69 70 71
72 73 74 75 76 77
78 79 80 81 82 83
84 85 86 87 88 89
66 67 68
69 70 71
72 73 74
75 76 77
78 79 80
81 82 83
84 85 86
87 88 89
90 91 92
echo '### Test distribute arguments at EOF to infinity jobslots'
### Test distribute arguments at EOF to infinity jobslots
@ -1626,7 +1635,7 @@ echo '### bug #34422: parallel -X --eta crashes with div by zero'
seq 2 | stdout parallel -X --eta echo | grep -E -v 'ETA:.*AVG'
Computers / CPU cores / Max jobs to run
1:local / 2 / 2
1:local / 8 / 2
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
echo '**'

View file

@ -1,7 +1,7 @@
echo '### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922'
### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922
seq 1000000 > /tmp/num1000000; stdout parallel --pipepart --progress -a /tmp/num1000000 --block 10k -j0 true |grep 1:local
1:local / 2 / 252
1:local / 8 / 252
echo '**'
**
testhalt() { echo '### testhalt --halt '$1; (yes 0 | head -n 10; seq 10) | stdout parallel -kj4 --halt $1 'sleep {= $_=$_*0.3+1 =}; exit {}'; echo $?; (seq 10; yes 0 | head -n 10) | stdout parallel -kj4 --halt $1 'sleep {= $_=$_*0.3+1 =}; exit {}'; echo $?; }; export -f testhalt; parallel -kj0 testhalt ::: now,fail=0 now,fail=1 now,fail=2 now,fail=30% now,fail=70% soon,fail=0 soon,fail=1 soon,fail=2 soon,fail=30% soon,fail=70% now,success=0 now,success=1 now,success=2 now,success=30% now,success=70% soon,success=0 soon,success=1 soon,success=2 soon,success=30% now,success=70%
@ -863,12 +863,12 @@ echo '**'
echo "### Test max line length -m -I"
### Test max line length -m -I
seq 1 60000 | parallel -I :: -m -j1 echo a::b::c | sort >/tmp/114-a$$; md5sum </tmp/114-a$$; export CHAR=$(cat /tmp/114-a$$ | wc -c); export LINES=$(cat /tmp/114-a$$ | wc -l); echo "Chars per line ($CHAR/$LINES): "$(echo "$CHAR/$LINES" | bc); rm /tmp/114-a$$
d10d5c84fc4716b21d90afa92cf44d73 -
14bacad229d8b0d32be0a2339c2a6af7 -
Chars per line (697810/11): 63437
echo "### Test max line length -X -I"
### Test max line length -X -I
seq 1 60000 | parallel -I :: -X -j1 echo a::b::c | sort >/tmp/114-b$$; md5sum </tmp/114-b$$; export CHAR=$(cat /tmp/114-b$$ | wc -c); export LINES=$(cat /tmp/114-b$$ | wc -l); echo "Chars per line ($CHAR/$LINES): "$(echo "$CHAR/$LINES" | bc); rm /tmp/114-b$$
47ec7550232044dc1e7a504705a45184 -
81c0a85162c989c07f666b827a30ce52 -
Chars per line (817788/13): 62906
echo '**'
**

View file

@ -4,7 +4,7 @@ echo "### BUG: The length for -X is not close to max (131072)"; seq 1 60000 |
seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc
1 5644 65514
seq 1 60000 | parallel -X echo |head -n 1 |wc
1 12770 65514
1 12771 65520
seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc
1 8098 65512
seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc

View file

@ -192,7 +192,7 @@ ls: cannot access bug46519.ccc: No such file or directory
echo '### Test --nice remote'
### Test --nice remote
stdout parallel --nice 1 -S lo -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b | perl -pe 's/\S*parallel-server\S*/one-server/;s:[a-z/\\+=0-9]{500,}:base64:i;'
ssh lo -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; base64;
ssh lo -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;\\\$SIG\\\{CHLD\\\}\\\=\\\"IGNORE\\\"\\\;my\\\$zip\\\=\\\(grep\\\{-x\\\$_\\\}\\\"/usr/local/bin/bzip2\\\"\\\)\\\[0\\\]\\\|\\\|\\\"bzip2\\\"\\\;my\\\(\\\$in,\\\$out,\\\$eval\\\)\\\;open3\\\(\\\$in,\\\$out,\\\"\\\>\\\&STDERR\\\",\\\$zip,\\\"-dc\\\"\\\)\\\;if\\\(my\\\$perlpid\\\=fork\\\)\\\{close\\\$in\\\;\\\$eval\\\=join\\\"\\\",\\\<\\\$out\\\>\\\;close\\\$out\\\;\\\}else\\\{close\\\$out\\\;print\\\$in\\\(decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\)\\\;close\\\$in\\\;exit\\\;\\\}wait\\\;eval\\\$eval\\\; base64;
a b
echo '**'
**

View file

@ -123,19 +123,25 @@ echo '### exported function to csh but with PARALLEL_SHELL=bash'
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset doit
OK
bug #47695: How to set $PATH on remote?
Welcome to Linux Mint 17.3 Rosa (GNU/Linux 3.19.0-32-generic x86_64)
Welcome to Linux Mint 17 Qiana (GNU/Linux 3.16.0-31-lowlatency x86_64)
Welcome to Linux Mint
* Documentation: http://www.linuxmint.com
0 updates are security updates.
BASH Path before: /bin:/usr/bin with no parallel
-bash: line 2: parallel: command not found
^^^^^^^^ Not found is OK
/bin:/usr/bin:/tmp OK
Welcome to Linux Mint 17.3 Rosa (GNU/Linux 3.19.0-32-generic x86_64)
Welcome to Linux Mint 17 Qiana (GNU/Linux 3.16.0-31-lowlatency x86_64)
Welcome to Linux Mint
* Documentation: http://www.linuxmint.com
0 updates are security updates.
Warning: no access to tty (Bad file descriptor).
Thus no job control in this shell.
CSH Path before: /bin:/usr/bin with no parallel

View file

@ -456,7 +456,7 @@ echo '### true < 32767-ys.xi'
### true < 32767-ys.xi
stdout xargs true < 32767-ys.xi
stdout parallel -k true < 32767-ys.xi
parallel: Error: Command line too long (98306 >= 65524) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ...
parallel: Error: Command line too long (98306 >= 65528) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ...
echo '### true < 16383-ys.xi'
### true < 16383-ys.xi
stdout xargs true < 16383-ys.xi

View file

@ -111,7 +111,7 @@ echo '### Test -m with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | par
20 179960 1286702
echo '### Test -X with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; wait; sleep 1
### Test -X with 60000 args
8646e0f7fd5f8795e4b1a831ede50e85 -
97715240fa65309902932877d24273db -
21 60000 1346682
echo '### Test -X with 60000 args and 5 expansions'
### Test -X with 60000 args and 5 expansions

View file

@ -92,7 +92,7 @@ With script in $PARALLEL /bin/bash=/home/tange/privat/parallel/testsuite
echo '### bug #42892: parallel -a nonexiting --pipepart'
### bug #42892: parallel -a nonexiting --pipepart
parallel --pipepart -a nonexisting wc
parallel: Error: nonexisting is not a seekable file.
parallel: Error: nonexisting is neither a file nor a block device
echo '### bug #42913: Dont use $SHELL but the shell currently running'
### bug #42913: Dont use $SHELL but the shell currently running
echo '## Unknown shell => $SHELL (bash)'

View file

@ -86,9 +86,9 @@ please cite as described in 'parallel --citation'.
echo '### bug #39787: --xargs broken'
### bug #39787: --xargs broken
nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"'
65514
65514
37866
65520
65520
37854
echo '### --delay should grow by 3 sec per arg'
### --delay should grow by 3 sec per arg
stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"'

View file

@ -1,9 +1,15 @@
bug #46120: Suspend should suspend (at least local) children
2048 4b:d0:08:9c:88:c1:b9:20:ce:69:91:73:e0:07:5d:13 /home/tange/.ssh/id_rsa (RSA)
2048 0a:c0:70:5b:ec:f6:c2:de:67:c3:53:7f:29:81:65:54 tange@hk (RSA1)
1024 93:d1:21:c5:ff:e3:c3:be:6d:73:ea:aa:1e:a2:dc:06 /home/tange/.ssh/id_dsa (DSA)
8192 e1:95:e3:ff:99:a6:3a:b5:53:5a:54:59:d0:72:94:7f /home/tange/.ssh/id_rsa (RSA)
4096 94:2a:e3:cb:6b:66:63:21:13:51:8d:e8:4e:09:49:b2 /home/tange/.ssh/id_rsa_openindiana (RSA)
stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 burnP6 ::: 1 | grep --colour=auto -q CPUTIME=1
Zero=OK 0
2048 4b:d0:08:9c:88:c1:b9:20:ce:69:91:73:e0:07:5d:13 /home/tange/.ssh/id_rsa (RSA)
2048 0a:c0:70:5b:ec:f6:c2:de:67:c3:53:7f:29:81:65:54 tange@hk (RSA1)
1024 93:d1:21:c5:ff:e3:c3:be:6d:73:ea:aa:1e:a2:dc:06 /home/tange/.ssh/id_dsa (DSA)
8192 e1:95:e3:ff:99:a6:3a:b5:53:5a:54:59:d0:72:94:7f /home/tange/.ssh/id_rsa (RSA)
4096 94:2a:e3:cb:6b:66:63:21:13:51:8d:e8:4e:09:49:b2 /home/tange/.ssh/id_rsa_openindiana (RSA)
echo 1 | stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 burnP6 | grep --colour=auto -q CPUTIME=1
Zero=OK 0

View file

@ -63,7 +63,7 @@ echo '### Test --number-of-cpus'; stdout $NICEPAR --number-of-cpus
1
echo '### Test --number-of-cores'; stdout $NICEPAR --number-of-cores
### Test --number-of-cores
2
8
echo '### Test --use-cpus-instead-of-cores'; (seq 1 8 | stdout parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done & (seq 1 8 | stdout parallel -j100% sleep) && echo cores done & echo 'Cores should complete first on machines with less than 8 physical CPUs'; wait
### Test --use-cpus-instead-of-cores
Cores should complete first on machines with less than 8 physical CPUs
@ -400,7 +400,7 @@ echo "### BUG: empty lines with --show-limit"
### BUG: empty lines with --show-limit
echo | $NICEPAR --show-limits
Maximal size of command: 131049
Maximal used size of command: 65524
Maximal used size of command: 65528
Execution of will continue now, and it will try to read its input
and run commands; if this is not what you wanted to happen, please

View file

@ -701,6 +701,11 @@ parallel@lo foo bar
alias myecho=echo
myvar="Joe's var is"
env_parallel -S $SERVER1 'myecho $myvar' ::: green
Unknown option: -S
Unknown option: parallel@lo
Unknown option: myecho $myvar
Unknown option: :::
Unknown option: green
env_parallel only works if it is a function. Do the below and restart your shell.
bash: Put this in /home/tange/.bashrc: . /usr/local/bin/env_parallel.bash
@ -712,9 +717,9 @@ zsh: Put this in /home/tange/.zshrc: . /usr/local/bin/env_parallel.zsh
Supports: functions, variables, arrays
fish: Put this in /home/tange/.config/fish/config.fish:
source (which env_parallel.fish)
. (which env_parallel.fish)
E.g. by doing:
echo 'source (which env_parallel.fish)' >> /home/tange/.config/fish/config.fish
echo '. (which env_parallel.fish)' >> /home/tange/.config/fish/config.fish
Supports: aliases, functions, variables, arrays
ksh: Put this in /home/tange/.kshrc: source /usr/local/bin/env_parallel.ksh
@ -722,7 +727,7 @@ ksh: Put this in /home/tange/.kshrc: source /usr/local/bin/env_parallel.ksh
Supports: aliases, functions, variables, arrays
pdksh: Put this in /home/tange/.profile: source /usr/local/bin/env_parallel.pdksh
E.g. by doing: echo 'source /usr/local/bin/env_parallel.pdksh' >> /home/tange/.profile
E.g. by doing: echo '. /usr/local/bin/env_parallel.pdksh' >> /home/tange/.profile
Supports: aliases, functions, variables, arrays
csh: Put this in /home/tange/.cshrc: source /usr/local/bin/env_parallel.csh
@ -733,6 +738,10 @@ tcsh: Put this in /home/tange/.tcshrc: source /usr/local/bin/env_parallel.tcsh
E.g. by doing: echo 'source /usr/local/bin/env_parallel.tcsh' >> /home/tange/.tcshrc
Supports: aliases, variables, arrays with no special chars
To install in all shells run:
env_parallel --install
For details: see man env_parallel
MYVAR='foo bar'
@ -854,7 +863,7 @@ _
}
export -f my_func3
parallel -vv --workdir ... --nice 17 --env _ --trc {}.out -S $SERVER1 my_func3 {} ::: abc-file
( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; BASE64 BASE64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ rmdir\\\ ./.TMPWORKDIR/\\\ ./.parallel/tmp/\\\ ./.parallel/\\\ 2\\\>/dev/null\\\;rm\\\ -rf\\\ ./.TMPWORKDIR\\\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ rmdir\\\ ./.TMPWORKDIR/\\\ ./.parallel/tmp/\\\ ./.parallel/\\\ 2\\\>/dev/null\\\;rm\\\ -rf\\\ ./.TMPWORKDIR\\\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status;
( ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh\ -l\ parallel ./abc-file lo:./.TMPWORKDIR );ssh -l parallel lo -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;\\\$SIG\\\{CHLD\\\}\\\=\\\"IGNORE\\\"\\\;my\\\$zip\\\=\\\(grep\\\{-x\\\$_\\\}\\\"/usr/local/bin/bzip2\\\"\\\)\\\[0\\\]\\\|\\\|\\\"bzip2\\\"\\\;my\\\(\\\$in,\\\$out,\\\$eval\\\)\\\;open3\\\(\\\$in,\\\$out,\\\"\\\>\\\&STDERR\\\",\\\$zip,\\\"-dc\\\"\\\)\\\;if\\\(my\\\$perlpid\\\=fork\\\)\\\{close\\\$in\\\;\\\$eval\\\=join\\\"\\\",\\\<\\\$out\\\>\\\;close\\\$out\\\;\\\}else\\\{close\\\$out\\\;print\\\$in\\\(decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\)\\\;close\\\$in\\\;exit\\\;\\\}wait\\\;eval\\\$eval\\\; BASE64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh\ -l\ parallel lo:./abc-file.out ./.;ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ rmdir\\\ ./.TMPWORKDIR/\\\ ./.parallel/tmp/\\\ ./.parallel/\\\ 2\\\>/dev/null\\\;rm\\\ -rf\\\ ./.TMPWORKDIR\\\;\);ssh -l parallel lo -- \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ rmdir\\\ ./.TMPWORKDIR/\\\ ./.parallel/tmp/\\\ ./.parallel/\\\ 2\\\>/dev/null\\\;rm\\\ -rf\\\ ./.TMPWORKDIR\\\;\);ssh -l parallel lo -- rm -rf .TMPWORKDIR; exit $_EXIT_status;
parallel --sqlandworker csv:////%2Ftmp%2Flog.csv seq ::: 10 ::: 12 13 14
cat /tmp/log.csv
99
@ -926,23 +935,23 @@ parallel: Error: echo is not a valid DBURL
parallel --sqlworker $DBURLTABLE echo ::: foo bar ::: baz quuz
parallel: Error: echo is not a valid DBURL
cat num1000000 | parallel --pipe wc
165668 165668 1048571
149796 149796 1048572
149796 149796 1048572
149796 149796 1048572
149796 149796 1048572
149796 149796 1048572
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
85352 85352 597465
cat num1000000 | parallel --pipe --block 2M wc
315465 315465 2097150
299593 299593 2097151
299593 299593 2097151
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
85349 85349 597444
cat num1000000 | parallel --pipe -j4 --round-robin wc
315464 315464 2097143
299592 299592 2097144
235148 235148 1646037
149796 149796 1048572
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
cat num1000000 | parallel --pipe -N140000 wc
140000 140000 868895
140000 140000 980000
@ -953,12 +962,12 @@ parallel: Error: echo is not a valid DBURL
140000 140000 980000
20000 20000 140001
cat num1000000 | parallel --pipe -L75 wc
165600 165600 1048095
149775 149775 1048425
149775 149775 1048425
149775 149775 1048425
149775 149775 1048425
149775 149775 1048425
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
999999 999999 9999999
85500 85500 598500
25 25 176
echo /foo, bar/, /baz, qux/, | parallel -kN1 --recend ', ' --pipe echo JOB{#}\;cat\;echo END
@ -1050,8 +1059,8 @@ JOB4
%head2
99
parallel --pipepart -a num1000000 --block 3m wc
444444 444444 3000003
428572 428572 3000004
999999 999999 9999999
999999 999999 9999999
126984 126984 888889
cat input_file | parallel command
/bin/bash: This is input_file: command not found

View file

@ -55,35 +55,35 @@ Yes it does
12 74 500
### Test -r
dburl pg://nongood@127.0.0.3:2227/
databasedriver pg user nongood password host 127.0.0.3 port 2227 database query
psql -h 127.0.0.3 -U nongood -p 2227
databasedriver pg user nongood password host 127.0.0.3 port 2227 database nongood query
psql -h 127.0.0.3 -U nongood -p 2227 -d nongood
psql: could not connect to server: Connection refused
Is the server running on host "127.0.0.3" and accepting
TCP/IP connections on port 2227?
psql -h 127.0.0.3 -U nongood -p 2227
psql -h 127.0.0.3 -U nongood -p 2227 -d nongood
psql: could not connect to server: Connection refused
Is the server running on host "127.0.0.3" and accepting
TCP/IP connections on port 2227?
psql -h 127.0.0.3 -U nongood -p 2227
psql -h 127.0.0.3 -U nongood -p 2227 -d nongood
psql: could not connect to server: Connection refused
Is the server running on host "127.0.0.3" and accepting
TCP/IP connections on port 2227?
### Test --retries=s
dburl pg://nongood@127.0.0.3:2227/
databasedriver pg user nongood password host 127.0.0.3 port 2227 database query
psql -h 127.0.0.3 -U nongood -p 2227
databasedriver pg user nongood password host 127.0.0.3 port 2227 database nongood query
psql -h 127.0.0.3 -U nongood -p 2227 -d nongood
psql: could not connect to server: Connection refused
Is the server running on host "127.0.0.3" and accepting
TCP/IP connections on port 2227?
psql -h 127.0.0.3 -U nongood -p 2227
psql -h 127.0.0.3 -U nongood -p 2227 -d nongood
psql: could not connect to server: Connection refused
Is the server running on host "127.0.0.3" and accepting
TCP/IP connections on port 2227?
psql -h 127.0.0.3 -U nongood -p 2227
psql -h 127.0.0.3 -U nongood -p 2227 -d nongood
psql: could not connect to server: Connection refused
Is the server running on host "127.0.0.3" and accepting
TCP/IP connections on port 2227?
psql -h 127.0.0.3 -U nongood -p 2227
psql -h 127.0.0.3 -U nongood -p 2227 -d nongood
psql: could not connect to server: Connection refused
Is the server running on host "127.0.0.3" and accepting
TCP/IP connections on port 2227?

View file

@ -77,7 +77,7 @@ OK
Input for ssh
-l parallel one-server -- mkdir -p ./.
-l parallel one-server rsync --server -lDrRze.iLs . ./.
-l parallel one-server -- exec perl -e @GNU_Parallel\=\(\"use\",\"IPC::Open3\;\",\"use\",\"MIME::Base64\"\)\;eval\"@GNU_Parallel\"\;my\$eval\=decode_base64\(join\"\",@ARGV\)\;eval\$eval\; base64
-l parallel one-server -- exec perl -e @GNU_Parallel\=\(\"use\",\"IPC::Open3\;\",\"use\",\"MIME::Base64\"\)\;eval\"@GNU_Parallel\"\;\$SIG\{CHLD\}\=\"IGNORE\"\;my\$zip\=\(grep\{-x\$_\}\"/usr/local/bin/bzip2\"\)\[0\]\|\|\"bzip2\"\;my\(\$in,\$out,\$eval\)\;open3\(\$in,\$out,\"\>\&STDERR\",\$zip,\"-dc\"\)\;if\(my\$perlpid\=fork\)\{close\$in\;\$eval\=join\"\",\<\$out\>\;close\$out\;\}else\{close\$out\;print\$in\(decode_base64\(join\"\",@ARGV\)\)\;close\$in\;exit\;\}wait\;eval\$eval\; base64
-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
'newlineX.out
-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
@ -90,7 +90,7 @@ Input for ssh
'newlineX.out2; sh -c rmdir\ ./tmp/\ ./\ 2\>/dev/null\;)
-l parallel one-server -- mkdir -p ./.
-l parallel one-server rsync --server -lDrRze.iLs . ./.
-l parallel one-server -- exec perl -e @GNU_Parallel\=\(\"use\",\"IPC::Open3\;\",\"use\",\"MIME::Base64\"\)\;eval\"@GNU_Parallel\"\;my\$eval\=decode_base64\(join\"\",@ARGV\)\;eval\$eval\; base64
-l parallel one-server -- exec perl -e @GNU_Parallel\=\(\"use\",\"IPC::Open3\;\",\"use\",\"MIME::Base64\"\)\;eval\"@GNU_Parallel\"\;\$SIG\{CHLD\}\=\"IGNORE\"\;my\$zip\=\(grep\{-x\$_\}\"/usr/local/bin/bzip2\"\)\[0\]\|\|\"bzip2\"\;my\(\$in,\$out,\$eval\)\;open3\(\$in,\$out,\"\>\&STDERR\",\$zip,\"-dc\"\)\;if\(my\$perlpid\=fork\)\{close\$in\;\$eval\=join\"\",\<\$out\>\;close\$out\;\}else\{close\$out\;print\$in\(decode_base64\(join\"\",@ARGV\)\)\;close\$in\;exit\;\}wait\;eval\$eval\; base64
-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
'newlineX.out
-l parallel one-server cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'

View file

@ -1,13 +1,13 @@
echo '### Test --return of weirdly named file'
### Test --return of weirdly named file
stdout parallel --return {} -vv -S parallel\@parallel-server3 echo '>'{} ::: 'aa<${#}" b' | perl -pe 's/\S*parallel-server\S*/one-server/;s:[a-z+=/\\0-9]{500,}:base64:i;'; rm 'aa<${#}" b'
ssh -l parallel one-server -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; base64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh\ -l\ parallel parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status;
ssh -l parallel one-server -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;\\\$SIG\\\{CHLD\\\}\\\=\\\"IGNORE\\\"\\\;my\\\$zip\\\=\\\(grep\\\{-x\\\$_\\\}\\\"/usr/local/bin/bzip2\\\"\\\)\\\[0\\\]\\\|\\\|\\\"bzip2\\\"\\\;my\\\(\\\$in,\\\$out,\\\$eval\\\)\\\;open3\\\(\\\$in,\\\$out,\\\"\\\>\\\&STDERR\\\",\\\$zip,\\\"-dc\\\"\\\)\\\;if\\\(my\\\$perlpid\\\=fork\\\)\\\{close\\\$in\\\;\\\$eval\\\=join\\\"\\\",\\\<\\\$out\\\>\\\;close\\\$out\\\;\\\}else\\\{close\\\$out\\\;print\\\$in\\\(decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\)\\\;close\\\$in\\\;exit\\\;\\\}wait\\\;eval\\\$eval\\\; base64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ././.\;\ rsync -rlDzR -essh\ -l\ parallel parallel-server3:./aa\\\<\\\$\\\{\\\#\\\}\\\"\\\ b ./.; exit $_EXIT_status;
echo '### Test if remote login shell is csh'
### Test if remote login shell is csh
stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c | perl -pe 's/\S*parallel-server\S*/one-server/;s:[a-z+=/\\0-9]{500,}:base64:i;'
ssh -l csh localhost -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; base64;
ssh -l csh localhost -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;\\\$SIG\\\{CHLD\\\}\\\=\\\"IGNORE\\\"\\\;my\\\$zip\\\=\\\(grep\\\{-x\\\$_\\\}\\\"/usr/local/bin/bzip2\\\"\\\)\\\[0\\\]\\\|\\\|\\\"bzip2\\\"\\\;my\\\(\\\$in,\\\$out,\\\$eval\\\)\\\;open3\\\(\\\$in,\\\$out,\\\"\\\>\\\&STDERR\\\",\\\$zip,\\\"-dc\\\"\\\)\\\;if\\\(my\\\$perlpid\\\=fork\\\)\\\{close\\\$in\\\;\\\$eval\\\=join\\\"\\\",\\\<\\\$out\\\>\\\;close\\\$out\\\;\\\}else\\\{close\\\$out\\\;print\\\$in\\\(decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\)\\\;close\\\$in\\\;exit\\\;\\\}wait\\\;eval\\\$eval\\\; base64;
3
ssh -l csh localhost -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; base64;
ssh -l csh localhost -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;\\\$SIG\\\{CHLD\\\}\\\=\\\"IGNORE\\\"\\\;my\\\$zip\\\=\\\(grep\\\{-x\\\$_\\\}\\\"/usr/local/bin/bzip2\\\"\\\)\\\[0\\\]\\\|\\\|\\\"bzip2\\\"\\\;my\\\(\\\$in,\\\$out,\\\$eval\\\)\\\;open3\\\(\\\$in,\\\$out,\\\"\\\>\\\&STDERR\\\",\\\$zip,\\\"-dc\\\"\\\)\\\;if\\\(my\\\$perlpid\\\=fork\\\)\\\{close\\\$in\\\;\\\$eval\\\=join\\\"\\\",\\\<\\\$out\\\>\\\;close\\\$out\\\;\\\}else\\\{close\\\$out\\\;print\\\$in\\\(decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\)\\\;close\\\$in\\\;exit\\\;\\\}wait\\\;eval\\\$eval\\\; base64;
3
ssh -l csh localhost -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; base64;
ssh -l csh localhost -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;\\\$SIG\\\{CHLD\\\}\\\=\\\"IGNORE\\\"\\\;my\\\$zip\\\=\\\(grep\\\{-x\\\$_\\\}\\\"/usr/local/bin/bzip2\\\"\\\)\\\[0\\\]\\\|\\\|\\\"bzip2\\\"\\\;my\\\(\\\$in,\\\$out,\\\$eval\\\)\\\;open3\\\(\\\$in,\\\$out,\\\"\\\>\\\&STDERR\\\",\\\$zip,\\\"-dc\\\"\\\)\\\;if\\\(my\\\$perlpid\\\=fork\\\)\\\{close\\\$in\\\;\\\$eval\\\=join\\\"\\\",\\\<\\\$out\\\>\\\;close\\\$out\\\;\\\}else\\\{close\\\$out\\\;print\\\$in\\\(decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\)\\\;close\\\$in\\\;exit\\\;\\\}wait\\\;eval\\\$eval\\\; base64;
3