parallel: --results -.json outputs a single JSON line per job.

This commit is contained in:
Ole Tange 2020-12-19 04:21:04 +01:00
parent e58cf1d7f7
commit a86a786189
4 changed files with 96 additions and 6 deletions

View file

@ -1886,6 +1886,10 @@ sub parse_options(@) {
# CSV with TAB as separator # CSV with TAB as separator
$Global::csvsep = "\t"; $Global::csvsep = "\t";
$Global::membuffer ||= 1; $Global::membuffer ||= 1;
} elsif($opt::results =~ /\.json$/i) {
# JSON output
$Global::jsonout ||= 1;
$Global::membuffer ||= 1;
} }
} }
if($opt::compress) { if($opt::compress) {
@ -2093,7 +2097,7 @@ sub parse_options(@) {
if(not defined $opt::jobs) { $opt::jobs = "100%"; } if(not defined $opt::jobs) { $opt::jobs = "100%"; }
open_joblog(); open_joblog();
open_csv(); open_json_csv();
if($opt::sqlmaster or $opt::sqlworker) { if($opt::sqlmaster or $opt::sqlworker) {
$Global::sql = SQL->new($opt::sqlmaster || $opt::sqlworker); $Global::sql = SQL->new($opt::sqlmaster || $opt::sqlworker);
} }
@ -2599,13 +2603,15 @@ sub open_joblog() {
} }
} }
sub open_csv() { sub open_json_csv() {
if($opt::results) { if($opt::results) {
# Output as CSV/TSV # Output as JSON/CSV/TSV
if($opt::results eq "-.csv" if($opt::results eq "-.csv"
or or
$opt::results eq "-.tsv") { $opt::results eq "-.tsv"
# Output as CSV/TSV on stdout or
$opt::results eq "-.json") {
# Output as JSON/CSV/TSV on stdout
open $Global::csv_fh, ">&", "STDOUT" or open $Global::csv_fh, ">&", "STDOUT" or
::die_bug("Can't dup STDOUT in csv: $!"); ::die_bug("Can't dup STDOUT in csv: $!");
# Do not print any other output to STDOUT # Do not print any other output to STDOUT
@ -10047,10 +10053,78 @@ sub print($) {
# Add output to CSV when finished # Add output to CSV when finished
$self->print_csv(); $self->print_csv();
} }
if($Global::jsonout) {
$self->print_json();
}
} }
return $returnsize - $self->returnsize(); return $returnsize - $self->returnsize();
} }
{
my %jsonmap;
sub print_json($) {
my $self = shift;
sub jsonquote($) {
my $a = shift;
if(not $jsonmap{"\001"}) {
map { $jsonmap{sprintf("%c",$_)} = sprintf '\u%04x', $_ } 0..31;
}
$a =~ s/\\/\\\\/g;
$a =~ s/\"/\\"/g;
$a =~ s/([\000-\037])/$jsonmap{$1}/g;
return $a;
}
my $cmd;
if($Global::verbose <= 1) {
$cmd = jsonquote($self->replaced());
} else {
# Verbose level > 1: Print the rsync and stuff
$cmd = jsonquote(join " ", @{$self->{'commandline'}});
}
my $record_ref = $self->{'commandline'}{'arg_list_flat_orig'};
# Memory optimization: Overwrite with the joined output
$self->{'output'}{1} = join("", @{$self->{'output'}{1}});
$self->{'output'}{2} = join("", @{$self->{'output'}{2}});
# {
# "Seq": 12,
# "Host": "/usr/bin/ssh foo@lo",
# "Starttime": 1608344711.743,
# "JobRuntime": 0.01,
# "Send": 0,
# "Receive": 10,
# "Exitval": 0,
# "Signal": 0,
# "Command": "echo 1",
# "V": [
# "1"
# ],
# "Stdout": "1\n",
# "Stderr": ""
# }
#
printf($Global::csv_fh
q({ "Seq": %s, "Host": "%s", "Starttime": %s, "JobRuntime": %s, ).
q("Send": %s, "Receive": %s, "Exitval": %s, "Signal": %s, ).
q("Command": "%s", "V": [ %s ], "Stdout": "%s", "Stderr": "%s" }).
"\n",
$self->seq(),
jsonquote($self->sshlogin()->string()),
$self->starttime(), sprintf("%0.3f",$self->runtime()),
$self->transfersize(), $self->returnsize(),
$self->exitstatus(), $self->exitsignal(), $cmd,
# \@$record_ref[1..$#$record_ref],
(join ",",
map { '"'.jsonquote($_).'"' } @$record_ref[1..$#$record_ref],
),
jsonquote($self->{'output'}{1}),
jsonquote($self->{'output'}{2})
);
}
}
{ {
my $header_printed; my $header_printed;

View file

@ -1858,7 +1858,16 @@ B<-.csv>/B<-.tsv> are special: It will give the file on stdout
(standard output). (standard output).
B<Replacement string output file> B<JSON file output> (alpha testing)
If I<name> ends in B<.json> the output will be a JSON-file
named I<name>.
B<-.json> is special: It will give the file on stdout (standard
output).
B<Replacement string output file> (alpha testing)
If I<name> contains a replacement string and the replaced result does If I<name> contains a replacement string and the replaced result does
not end in /, then the standard output will be stored in a file named not end in /, then the standard output will be stored in a file named

View file

@ -873,6 +873,11 @@ par_group-by_colsep_space() {
input ' ' | parallel --pipe --group-by 2 --colsep ' ' -kN1 wc input ' ' | parallel --pipe --group-by 2 --colsep ' ' -kN1 wc
} }
par_json() {
printf '"\t\\"' | parallel --results -.json echo :::: - ::: '"' '\\' |
perl -pe 's/\d/0/g'
}
export -f $(compgen -A function | grep par_) export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | LC_ALL=C sort | compgen -A function | grep par_ | LC_ALL=C sort |
parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' | parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |

View file

@ -233,6 +233,8 @@ par_jobslot_jobnumber_pipe 1
par_jobslot_jobnumber_pipe 1 par_jobslot_jobnumber_pipe 1
par_jobslot_jobnumber_pipe 1 par_jobslot_jobnumber_pipe 1
par_jobslot_jobnumber_pipe 1 par_jobslot_jobnumber_pipe 1
par_json { "Seq": 0, "Host": ":", "Starttime": 0000000000.000, "JobRuntime": 0.000, "Send": 0, "Receive": 0, "Exitval": 0, "Signal": 0, "Command": "echo '\"\u0000\\\"' '\"'", "V": [ "\"\u0000\\\"","\"" ], "Stdout": "\"\u0000\\\" \"\u000a", "Stderr": "" }
par_json { "Seq": 0, "Host": ":", "Starttime": 0000000000.000, "JobRuntime": 0.000, "Send": 0, "Receive": 0, "Exitval": 0, "Signal": 0, "Command": "echo '\"\u0000\\\"' '\\\\'", "V": [ "\"\u0000\\\"","\\\\" ], "Stdout": "\"\u0000\\\" \\\\\u000a", "Stderr": "" }
par_l0_is_l1 ### Because of --tollef -l, then -l0 == -l1, sorry par_l0_is_l1 ### Because of --tollef -l, then -l0 == -l1, sorry
par_l0_is_l1 l0 1 par_l0_is_l1 l0 1
par_l0_is_l1 l0 2 par_l0_is_l1 l0 2