From a86a786189e05e08f10f88ec01e7c2cccba11f40 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 19 Dec 2020 04:21:04 +0100 Subject: [PATCH] parallel: --results -.json outputs a single JSON line per job. --- src/parallel | 84 +++++++++++++++++-- src/parallel.pod | 11 ++- testsuite/tests-to-run/parallel-local-0.3s.sh | 5 ++ testsuite/wanted-results/parallel-local-0.3s | 2 + 4 files changed, 96 insertions(+), 6 deletions(-) diff --git a/src/parallel b/src/parallel index 0e7c915e..685c489f 100755 --- a/src/parallel +++ b/src/parallel @@ -1886,6 +1886,10 @@ sub parse_options(@) { # CSV with TAB as separator $Global::csvsep = "\t"; $Global::membuffer ||= 1; + } elsif($opt::results =~ /\.json$/i) { + # JSON output + $Global::jsonout ||= 1; + $Global::membuffer ||= 1; } } if($opt::compress) { @@ -2093,7 +2097,7 @@ sub parse_options(@) { if(not defined $opt::jobs) { $opt::jobs = "100%"; } open_joblog(); - open_csv(); + open_json_csv(); if($opt::sqlmaster or $opt::sqlworker) { $Global::sql = SQL->new($opt::sqlmaster || $opt::sqlworker); } @@ -2599,13 +2603,15 @@ sub open_joblog() { } } -sub open_csv() { +sub open_json_csv() { if($opt::results) { - # Output as CSV/TSV + # Output as JSON/CSV/TSV if($opt::results eq "-.csv" or - $opt::results eq "-.tsv") { - # Output as CSV/TSV on stdout + $opt::results eq "-.tsv" + or + $opt::results eq "-.json") { + # Output as JSON/CSV/TSV on stdout open $Global::csv_fh, ">&", "STDOUT" or ::die_bug("Can't dup STDOUT in csv: $!"); # Do not print any other output to STDOUT @@ -10047,10 +10053,78 @@ sub print($) { # Add output to CSV when finished $self->print_csv(); } + if($Global::jsonout) { + $self->print_json(); + } } return $returnsize - $self->returnsize(); } +{ + my %jsonmap; + + sub print_json($) { + my $self = shift; + sub jsonquote($) { + my $a = shift; + if(not $jsonmap{"\001"}) { + map { $jsonmap{sprintf("%c",$_)} = sprintf '\u%04x', $_ } 0..31; + } + $a =~ s/\\/\\\\/g; + $a =~ s/\"/\\"/g; + $a =~ s/([\000-\037])/$jsonmap{$1}/g; + return $a; + } + + my $cmd; + if($Global::verbose <= 1) { + $cmd = jsonquote($self->replaced()); + } else { + # Verbose level > 1: Print the rsync and stuff + $cmd = jsonquote(join " ", @{$self->{'commandline'}}); + } + my $record_ref = $self->{'commandline'}{'arg_list_flat_orig'}; + + # Memory optimization: Overwrite with the joined output + $self->{'output'}{1} = join("", @{$self->{'output'}{1}}); + $self->{'output'}{2} = join("", @{$self->{'output'}{2}}); + # { + # "Seq": 12, + # "Host": "/usr/bin/ssh foo@lo", + # "Starttime": 1608344711.743, + # "JobRuntime": 0.01, + # "Send": 0, + # "Receive": 10, + # "Exitval": 0, + # "Signal": 0, + # "Command": "echo 1", + # "V": [ + # "1" + # ], + # "Stdout": "1\n", + # "Stderr": "" + # } + # + printf($Global::csv_fh + q({ "Seq": %s, "Host": "%s", "Starttime": %s, "JobRuntime": %s, ). + q("Send": %s, "Receive": %s, "Exitval": %s, "Signal": %s, ). + q("Command": "%s", "V": [ %s ], "Stdout": "%s", "Stderr": "%s" }). + "\n", + $self->seq(), + jsonquote($self->sshlogin()->string()), + $self->starttime(), sprintf("%0.3f",$self->runtime()), + $self->transfersize(), $self->returnsize(), + $self->exitstatus(), $self->exitsignal(), $cmd, + # \@$record_ref[1..$#$record_ref], + (join ",", + map { '"'.jsonquote($_).'"' } @$record_ref[1..$#$record_ref], + ), + jsonquote($self->{'output'}{1}), + jsonquote($self->{'output'}{2}) + ); + } +} + { my $header_printed; diff --git a/src/parallel.pod b/src/parallel.pod index 7705ad16..78c25a01 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1858,7 +1858,16 @@ B<-.csv>/B<-.tsv> are special: It will give the file on stdout (standard output). -B +B (alpha testing) + +If I ends in B<.json> the output will be a JSON-file +named I. + +B<-.json> is special: It will give the file on stdout (standard +output). + + +B (alpha testing) If I contains a replacement string and the replaced result does not end in /, then the standard output will be stored in a file named diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index fdc0b364..0894945c 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -873,6 +873,11 @@ par_group-by_colsep_space() { input ' ' | parallel --pipe --group-by 2 --colsep ' ' -kN1 wc } +par_json() { + printf '"\t\\"' | parallel --results -.json echo :::: - ::: '"' '\\' | + perl -pe 's/\d/0/g' +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | LC_ALL=C sort | parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' | diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 6f90d375..d5042b01 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -233,6 +233,8 @@ par_jobslot_jobnumber_pipe 1 par_jobslot_jobnumber_pipe 1 par_jobslot_jobnumber_pipe 1 par_jobslot_jobnumber_pipe 1 +par_json { "Seq": 0, "Host": ":", "Starttime": 0000000000.000, "JobRuntime": 0.000, "Send": 0, "Receive": 0, "Exitval": 0, "Signal": 0, "Command": "echo '\"\u0000\\\"' '\"'", "V": [ "\"\u0000\\\"","\"" ], "Stdout": "\"\u0000\\\" \"\u000a", "Stderr": "" } +par_json { "Seq": 0, "Host": ":", "Starttime": 0000000000.000, "JobRuntime": 0.000, "Send": 0, "Receive": 0, "Exitval": 0, "Signal": 0, "Command": "echo '\"\u0000\\\"' '\\\\'", "V": [ "\"\u0000\\\"","\\\\" ], "Stdout": "\"\u0000\\\" \\\\\u000a", "Stderr": "" } par_l0_is_l1 ### Because of --tollef -l, then -l0 == -l1, sorry par_l0_is_l1 l0 1 par_l0_is_l1 l0 2