360 lines
7.7 KiB
Bash
Executable file
360 lines
7.7 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
: <<=cut
|
|
=pod
|
|
|
|
=head1 NAME
|
|
|
|
transpose - transpose CSV file
|
|
|
|
=head1 SYNOPSIS
|
|
|
|
B<transpose> [-d I<delim>] [-b I<blocksize>] [I<input>]
|
|
|
|
=head1 DESCRIPTION
|
|
|
|
B<transpose> will read a CSV fie
|
|
|
|
=head1 OPTIONS
|
|
|
|
=over 9
|
|
|
|
=item I<input>
|
|
|
|
Input CSV file. If none is given reads from STDIN (standard input).
|
|
|
|
|
|
=item B<-d> I<delim>
|
|
|
|
Use I<delim> as delimiter in input and output.
|
|
|
|
|
|
=item B<-b> I<blocksize>
|
|
|
|
Pass chunks of I<blocksize> bytes to the internal transposer. Memory
|
|
usage will be 10 times I<blocksiz> per CPU core. Default is 10M.
|
|
|
|
|
|
=back
|
|
|
|
|
|
=head1 EXAMPLES
|
|
|
|
=head2 EXAMPLE: Transpose a medium sized TSV file
|
|
|
|
cat medium.tsv | transpose -d '\t' > muidem.tsv
|
|
|
|
=head1 DESIGN
|
|
|
|
B<transpose> is designed to deal efficiently with medium sized data
|
|
(up to 30 TB per file) on systems with 100 MB RAM per CPU core. It
|
|
works by chopping the input into 10 MB blocks. Each block is
|
|
transposed in parallel and saved to disk. Then these files are pasted
|
|
together and finally removed.
|
|
|
|
=head1 REPORTING BUGS
|
|
|
|
Report bugs to <tange@gnu.org>.
|
|
|
|
|
|
=head1 AUTHOR
|
|
|
|
Copyright (C) 2013-2018 Ole Tange, http://ole.tange.dk and Free
|
|
Software Foundation, Inc.
|
|
|
|
|
|
=head1 LICENSE
|
|
|
|
Copyright (C) 2013 Free Software Foundation, Inc.
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 3 of the License, or
|
|
at your option any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
=head2 Documentation license I
|
|
|
|
Permission is granted to copy, distribute and/or modify this documentation
|
|
under the terms of the GNU Free Documentation License, Version 1.3 or
|
|
any later version published by the Free Software Foundation; with no
|
|
Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
|
|
Texts. A copy of the license is included in the file fdl.txt.
|
|
|
|
=head2 Documentation license II
|
|
|
|
You are free:
|
|
|
|
=over 9
|
|
|
|
=item B<to Share>
|
|
|
|
to copy, distribute and transmit the work
|
|
|
|
=item B<to Remix>
|
|
|
|
to adapt the work
|
|
|
|
=back
|
|
|
|
Under the following conditions:
|
|
|
|
=over 9
|
|
|
|
=item B<Attribution>
|
|
|
|
You must attribute the work in the manner specified by the author or
|
|
licensor (but not in any way that suggests that they endorse you or
|
|
your use of the work).
|
|
|
|
=item B<Share Alike>
|
|
|
|
If you alter, transform, or build upon this work, you may distribute
|
|
the resulting work only under the same, similar or a compatible
|
|
license.
|
|
|
|
=back
|
|
|
|
With the understanding that:
|
|
|
|
=over 9
|
|
|
|
=item B<Waiver>
|
|
|
|
Any of the above conditions can be waived if you get permission from
|
|
the copyright holder.
|
|
|
|
=item B<Public Domain>
|
|
|
|
Where the work or any of its elements is in the public domain under
|
|
applicable law, that status is in no way affected by the license.
|
|
|
|
=item B<Other Rights>
|
|
|
|
In no way are any of the following rights affected by the license:
|
|
|
|
=over 2
|
|
|
|
=item *
|
|
|
|
Your fair dealing or fair use rights, or other applicable
|
|
copyright exceptions and limitations;
|
|
|
|
=item *
|
|
|
|
The author's moral rights;
|
|
|
|
=item *
|
|
|
|
Rights other persons may have either in the work itself or in
|
|
how the work is used, such as publicity or privacy rights.
|
|
|
|
=back
|
|
|
|
=back
|
|
|
|
=over 9
|
|
|
|
=item B<Notice>
|
|
|
|
For any reuse or distribution, you must make clear to others the
|
|
license terms of this work.
|
|
|
|
=back
|
|
|
|
A copy of the full license is included in the file as cc-by-sa.txt.
|
|
|
|
=head1 DEPENDENCIES
|
|
|
|
B<transpose> uses Perl, B<paste>, B<bash> and B<parallel>.
|
|
|
|
|
|
=head1 SEE ALSO
|
|
|
|
B<bash>(1), B<parallel>(1), B<paste>(1)
|
|
|
|
=cut
|
|
|
|
|
|
# transpose [-d delimiter] [-b blocksize] table.csv > transposed.csv
|
|
# cat table.csv | transpose [-d delimiter] [-b blocksize] > transposed.csv
|
|
|
|
transpose_inner() {
|
|
# simple in-memory transpose
|
|
# -d sep
|
|
# Input:
|
|
# data to be transposed
|
|
# Output:
|
|
# transposed data
|
|
perl <(cat <<'cut-here-UbsAqi0j6GoOuk5W5yWA'
|
|
use Text::CSV;
|
|
use Getopt::Long;
|
|
|
|
Getopt::Long::Configure("bundling","require_order");
|
|
my $retval = GetOptions("debug|D=s" => \$opt::debug,
|
|
"delimiter|d=s" => \$opt::delimiter,
|
|
"verbose|v" => \@opt::verbose,
|
|
"simple|s" => \$opt::simple,
|
|
);
|
|
|
|
if(defined $opt::delimiter) {
|
|
simple();
|
|
} else {
|
|
die("-d must be set");
|
|
}
|
|
|
|
sub simple {
|
|
my (@table);
|
|
my $col = 0;
|
|
my $csv_setting = { binary => 1, sep_char => $opt::delimiter };
|
|
my $sep = $csv_setting->{sep_char};
|
|
my $csv = Text::CSV->new($csv_setting)
|
|
or die "Cannot use CSV: ".Text::CSV->error_diag ();
|
|
|
|
while(my $l = <>) {
|
|
if(not $csv->parse($l)) {
|
|
die "CSV has unexpected format";
|
|
}
|
|
# append to each row
|
|
my $row = 0;
|
|
for($csv->fields()) {
|
|
$table[$row][$col] = defined($_) ? $_ : '';
|
|
$row++;
|
|
}
|
|
$col++;
|
|
}
|
|
print map { join($sep,@$_),"\n" } @table;
|
|
}
|
|
cut-here-UbsAqi0j6GoOuk5W5yWA
|
|
) "$@"
|
|
}
|
|
export -f transpose_inner
|
|
|
|
stdin_to_paste_files() {
|
|
# Run transpose_inner on blocks from stdin
|
|
# output each block as file name
|
|
local block_size
|
|
local sep
|
|
block_size="$1"
|
|
sep="$2"
|
|
PARALLEL="-k --files --block $block_size" \
|
|
parallel --pipe transpose_inner -d "'$sep'"
|
|
}
|
|
|
|
file_to_paste_files() {
|
|
# Run transpose_inner on blocks from $file
|
|
# output each block as file name
|
|
local block_size
|
|
local sep
|
|
block_size="$1"
|
|
sep="$2"
|
|
file="$3"
|
|
PARALLEL="-k --files --block $block_size" \
|
|
parallel --pipe-part -a "$file" transpose_inner -d "'$sep'"
|
|
}
|
|
|
|
super_paste() {
|
|
# Like 'paste' up to 1000000 files
|
|
# More than 250000 files requires extra filehandles for GNU Parallel
|
|
# The files are read from stdin
|
|
local sep
|
|
local paste_files
|
|
local fifo
|
|
|
|
other_commands() {
|
|
printf "\rSIGINT caught "
|
|
ls -l $paste_files
|
|
cat $paste_files | parallel "eval rm -f {} $fifo{0#}"
|
|
rm $paste_files
|
|
}
|
|
|
|
trap 'other_commands' SIGINT
|
|
|
|
sep="$1"
|
|
paste_files=`tempfile`
|
|
# basename
|
|
fifo=`tempfile`
|
|
rm $fifo
|
|
# Group files from stdin in groups of 1000 files
|
|
parallel -k -n1000 echo > $paste_files
|
|
|
|
# Define replacement string {0#} to 0-pad job number
|
|
export PARALLEL="--rpl "\''{0#} $f=1+int("".(log(total_jobs())/log(10)));
|
|
$_=sprintf("%0${f}d",seq())'\'
|
|
|
|
# Make fifos that can be read from
|
|
cat $paste_files | parallel "rm -f $fifo{0#}; mkfifo $fifo{0#}"
|
|
|
|
# Start a paste process for every 1000 files
|
|
cat $paste_files | parallel -j0 "eval paste -d \''$sep'\' {} > $fifo{0#}" &
|
|
|
|
# Paste all the fifos
|
|
eval paste -d "'$sep'" $fifo*
|
|
|
|
# Cleanup
|
|
cat $paste_files | parallel "eval rm -f {} $fifo{0#}"
|
|
rm $paste_files
|
|
}
|
|
|
|
stdin_detect_sep() {
|
|
# Read the first 3 lines and detect the separator
|
|
# Save the read input to file
|
|
local file
|
|
file="$1"
|
|
# TODO
|
|
echo "$d"
|
|
}
|
|
|
|
usage() {
|
|
echo "Usage: $0 [-d delimiter] [-b blocksize]" 1>&2; exit 1;
|
|
}
|
|
|
|
main() {
|
|
block_size=10M
|
|
while getopts ":b:d:" o; do
|
|
case "${o}" in
|
|
d)
|
|
d="$(printf "${OPTARG}")"
|
|
if [ "'" = "${d}" ] ; then
|
|
echo "Delimiter cannot be '"
|
|
usage
|
|
exit
|
|
fi
|
|
;;
|
|
b)
|
|
block_size="${OPTARG}"
|
|
;;
|
|
*)
|
|
usage
|
|
;;
|
|
esac
|
|
done
|
|
shift $((OPTIND-1))
|
|
|
|
if [ -z "${d}" ] ; then
|
|
d="$(printf "\t")"
|
|
fi
|
|
|
|
file="$@"
|
|
first_lines=`tempfile`
|
|
if [ -z "$file" ]; then
|
|
sep="$(stdin_detect_sep $first_lines)"
|
|
(cat $first_lines; rm $first_lines; cat) |
|
|
stdin_to_paste_files $block_size "$sep" | super_paste "$sep"
|
|
else
|
|
sep="$(stdin_detect_sep < "$file" $first_lines)"
|
|
rm $first_lines
|
|
file_to_paste_files $block_size "$sep" "$file" | super_paste "$sep"
|
|
fi
|
|
}
|
|
|
|
# Make sure the whole file is read before starting
|
|
main "$@"
|