#!/usr/bin/perl -w =head1 NAME sql - execute a command on a database determined by a dburl =head1 SYNOPSIS B [B<-hnr>] [B<--table-size>] [B<--db-size>] [B<-p> I] [B<-s> I] I [I] B<#!/usr/bin/sql> B<--shebang> [options] I =head1 DESCRIPTION GNU B aims to give a simple, unified interface for accessing databases through all the different databases' command line clients. So far the focus has been on giving a common way to specify login information (protocol, username, password, hostname, and port number), size (database and table size), and running queries. The database is addressed using a dburl. If I is left out you will get that database's interactive shell. GNU B is often used in combination with GNU B. =over 9 =item I A DBURL has the following syntax: vendor://[[user][:password]@][host][:port]/[database] See the section DBURL below. =item I The SQL command to run. Example: select 1+2 If you use * or ? remember to quote them, as the shell may otherwise expand them. If no command is given SQL is read from the keyboard or STDIN. Example: echo 'select 1+2' | sql mysql:/// =item B<--db-size> =item B<--dbsize> Size of database. Show the size of the database on disk. =item B<--help> =item B<-h> Print a summary of the options to GNU B and exit. =item B<--html> HTML output. Turn on HTML tabular output. =item B<--show-processlist> =item B<--proclist> =item B<--listproc> Show the list of running queries. =item B<--noheaders> =item B<--no-headers> =item B<-n> Remove headers and footers and print only tuples. Bug in Oracle: it still prints number of rows found. =item B<-p> I The string following -p will be given to the database connection program as arguments. Multiple -p's will be joined with space. Example: I<-p "-U username"> can also be written I<-p -U -p username>. =item B<-r> Try 3 times. Short version of I<--retries 3>. =item B<--retries> I Try I times. If the client program returns with an error, retry the command. Default is I<--retries 1>. =item B<--sep> I =item B<-s> I Field separator. Use I as seperator between columns. =item B<--skip-first-line> Do not use the first line of input (used by GNU B itself when called with B<--shebang>). =item B<--table-size> =item B<--tablesize> Size of tables. Show the size of the tables in the database. =item B<--version> =item B<-V> Print the version GNU B and exit. =item B<--shebang> =item B<-Y> GNU B can be called as a shebang (#!) command as the first line of a script. Like this: #!/usr/bin/sql -Y mysql:/// select * from users; For this to work B<--shebang> or B<-Y> must be set as the first option. =back =head1 DBURL A DBURL has the following syntax: vendor://[user[:password]@][host][:port]/[database] Examples: mysql://user:pa55w0rd@mysqlserver/database oracle://scott:tiger@oracleserver/xe postgresql://user:pa55w0rd@pg.example.com/db pg:/// Currently supported vendors: MySQL (mysql), MySQL with SSL (mysqls, mysqlssl), Oracle (oracle, ora), PostgreSQL (postgresql, pg, pgsql, postgres), PostgreSQL with SSL (postgresqlssl, pgs, pgsqlssl, postgresssl, pgssl, postgresqls, pgsqls, postgress) Aliases must start with ':' and are read from /etc/sql/aliases and ~/.sql/aliases. The user's own ~/.sql/aliases should only be readable by the user. Example of aliases: :myalias1 pg://user:pa55w0rd@pg.example.com/db :myalias2 ora://scott:tiger@ora1.example.com/xe # Short form of mysql://`whoami`:nopassword@localhost:3306/`whoami` :myalias3 mysql:/// # Short form of mysql://`whoami`:nopassword@localhost:33333/mydb :myalias4 mysql://:33333/mydb # Alias for an alias :m :myalias4 =head1 EXAMPLES =head2 Copy a PostgreSQL database pg_dump my_database | sql pg://user:pass@pgserver/my_new_db =head2 Empty all tables in a MySQL database sql -n mysql:/// 'show tables' | parallel sql mysql:/// delete from {}; =head2 Drop all tables in a PostgreSQL database sql -n pg:/// '\dt' | awk '{print $3}' | parallel -r sql pg:/// drop table {}; =head2 Run as a script Create a script called I: #!/usr/bin/sql -Y mysql:/// select * from users; Then do: chmod 755 demosql; ./demosql =head2 Use --colsep to process multiple columns Use GNU B's B<--colsep> to separate columns: sql :mydburl 'select * from bar' | parallel --colsep '\t' do_stuff {4} {1} =head1 REPORTING BUGS GNU B is part of GNU B. Report bugs to . =head1 AUTHOR Copyright (C) 2008,2009,2010 Ole Tange http://ole.tange.dk =head1 LICENSE Copyright (C) 2007,2008,2009,2010 Free Software Foundation, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or at your option any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . =head2 Documentation license I Permission is granted to copy, distribute and/or modify this documentation under the terms of the GNU Free Documentation License, Version 1.3 or any later version published by the Free Software Foundation; with no Invariant Sections, with no Front-Cover Texts, and with no Back-Cover Texts. A copy of the license is included in the file fdl.txt. =head2 Documentation license II You are free: =over 9 =item B to copy, distribute and transmit the work =item B to adapt the work =back Under the following conditions: =over 9 =item B You must attribute the work in the manner specified by the author or licensor (but not in any way that suggests that they endorse you or your use of the work). =item B If you alter, transform, or build upon this work, you may distribute the resulting work only under the same, similar or a compatible license. =back With the understanding that: =over 9 =item B Any of the above conditions can be waived if you get permission from the copyright holder. =item B Where the work or any of its elements is in the public domain under applicable law, that status is in no way affected by the license. =item B In no way are any of the following rights affected by the license: =over 9 =item * Your fair dealing or fair use rights, or other applicable copyright exceptions and limitations; =item * The author's moral rights; =item * Rights other persons may have either in the work itself or in how the work is used, such as publicity or privacy rights. =back =item B For any reuse or distribution, you must make clear to others the license terms of this work. =back A copy of the full license is included in the file as cc-by-sa.txt. =head1 DEPENDENCIES GNU B uses Perl. If B is installed, MySQL dburls will work. If B is installed, PostgreSQL dburls will work. If B is installed, Oracle dburls will work. If B is installed, GNU B will have a command history for Oracle. =head1 FILES ~/.sql/aliases - user's own aliases with DBURLs /etc/sql/aliases - common aliases with DBURLs =head1 SEE ALSO B(1), B(1), B(1), B(1) =cut use Getopt::Long; use strict; parse_options(); my $pass_through_options = (defined $::opt_p) ? join(" ",@{$::opt_p}) : ""; my $dburl_or_alias = shift; if (not defined $dburl_or_alias) { Usage("No DBURL given"); exit -1; } my %dburl = parse_dburl(get_alias($dburl_or_alias)); my $interactive_command; my $batch_command; my $database_driver = database_driver_alias($dburl{'databasedriver'}); if($database_driver eq "mysql" or $database_driver eq "mysqlssl") { ($batch_command,$interactive_command) = mysql_commands($database_driver,%dburl); } elsif($database_driver eq "postgresql" or $database_driver eq "postgresqlssl") { ($batch_command,$interactive_command) = postgresql_commands($database_driver,%dburl); } elsif($database_driver eq "oracle") { ($batch_command,$interactive_command) = oracle_commands($database_driver,%dburl); } elsif($database_driver eq "sqlite" or $database_driver eq "sqlite3") { ($batch_command,$interactive_command) = sqlite_commands($database_driver,%dburl); } my $err; my $retries; $retries ||= defined $::opt_retries ? $::opt_retries : undef; $retries ||= defined $::opt_retry ? $::opt_retry * 3 : undef; $retries ||= 1; if(defined $::opt_processlist) { unshift @ARGV, processlist($database_driver,%dburl); } if(defined $::opt_dbsize) { unshift @ARGV, dbsize($database_driver,%dburl); } if(defined $::opt_tablesize) { unshift @ARGV, tablesize($database_driver,%dburl); } if($dburl{'statement'}) { unshift @ARGV,$dburl{'statement'}; } do { if(@ARGV) { $::opt_debug and print "$batch_command\n"; open(M,"|$batch_command") || die("mysql/psql/sqlplus not in path"); print M "@ARGV"; close M; } elsif (is_stdin_terminal()) { $::opt_debug and print "$interactive_command\n"; system("$interactive_command"); } else { $::opt_debug and print "$batch_command\n"; system("$batch_command"); } $err = $?>>8; } while (--$retries and $err); $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { $Global::version = 20100906; $Global::progname = 'sql'; # This must be done first as this may exec myself if(defined $ARGV[0] and ($ARGV[0]=~/^-Y/ or $ARGV[0]=~/^--shebang /)) { # Program is called from #! line in script $ARGV[0]=~s/^-Y //; # remove -Y if on its own $ARGV[0]=~s/^-Y/-/; # remove -Y if bundled with other options $ARGV[0]=~s/^--shebang //; # remove --shebang if it is set my $argfile = pop @ARGV; # exec myself to split @ARGV into separate fields exec "$0 --skip-first-line < $argfile @ARGV"; } Getopt::Long::Configure ("bundling","require_order"); GetOptions("passthrough|p=s@" => \$::opt_p, "sep|s=s" => \$::opt_s, "html" => \$::opt_html, "show-processlist|proclist|listproc" => \$::opt_processlist, "db-size|dbsize" => \$::opt_dbsize, "table-size|tablesize" => \$::opt_tablesize, "noheaders|no-headers|n" => \$::opt_n, "r" => \$::opt_retry, "retries=s" => \$::opt_retries, "debug" => \$::opt_debug, # Shebang #!/usr/bin/parallel -Yotheroptions "Y|shebang" => \$::opt_shebang, "skip-first-line" => \$::opt_skip_first_line, # GNU requirements "help|h" => \$::opt_help, "version|V" => \$::opt_version, ) || die_usage(); if(defined $::opt_help) { die_usage(); } if(defined $::opt_version) { version(); exit(0); } } sub database_driver_alias { my $driver = shift; my %database_driver_alias = ("mysql" => "mysql", "mysqls" => "mysqlssl", "mysqlssl" => "mysqlssl", "oracle" => "oracle", "ora" => "oracle", "oracles" => "oraclessl", "oras" => "oraclessl", "oraclessl" => "oraclessl", "orassl" => "oraclessl", "postgresql" => "postgresql", "pgsql" => "postgresql", "postgres" => "postgresql", "pg" => "postgresql", "postgresqlssl" => "postgresqlssl", "pgsqlssl" => "postgresqlssl", "postgresssl" => "postgresqlssl", "pgssl" => "postgresqlssl", "postgresqls" => "postgresqlssl", "pgsqls" => "postgresqlssl", "postgress" => "postgresqlssl", "pgs" => "postgresqlssl", "sqlite" => "sqlite", "sqlite2" => "sqlite", "sqlite3" => "sqlite3", ); return $database_driver_alias{$driver}; } sub mysql_commands { my ($database_driver,%opt) = (@_); find_command_in_path("mysql") || die ("mysql not in path"); if(defined($::opt_s)) { die "Field separator not implemented for mysql" } my $password = defined($opt{'password'}) ? "--password=".$opt{'password'} : ""; my $host = defined($opt{'host'}) ? "--host=".$opt{'host'} : ""; my $port = defined($opt{'port'}) ? "--port=".$opt{'port'} : ""; my $user = defined($opt{'user'}) ? "--user=".$opt{'user'} : ""; my $database = defined($opt{'database'}) ? $opt{'database'} : ""; my $html = defined($::opt_html) ? "--html" : ""; my $no_headers = defined($::opt_n) ? "--skip-column-names" : ""; my $ssl = ""; if ($database_driver eq "mysqlssl") { $ssl="--ssl"; } # -C: Compression if both ends support it $batch_command = "mysql -C $pass_through_options $no_headers $html $ssl $host $user $port $password $database"; $interactive_command = $batch_command; return($batch_command,$interactive_command); } sub postgresql_commands { my ($database_driver,%opt) = (@_); find_command_in_path("psql") || die ("psql not in path"); my $sep = ($::opt_s) ? "-A --field-separator '$::opt_s'" : ""; my $password = defined($opt{'password'}) ? "PGPASSWORD=".$opt{'password'} : ""; my $host = defined($opt{'host'}) ? "-h ".$opt{'host'} : ""; my $port = defined($opt{'port'}) ? "-p ".$opt{'port'} : ""; my $user = defined($opt{'user'}) ? "-U ".$opt{'user'} : ""; my $database = defined($opt{'database'}) ? "-d ".$opt{'database'} : ""; my $html = defined($::opt_html) ? "--html" : ""; my $no_headers = defined($::opt_n) ? "--tuples-only" : ""; my $ssl = ""; if ($database_driver eq "postgresqlssl") { $ssl="PGSSLMODE=require"; } $batch_command = "$ssl $password psql $pass_through_options $sep $no_headers $html $host $user $port $database"; $interactive_command = $batch_command; return($batch_command,$interactive_command); } sub oracle_commands { my ($database_driver,%opt) = (@_); # oracle://user:pass@grum:1521/XE becomes: # sqlplus 'user/pass@(DESCRIPTION = (ADDRESS = (PROTOCOL = TCP)(HOST = grum)(PORT = 1521)) (CONNECT_DATA =(SERVER = DEDICATED)(SERVICE_NAME = XE) ))' find_command_in_path("sqlplus") || die ("sqlplus not in path"); # Readline support: if rlwrap installed run rlwrap sqlplus my $rlwrap = find_command_in_path("rlwrap"); # Set good defaults in the inifile $Global::Initfile = "/tmp/$$.sql.init"; open(INIT,">".$Global::Initfile) || die; print INIT "SET LINESIZE 32767\n"; $::opt_debug and print "SET LINESIZE 32767\n"; print INIT "SET WRAP OFF\n"; $::opt_debug and print "SET WRAP OFF\n"; if(defined($::opt_html)) { print INIT "SET MARK HTML ON\n"; $::opt_debug and print "SET MARK HTML ON\n"; } if(defined($::opt_n)) { print INIT "SET PAGES 0\n"; $::opt_debug and print "SET PAGES 0\n"; } else { print INIT "SET PAGES 50000\n"; $::opt_debug and print "SET PAGES 50000\n"; } if(defined($::opt_s)) { print INIT "SET COLSEP $::opt_s\n"; $::opt_debug and print "SET COLSEP $::opt_s\n"; } close INIT; my $password = defined($opt{'password'}) ? "/".$opt{'password'} : ""; my $host = defined($opt{'host'}) ? $opt{'host'} : "localhost"; my $port = defined($opt{'port'}) ? $opt{'port'} : "1521"; my $user = defined($opt{'user'}) ? $opt{'user'} : ""; # Database is called service in Oracle lingo my $service = defined($opt{'database'}) ? "(SERVICE_NAME = ".$opt{'database'}.")" : ""; my $tns = "(DESCRIPTION = (ADDRESS = (PROTOCOL = TCP)(HOST = $host)(PORT = $port)) ". "(CONNECT_DATA =(SERVER = DEDICATED)$service ))"; my $ssl = ""; # -L: Do not re-ask for password if it is wrong my $common_options = "-L $pass_through_options '$user$password\@$tns' \@$Global::Initfile"; my $batch_command = "sqlplus -S ".$common_options; my $interactive_command = "$rlwrap sqlplus ".$common_options; return($batch_command,$interactive_command); } sub sqlite_commands { my ($database_driver,%opt) = (@_); if(not find_command_in_path($database_driver)) { print STDERR "Database driver '$database_driver' not supported\n"; exit -1; } my $sep = defined($::opt_s) ? "-separator '$::opt_s'" : ""; my $password = defined($opt{'password'}) ? "--password=".$opt{'password'} : ""; my $host = defined($opt{'host'}) ? "--host=".$opt{'host'} : ""; my $port = defined($opt{'port'}) ? "--port=".$opt{'port'} : ""; my $user = defined($opt{'user'}) ? "--user=".$opt{'user'} : ""; my $database = defined($opt{'database'}) ? $opt{'database'} : ""; my $html = defined($::opt_html) ? "-html" : ""; my $no_headers = defined($::opt_n) ? "-noheader" : "-header"; my $ssl = ""; $batch_command = "$database_driver $pass_through_options $sep $no_headers $html $database"; $interactive_command = $batch_command; return($batch_command,$interactive_command); } # Return the code for 'show processlist' in the chosen database dialect sub processlist { my $dbdriver = shift; my %dburl = @_; my %processlist = ("mysql" => "show processlist;", "postgresql" => ("SELECT ". " datname AS database,". " usename AS username,". " now()-xact_start AS runtime,". " current_query ". "FROM pg_stat_activity ". "ORDER BY runtime DESC;"), "oracle" => ("SELECT ". ' CPU_TIME/100000, SYS.V_$SQL.SQL_TEXT, USERNAME '. "FROM ". ' SYS.V_$SQL, SYS.V_$SESSION '. "WHERE ". ' SYS.V_$SQL.SQL_ID = SYS.V_$SESSION.SQL_ID(+) '. "AND username IS NOT NULL ". "ORDER BY CPU_TIME DESC;"), ); if($processlist{$dbdriver}) { return $processlist{$dbdriver}; } else { print STDERR "processlist is not implemented for $dbdriver\n"; exit 1; } } # Return the code for 'show database size' in the chosen database dialect sub dbsize { my $dbdriver = shift; my %dburl = @_; my %processlist = ("mysql" => ( 'SELECT '. ' table_schema "Data Base Name", '. ' SUM(data_length+index_length)/1024/1024 "Data Base Size in MB" '. 'FROM information_schema.TABLES '. 'GROUP BY table_schema;'), "postgresql" => ( "SELECT ". " pg_database_size('".$dburl{'database'}."') AS bytes,". " pg_size_pretty(pg_database_size('".$dburl{'database'}."')) AS human_readable;"), "sqlite" => ( "SELECT ".(undef_as_zero(-s $dburl{'database'}))." AS bytes;"), "sqlite3" => ( "SELECT ".(undef_as_zero(-s $dburl{'database'}))." AS bytes;"), ); if($processlist{$dbdriver}) { return $processlist{$dbdriver}; } else { print STDERR "dbsize is not implemented for $dbdriver\n"; exit 1; } } # Return the code for 'show database size' in the chosen database dialect sub tablesize { my $dbdriver = shift; my $database = shift; my %processlist = ("postgresql" => ( "SELECT relname, relpages*8 AS kb, reltuples::int AS \"live+dead rows\" ". "FROM pg_class c ". "ORDER BY relpages DESC;"), "mysql" => ( "select table_name, TABLE_ROWS, DATA_LENGTH,INDEX_LENGTH from INFORMATION_SCHEMA.tables;"), ); if($processlist{$dbdriver}) { return $processlist{$dbdriver}; } else { print STDERR "table size is not implemented for $dbdriver\n"; exit 1; } } sub is_stdin_terminal { return (-t STDIN); } sub find_command_in_path { # Find the command if it exists in the current path my $command = shift; my $path = `which $command`; chomp $path; return $path; } sub Usage { if(@_) { print "Error:\n"; print map{ "$_\n" } @_; print "\n"; } print "sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command]\n"; } sub get_alias { my $alias = shift; if ($alias !~ /^:/) { return $alias; } # Find the alias my $path; if (-l $0) { ($path) = readlink($0) =~ m|^(.*)/|; } else { ($path) = $0 =~ m|^(.*)/|; } my @deprecated = ("$ENV{HOME}/.dburl.aliases", "$path/dburl.aliases", "$path/dburl.aliases.dist"); for (@deprecated) { if(-r $_) { print STDERR "$_ is deprecated. Use .sql/aliases instead (read man sql)\n"; } } my @urlalias=(); check_permissions("$ENV{HOME}/.sql/aliases"); check_permissions("$ENV{HOME}/.dburl.aliases"); my @search = ("$ENV{HOME}/.sql/aliases", "$ENV{HOME}/.dburl.aliases", "/etc/sql/aliases", "$path/dburl.aliases", "$path/dburl.aliases.dist"); for my $alias_file (@search) { if(-r $alias_file) { push @urlalias, `cat "$alias_file"`; } } my ($alias_part,$rest) = $alias=~/(:\w+)(.*)/; my $dburl; for (@urlalias) { /^$alias_part\s+(\S+)/ and do { $dburl = $1; last; } } if($dburl) { return get_alias($dburl.$rest); } else { Usage("$alias is not defined in @search"); } } sub check_permissions { my $file = shift; if(-e $file) { if(not -o $file) { my $username = (getpwuid($<))[0]; print STDERR "$file should be owned by $username: chown $username $file\n"; } my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, $atime,$mtime,$ctime,$blksize,$blocks) = stat($file); if($mode & 077) { my $username = (getpwuid($<))[0]; print STDERR "$file should be only be readable by $username: chmod 600 $file\n"; } } } sub parse_dburl { my $url = shift; my %options = (); # mysql://[[user][:password]@][host][:port]/[database[?sql statement]] if($url=~m!((?:oracle|ora|mysql|pg|postgres|postgresql)(?:s|ssl|)| (?:sqlite|sqlite2|sqlite3)):// # Databasedriver ($1) (?: ([^:@]*) # Username ($2) (?: :([^@]*) # Password ($3) )? @)? ([^:/]*)? # Hostname ($4) (?: : ([^/]*)? # Port ($5) )? (?: / ([^?]*)? # Database ($6) )? (?: \? (.*)? # Statement ($7) )? !x) { $options{databasedriver} = undef_if_empty($1); $options{user} = undef_if_empty($2); $options{password} = undef_if_empty($3); $options{host} = undef_if_empty($4); $options{port} = undef_if_empty($5); $options{database} = undef_if_empty($6); $options{statement} = undef_if_empty($7); } else { Usage("$url is not a valid DBURL"); exit -1; } return %options; } sub undef_if_empty { if(defined($_[0]) and $_[0] eq "") { return undef; } return $_[0]; } sub undef_as_zero { my $a = shift; return $a ? $a : 0; } sub version { # Returns: N/A print join("\n", "GNU $Global::progname $Global::version", "Copyright (C) 2009,2010 Ole Tange and Free Software Foundation, Inc.", "License GPLv3+: GNU GPL version 3 or later ", "This is free software: you are free to change and redistribute it.", "GNU $Global::progname comes with no warranty.", "", "Web site: http://www.gnu.org/software/${Global::progname}\n" ); } sub die_usage { # Returns: N/A usage(); exit(255); } sub usage { # Returns: N/A print "Usage:\n"; print "$Global::progname [options] dburl [sqlcommand]\n"; print "$Global::progname [options] dburl < sql_command_file\n"; print "\n"; print "See 'man $Global::progname' for the options\n"; } # TODO --list-databases: psql -l eller '\l'. mysql show databases. Oracle ? $::opt_skip_first_line = $::opt_shebang = 0;