From ae51d77f0550134fe0969423b10fb927953ae3ec Mon Sep 17 00:00:00 2001 From: Karl Southern Date: Wed, 30 Dec 2015 12:04:23 +0000 Subject: [PATCH] Move examples and split up connection code Bump version --- README.md | 118 +-------------------------- examples/apache-phoenix-hbase-sql.md | 16 ++++ examples/mariadb.md | 16 ++++ examples/mysql.md | 16 ++++ examples/odps.md | 20 +++++ examples/oracle.md | 16 ++++ examples/postgres.md | 15 ++++ examples/sql-server.md | 14 ++++ examples/sqlite.md | 17 ++++ lib/logstash/outputs/jdbc.rb | 38 ++++++--- logstash-output-jdbc.gemspec | 2 +- 11 files changed, 158 insertions(+), 130 deletions(-) create mode 100644 examples/apache-phoenix-hbase-sql.md create mode 100644 examples/mariadb.md create mode 100644 examples/mysql.md create mode 100644 examples/odps.md create mode 100644 examples/oracle.md create mode 100644 examples/postgres.md create mode 100644 examples/sql-server.md create mode 100644 examples/sqlite.md diff --git a/README.md b/README.md index e1f1cb2..805ff04 100644 --- a/README.md +++ b/README.md @@ -59,120 +59,6 @@ Tests are not yet 100% complete. | max_flush_exceptions | Number | Number of sequential flushes which cause an exception, before we stop logstash. Set to a value less than 1 if you never want it to stop. This should be carefully configured with relation to idle_flush_time if your SQL instance is not highly available. | No | 0 | ## Example configurations +Example logstash configurations, can now be found in the examples directory. Where possible we try to link every configuration with a tested jar. + If you have a working sample configuration, for a DB thats not listed, pull requests are welcome. - -### SQLite3 - * Tested using https://bitbucket.org/xerial/sqlite-jdbc - * SQLite setup - `echo "CREATE table log (host text, timestamp datetime, message text);" | sqlite3 test.db` -``` -input -{ - stdin { } -} -output { - stdout { } - - jdbc { - connection_string => 'jdbc:sqlite:test.db' - statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ] - } -} -``` - -### SQL Server - * Tested using http://msdn.microsoft.com/en-gb/sqlserver/aa937724.aspx -``` -input -{ - stdin { } -} -output { - jdbc { - connection_string => "jdbc:sqlserver://server:1433;databaseName=databasename;user=username;password=password;autoReconnect=true;" - statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ] - } -} -``` - -### Postgres -With thanks to [@roflmao](https://github.com/roflmao) -``` -input -{ - stdin { } -} -output { - jdbc { - connection_string => 'jdbc:postgresql://hostname:5432/database?user=username&password=password' - statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ] - } -} -``` - -### Oracle -With thanks to [@josemazo](https://github.com/josemazo) - * Tested with Express Edition 11g Release 2 - * Tested using http://www.oracle.com/technetwork/database/enterprise-edition/jdbc-112010-090769.html (ojdbc6.jar) -``` -input -{ - stdin { } -} -output { - jdbc { - connection_string => "jdbc:oracle:thin:USER/PASS@HOST:PORT:SID" - statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ] - } -} -``` - -### Mysql -With thanks to [@jMonsinjon](https://github.com/jMonsinjon) - * Tested with Version 14.14 Distrib 5.5.43, for debian-linux-gnu (x86_64) - * Tested using http://dev.mysql.com/downloads/file.php?id=457911 (mysql-connector-java-5.1.36-bin.jar) -``` -input -{ - stdin { } -} -output { - jdbc { - connection_string => "jdbc:mysql://HOSTNAME/DATABASE?user=USER&password=PASSWORD" - statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ] - } -} -``` - -### MariaDB - * Tested with Ubuntu 14.04.3 LTS, Server version: 10.1.9-MariaDB-1~trusty-log mariadb.org binary distribution - * Tested using https://downloads.mariadb.com/enterprise/tqge-whfa/connectors/java/connector-java-1.3.2/mariadb-java-client-1.3.2.jar (mariadb-java-client-1.3.2.jar) -``` -input -{ - stdin { } -} -output { - jdbc { - connection_string => "jdbc:mariadb://HOSTNAME/DATABASE?user=USER&password=PASSWORD" - statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ] - } - -} -``` - -### Apache Phoenix (HBase SQL) - * Tested with Ubuntu 14.04.03 / Logstash 2.1 / Apache Phoenix 4.6 - * HBase and Zookeeper must be both accessible from logstash machine -``` -input -{ - stdin { } -} -output { - jdbc { - connection_string => "jdbc:phoenix:ZOOKEEPER_HOSTNAME" - statement => [ "UPSERT INTO EVENTS log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ] - } - -} -``` diff --git a/examples/apache-phoenix-hbase-sql.md b/examples/apache-phoenix-hbase-sql.md new file mode 100644 index 0000000..8bfb3fe --- /dev/null +++ b/examples/apache-phoenix-hbase-sql.md @@ -0,0 +1,16 @@ +# Example: Apache Phoenix (HBase SQL) + * Tested with Ubuntu 14.04.03 / Logstash 2.1 / Apache Phoenix 4.6 + * HBase and Zookeeper must be both accessible from logstash machine +``` +input +{ + stdin { } +} +output { + jdbc { + connection_string => "jdbc:phoenix:ZOOKEEPER_HOSTNAME" + statement => [ "UPSERT INTO EVENTS log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ] + } + +} +``` diff --git a/examples/mariadb.md b/examples/mariadb.md new file mode 100644 index 0000000..eaf9856 --- /dev/null +++ b/examples/mariadb.md @@ -0,0 +1,16 @@ +# Example: MariaDB + * Tested with Ubuntu 14.04.3 LTS, Server version: 10.1.9-MariaDB-1~trusty-log mariadb.org binary distribution + * Tested using https://downloads.mariadb.com/enterprise/tqge-whfa/connectors/java/connector-java-1.3.2/mariadb-java-client-1.3.2.jar (mariadb-java-client-1.3.2.jar) +``` +input +{ + stdin { } +} +output { + jdbc { + connection_string => "jdbc:mariadb://HOSTNAME/DATABASE?user=USER&password=PASSWORD" + statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ] + } + +} +``` diff --git a/examples/mysql.md b/examples/mysql.md new file mode 100644 index 0000000..9241d45 --- /dev/null +++ b/examples/mysql.md @@ -0,0 +1,16 @@ +# Example: Mysql +With thanks to [@jMonsinjon](https://github.com/jMonsinjon) + * Tested with Version 14.14 Distrib 5.5.43, for debian-linux-gnu (x86_64) + * Tested using http://dev.mysql.com/downloads/file.php?id=457911 (mysql-connector-java-5.1.36-bin.jar) +``` +input +{ + stdin { } +} +output { + jdbc { + connection_string => "jdbc:mysql://HOSTNAME/DATABASE?user=USER&password=PASSWORD" + statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ] + } +} +``` diff --git a/examples/odps.md b/examples/odps.md new file mode 100644 index 0000000..4650db8 --- /dev/null +++ b/examples/odps.md @@ -0,0 +1,20 @@ +# Example: ODPS +With thanks to [@onesuper](https://github.com/onesuper) +``` +input +{ + stdin { } +} +output { + jdbc { + driver_class => "com.aliyun.odps.jdbc.OdpsDriver" + driver_auto_commit => false + connection_string => "jdbc:odps:http://service.odps.aliyun.com/api?project=meta_dev&loglevel=DEBUG" + username => "abcd" + password => "1234" + max_pool_size => 5 + flush_size => 10 + statement => [ "INSERT INTO test_logstash VALUES(?, ?, ?);", "host", "@timestamp", "message" ] + } +} +``` diff --git a/examples/oracle.md b/examples/oracle.md new file mode 100644 index 0000000..0e33f62 --- /dev/null +++ b/examples/oracle.md @@ -0,0 +1,16 @@ +# Example: Oracle +With thanks to [@josemazo](https://github.com/josemazo) + * Tested with Express Edition 11g Release 2 + * Tested using http://www.oracle.com/technetwork/database/enterprise-edition/jdbc-112010-090769.html (ojdbc6.jar) +``` +input +{ + stdin { } +} +output { + jdbc { + connection_string => "jdbc:oracle:thin:USER/PASS@HOST:PORT:SID" + statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ] + } +} +``` diff --git a/examples/postgres.md b/examples/postgres.md new file mode 100644 index 0000000..39a3a66 --- /dev/null +++ b/examples/postgres.md @@ -0,0 +1,15 @@ +# Example: Postgres +With thanks to [@roflmao](https://github.com/roflmao) +``` +input +{ + stdin { } +} +output { + jdbc { + connection_string => 'jdbc:postgresql://hostname:5432/database?user=username&password=password' + statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ] + } +} +``` + diff --git a/examples/sql-server.md b/examples/sql-server.md new file mode 100644 index 0000000..59064e5 --- /dev/null +++ b/examples/sql-server.md @@ -0,0 +1,14 @@ +# Example: SQL Server + * Tested using http://msdn.microsoft.com/en-gb/sqlserver/aa937724.aspx +``` +input +{ + stdin { } +} +output { + jdbc { + connection_string => "jdbc:sqlserver://server:1433;databaseName=databasename;user=username;password=password;autoReconnect=true;" + statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ] + } +} +``` diff --git a/examples/sqlite.md b/examples/sqlite.md new file mode 100644 index 0000000..2e421bb --- /dev/null +++ b/examples/sqlite.md @@ -0,0 +1,17 @@ +# Example: SQLite3 + * Tested using https://bitbucket.org/xerial/sqlite-jdbc + * SQLite setup - `echo "CREATE table log (host text, timestamp datetime, message text);" | sqlite3 test.db` +``` +input +{ + stdin { } +} +output { + stdout { } + + jdbc { + connection_string => 'jdbc:sqlite:test.db' + statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ] + } +} +``` diff --git a/lib/logstash/outputs/jdbc.rb b/lib/logstash/outputs/jdbc.rb index 1db7d45..ba41064 100644 --- a/lib/logstash/outputs/jdbc.rb +++ b/lib/logstash/outputs/jdbc.rb @@ -83,19 +83,6 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base load_jar_files! - @pool = Java::ComZaxxerHikari::HikariDataSource.new - - @pool.setAutoCommit(@driver_auto_commit) - @pool.setDriverClassName(@driver_class) if @driver_class - - @pool.setJdbcUrl(@connection_string) - - @pool.setUsername(@username) if @username - @pool.setPassword(@password) if @password - - @pool.setMaximumPoolSize(@max_pool_size) - @pool.setConnectionTimeout(@connection_timeout) - @exceptions_tracker = RingBuffer.new(@max_flush_exceptions) if (@flush_size > 1000) @@ -109,6 +96,8 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base if (!@unsafe_statement and @statement.length < 2) @logger.error("JDBC - Statement has no parameters. No events will be inserted into SQL as you're not passing any event data. Likely configuration error.") end + + setup_and_test_pool! buffer_initialize( :max_items => @flush_size, @@ -152,6 +141,29 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base private + def setup_and_test_pool! + # Setup pool + @pool = Java::ComZaxxerHikari::HikariDataSource.new + + @pool.setAutoCommit(@driver_auto_commit) + @pool.setDriverClassName(@driver_class) if @driver_class + + @pool.setJdbcUrl(@connection_string) + + @pool.setUsername(@username) if @username + @pool.setPassword(@password) if @password + + @pool.setMaximumPoolSize(@max_pool_size) + @pool.setConnectionTimeout(@connection_timeout) + + # Test connection + test_connection = @pool.getConnection() + unless test_connection.isValid(10) + @logger.error("JDBC - Connection is not valid. Please check connection string or that your JDBC endpoint is available.") + end + test_connection.close() + end + def load_jar_files! # Load jar from driver path unless @driver_jar_path.nil? diff --git a/logstash-output-jdbc.gemspec b/logstash-output-jdbc.gemspec index cb0b812..f04d1e4 100644 --- a/logstash-output-jdbc.gemspec +++ b/logstash-output-jdbc.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-jdbc' - s.version = "0.2.2.rc1" + s.version = "0.2.2" s.licenses = [ "Apache License (2.0)" ] s.summary = "This plugin allows you to output to SQL, via JDBC" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"