From 093b2bbb9264ad4caf8471bb153342ae810f0d86 Mon Sep 17 00:00:00 2001 From: Karl Southern Date: Sun, 1 Jun 2014 13:16:33 +0100 Subject: [PATCH] Fixes up README and adds tested support to buffering --- README.md | 47 +++++++++++++++++++++++++++++------- lib/logstash/outputs/jdbc.rb | 20 ++++++--------- 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 02e0b91..6cf85c7 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,55 @@ -logstash-jdbc -============= +# logstash-jdbc JDBC output plugin for Logstash. This plugin is provided as an external plugin and is not part of the Logstash project. -Warning -------- +## Warning This has not yet been extensively tested with all JDBC drivers and may not yet work for you. -Installation ------------- +## Installation - Copy lib directory contents into your logstash installation. + - Create the directory vendor/jar/jdbc in your logstash installation (`mkdir -p vendor/jar/jdbc/`) - Add JDBC jar files to vendor/jar/jdbc in your logstash installation - Configure -Example configuration ---------------------- +## Configuration options + * driver_class, string, JDBC driver class to load + * connection_string, string, JDBC connection string + * statement, array, an array of strings representing the SQL statement to run. Index 0 is the SQL statement that is prepared, all other array entries are passed in as parameters (in order). See example configurations below. + * flush_size, number, default = 1000, number of entries to buffer before sending to SQL + * idle_flush_time, number, default = 1, number of idle seconds before sending data to SQL, even if the flush_size has not been reached + +## Example configurations +### SQLite3 + * Tested using https://bitbucket.org/xerial/sqlite-jdbc + * SQLite setup - `echo "CREATE table log (host text, timestamp datetime, message text);" | sqlite3 test.db` ``` +input +{ + stdin { } +} +output { + stdout { } + + jdbc { + driver_class => 'org.sqlite.JDBC' + connection_string => 'jdbc:sqlite:test.db' + statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "%{host}", "%{@timestamp}", "%{message}" ] + } +} +``` + +### SQL Server + * Tested using http://msdn.microsoft.com/en-gb/sqlserver/aa937724.aspx +``` +input +{ + stdin { } +} output { jdbc { driver_class => 'com.microsoft.sqlserver.jdbc.SQLServerDriver' connection_string => "jdbc:sqlserver://server:1433;databaseName=databasename;user=username;password=password;autoReconnect=true;" - statement => [ "INSERT INTO filezilla (host, connection_id, timestamp, username, client, command) VALUES(?, ?, ?, ?, ?, ?)", "%{host}", "%{connection_id}", "%{timestamp}", "%{username}", "%{client}", "%{command}" ] + statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "%{host}", "%{@timestamp}", "%{message}" ] } } ``` diff --git a/lib/logstash/outputs/jdbc.rb b/lib/logstash/outputs/jdbc.rb index 38e3168..9fb1b76 100644 --- a/lib/logstash/outputs/jdbc.rb +++ b/lib/logstash/outputs/jdbc.rb @@ -19,11 +19,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base # [ "insert into table (message) values(?)", "%{message}" ] config :statement, :validate => :array, :required => true - # This plugin uses the bulk index api for improved performance. - # To make efficient bulk insert calls, we will buffer a certain number of - # events before flushing that out to SQL. This setting - # controls how many events will be buffered before sending a batch - # of events. + # We buffer a certain number of events before flushing that out to SQL. + # This setting controls how many events will be buffered before sending a + # batch of events. config :flush_size, :validate => :number, :default => 1000 # The amount of time since last flush before a flush is forced. @@ -83,17 +81,13 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base end begin - @logger.debug("Sending SQL to server", :event => event, :sql => statement.toString()) + @logger.debug("Sending SQL to server", :sql => statement.toString()) statement.executeBatch() - rescue Exception => e - @logger.error("JDBC Exception", :exception => e) - + rescue => e # Raising an exception will incur a retry from Stud::Buffer. # Since the exceutebatch failed this should mean any events failed to be - # inserted will be re-run. - # We're only capturing the exception so we can pass it to the logger, log - # it and then re-raise it. - raise Exception.new("JDBC - Flush failed - #{e.message}") + # inserted will be re-run. We're going to log it for the lols anyway. + @logger.error("JDBC Exception", :exception => e) end statement.close()