From 7b337a8b913d9334de0951eb29a08d798b4b98f5 Mon Sep 17 00:00:00 2001 From: Karl Southern Date: Sun, 29 May 2016 13:40:47 +0100 Subject: [PATCH] Backport functionality from v5 branch. --- .gitignore | 1 + CHANGELOG.md | 4 +++ lib/logstash/outputs/jdbc.rb | 60 +++++++++++++++++++++++++++++++++--- logstash-output-jdbc.gemspec | 2 +- spec/outputs/jdbc_spec.rb | 5 +-- 5 files changed, 63 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 3762b35..b3bcc90 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ Gemfile.lock Gemfile.bak .bundle +.vagrant diff --git a/CHANGELOG.md b/CHANGELOG.md index 176fe47..1b06204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Change Log All notable changes to this project will be documented in this file, from 0.2.0. +## [0.2.7.rc1] - 2016-05-29 + - Backport retry exception logic from v5 branch + - Backport improved timestamp compatibility from v5 branch + ## [0.2.6] - 2016-05-02 - Fix for exception infinite loop diff --git a/lib/logstash/outputs/jdbc.rb b/lib/logstash/outputs/jdbc.rb index 7009756..09844c5 100644 --- a/lib/logstash/outputs/jdbc.rb +++ b/lib/logstash/outputs/jdbc.rb @@ -6,10 +6,33 @@ require "java" require "logstash-output-jdbc_jars" require "logstash-output-jdbc_ring-buffer" +# Write events to a SQL engine, using JDBC. +# +# It is upto the user of the plugin to correctly configure the plugin. This +# includes correctly crafting the SQL statement, and matching the number of +# parameters correctly. class LogStash::Outputs::Jdbc < LogStash::Outputs::Base # Adds buffer support include Stud::Buffer + STRFTIME_FMT = '%Y-%m-%d %T.%L'.freeze + + RETRYABLE_SQLSTATE_CLASSES = [ + # Classes of retryable SQLSTATE codes + # Not all in the class will be retryable. However, this is the best that + # we've got right now. + # If a custom state code is required, set it in retry_sql_states. + '08', # Connection Exception + '24', # Invalid Cursor State (Maybe retry-able in some circumstances) + '25', # Invalid Transaction State + '40', # Transaction Rollback + '53', # Insufficient Resources + '54', # Program Limit Exceeded (MAYBE) + '55', # Object Not In Prerequisite State + '57', # Operator Intervention + '58', # System Error + ].freeze + config_name "jdbc" # Driver class - Reintroduced for https://github.com/theangryangel/logstash-output-jdbc/issues/26 @@ -196,8 +219,15 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base def safe_flush(events, teardown=false) connection = nil statement = nil + begin connection = @pool.getConnection() + rescue => e + log_jdbc_exception(e) + raise + end + + begin statement = connection.prepareStatement(@statement[0]) events.each do |event| @@ -213,6 +243,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base @exceptions_tracker << nil rescue => e log_jdbc_exception(e) + if retry_exception?(e) + raise + end ensure statement.close() unless statement.nil? connection.close() unless connection.nil? @@ -224,7 +257,12 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base statement = nil begin connection = @pool.getConnection() + rescue => e + log_jdbc_exception(e) + raise + end + begin events.each do |event| next if event.cancelled? @@ -240,6 +278,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base end rescue => e log_jdbc_exception(e) + if retry_exception?(e) + raise + end ensure statement.close() unless statement.nil? connection.close() unless connection.nil? @@ -250,11 +291,17 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base @statement[1..-1].each_with_index do |i, idx| case event[i] when Time - # Most reliable solution, cross JDBC driver - statement.setString(idx + 1, event[i].iso8601()) + # See LogStash::Timestamp, below, for the why behind strftime. + statement.setString(idx + 1, event[i].strftime(STRFTIME_FMT)) when LogStash::Timestamp - # Most reliable solution, cross JDBC driver - statement.setString(idx + 1, event[i].to_iso8601()) + # XXX: Using setString as opposed to setTimestamp, because setTimestamp + # doesn't behave correctly in some drivers (Known: sqlite) + # + # Additionally this does not use `to_iso8601`, since some SQL databases + # choke on the 'T' in the string (Known: Derby). + # + # strftime appears to be the most reliable across drivers. + statement.setString(idx + 1, event[i].time.strftime(STRFTIME_FMT)) when Fixnum, Integer statement.setInt(idx + 1, event[i]) when Float @@ -285,4 +332,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base break if current_exception == nil end end + + def retry_exception?(exception) + return (exception.class != java.sql.SQLException or + RETRYABLE_SQLSTATE_CLASSES.include?(e.getSQLState[0,2])) + end end # class LogStash::Outputs::jdbc diff --git a/logstash-output-jdbc.gemspec b/logstash-output-jdbc.gemspec index d6df8d5..8706f3d 100644 --- a/logstash-output-jdbc.gemspec +++ b/logstash-output-jdbc.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-jdbc' - s.version = "0.2.6" + s.version = "0.2.7.rc1" s.licenses = [ "Apache License (2.0)" ] s.summary = "This plugin allows you to output to SQL, via JDBC" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/outputs/jdbc_spec.rb b/spec/outputs/jdbc_spec.rb index fd783b2..9ca082f 100644 --- a/spec/outputs/jdbc_spec.rb +++ b/spec/outputs/jdbc_spec.rb @@ -10,10 +10,7 @@ describe LogStash::Outputs::Jdbc do "driver_class" => "org.apache.derby.jdbc.EmbeddedDriver", "connection_string" => "jdbc:derby:memory:testdb;create=true", "driver_jar_path" => ENV['JDBC_DERBY_JAR'], - # Grumble. Grumble. - # Derby doesn't like 'T' in timestamps as of current writing, so for now - # we'll just use CURRENT_TIMESTAMP as opposed to the event @timestamp - "statement" => [ "insert into log (created_at, message) values(CURRENT_TIMESTAMP, ?)", "message" ] + "statement" => [ "insert into log (created_at, message) values(?, ?)", "@timestamp" "message" ] } end