Backport functionality from v5 branch.

This commit is contained in:
Karl Southern 2016-05-29 13:40:47 +01:00
parent 927e532b2a
commit 7b337a8b91
5 changed files with 63 additions and 9 deletions

1
.gitignore vendored
View File

@ -2,3 +2,4 @@
Gemfile.lock Gemfile.lock
Gemfile.bak Gemfile.bak
.bundle .bundle
.vagrant

View File

@ -1,6 +1,10 @@
# Change Log # Change Log
All notable changes to this project will be documented in this file, from 0.2.0. All notable changes to this project will be documented in this file, from 0.2.0.
## [0.2.7.rc1] - 2016-05-29
- Backport retry exception logic from v5 branch
- Backport improved timestamp compatibility from v5 branch
## [0.2.6] - 2016-05-02 ## [0.2.6] - 2016-05-02
- Fix for exception infinite loop - Fix for exception infinite loop

View File

@ -6,10 +6,33 @@ require "java"
require "logstash-output-jdbc_jars" require "logstash-output-jdbc_jars"
require "logstash-output-jdbc_ring-buffer" require "logstash-output-jdbc_ring-buffer"
# Write events to a SQL engine, using JDBC.
#
# It is upto the user of the plugin to correctly configure the plugin. This
# includes correctly crafting the SQL statement, and matching the number of
# parameters correctly.
class LogStash::Outputs::Jdbc < LogStash::Outputs::Base class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
# Adds buffer support # Adds buffer support
include Stud::Buffer include Stud::Buffer
STRFTIME_FMT = '%Y-%m-%d %T.%L'.freeze
RETRYABLE_SQLSTATE_CLASSES = [
# Classes of retryable SQLSTATE codes
# Not all in the class will be retryable. However, this is the best that
# we've got right now.
# If a custom state code is required, set it in retry_sql_states.
'08', # Connection Exception
'24', # Invalid Cursor State (Maybe retry-able in some circumstances)
'25', # Invalid Transaction State
'40', # Transaction Rollback
'53', # Insufficient Resources
'54', # Program Limit Exceeded (MAYBE)
'55', # Object Not In Prerequisite State
'57', # Operator Intervention
'58', # System Error
].freeze
config_name "jdbc" config_name "jdbc"
# Driver class - Reintroduced for https://github.com/theangryangel/logstash-output-jdbc/issues/26 # Driver class - Reintroduced for https://github.com/theangryangel/logstash-output-jdbc/issues/26
@ -196,8 +219,15 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
def safe_flush(events, teardown=false) def safe_flush(events, teardown=false)
connection = nil connection = nil
statement = nil statement = nil
begin begin
connection = @pool.getConnection() connection = @pool.getConnection()
rescue => e
log_jdbc_exception(e)
raise
end
begin
statement = connection.prepareStatement(@statement[0]) statement = connection.prepareStatement(@statement[0])
events.each do |event| events.each do |event|
@ -213,6 +243,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
@exceptions_tracker << nil @exceptions_tracker << nil
rescue => e rescue => e
log_jdbc_exception(e) log_jdbc_exception(e)
if retry_exception?(e)
raise
end
ensure ensure
statement.close() unless statement.nil? statement.close() unless statement.nil?
connection.close() unless connection.nil? connection.close() unless connection.nil?
@ -224,7 +257,12 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
statement = nil statement = nil
begin begin
connection = @pool.getConnection() connection = @pool.getConnection()
rescue => e
log_jdbc_exception(e)
raise
end
begin
events.each do |event| events.each do |event|
next if event.cancelled? next if event.cancelled?
@ -240,6 +278,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
end end
rescue => e rescue => e
log_jdbc_exception(e) log_jdbc_exception(e)
if retry_exception?(e)
raise
end
ensure ensure
statement.close() unless statement.nil? statement.close() unless statement.nil?
connection.close() unless connection.nil? connection.close() unless connection.nil?
@ -250,11 +291,17 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
@statement[1..-1].each_with_index do |i, idx| @statement[1..-1].each_with_index do |i, idx|
case event[i] case event[i]
when Time when Time
# Most reliable solution, cross JDBC driver # See LogStash::Timestamp, below, for the why behind strftime.
statement.setString(idx + 1, event[i].iso8601()) statement.setString(idx + 1, event[i].strftime(STRFTIME_FMT))
when LogStash::Timestamp when LogStash::Timestamp
# Most reliable solution, cross JDBC driver # XXX: Using setString as opposed to setTimestamp, because setTimestamp
statement.setString(idx + 1, event[i].to_iso8601()) # doesn't behave correctly in some drivers (Known: sqlite)
#
# Additionally this does not use `to_iso8601`, since some SQL databases
# choke on the 'T' in the string (Known: Derby).
#
# strftime appears to be the most reliable across drivers.
statement.setString(idx + 1, event[i].time.strftime(STRFTIME_FMT))
when Fixnum, Integer when Fixnum, Integer
statement.setInt(idx + 1, event[i]) statement.setInt(idx + 1, event[i])
when Float when Float
@ -285,4 +332,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
break if current_exception == nil break if current_exception == nil
end end
end end
def retry_exception?(exception)
return (exception.class != java.sql.SQLException or
RETRYABLE_SQLSTATE_CLASSES.include?(e.getSQLState[0,2]))
end
end # class LogStash::Outputs::jdbc end # class LogStash::Outputs::jdbc

View File

@ -1,6 +1,6 @@
Gem::Specification.new do |s| Gem::Specification.new do |s|
s.name = 'logstash-output-jdbc' s.name = 'logstash-output-jdbc'
s.version = "0.2.6" s.version = "0.2.7.rc1"
s.licenses = [ "Apache License (2.0)" ] s.licenses = [ "Apache License (2.0)" ]
s.summary = "This plugin allows you to output to SQL, via JDBC" s.summary = "This plugin allows you to output to SQL, via JDBC"
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

View File

@ -10,10 +10,7 @@ describe LogStash::Outputs::Jdbc do
"driver_class" => "org.apache.derby.jdbc.EmbeddedDriver", "driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
"connection_string" => "jdbc:derby:memory:testdb;create=true", "connection_string" => "jdbc:derby:memory:testdb;create=true",
"driver_jar_path" => ENV['JDBC_DERBY_JAR'], "driver_jar_path" => ENV['JDBC_DERBY_JAR'],
# Grumble. Grumble. "statement" => [ "insert into log (created_at, message) values(?, ?)", "@timestamp" "message" ]
# Derby doesn't like 'T' in timestamps as of current writing, so for now
# we'll just use CURRENT_TIMESTAMP as opposed to the event @timestamp
"statement" => [ "insert into log (created_at, message) values(CURRENT_TIMESTAMP, ?)", "message" ]
} }
end end