will/logstash-output-jdbc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rubocop, in preparation for pre-release rake task that sanity checks for release

Browse Source
This commit is contained in:
Karl Southern 2016-05-17 16:21:37 +01:00
parent 5f0f897114
commit d362e791e5
12 changed files with 191 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.rubocop.yml Normal file
View File

@ -0,0 +1,25 @@
# I don't care for underscores in numbers.
Style/NumericLiterals:
Enabled: false
Style/ClassAndModuleChildren:
Enabled: false
Metrics/AbcSize:
Enabled: false
Metrics/CyclomaticComplexity:
Max: 9
Metrics/PerceivedComplexity:
Max: 10
Metrics/LineLength:
Enabled: false
Metrics/MethodLength:
Max: 50
Style/FileName:
Exclude:
- 'lib/logstash-output-jdbc_jars.rb'

8
Rakefile
View File

@ -1,9 +1,9 @@
require "logstash/devutils/rake" require 'logstash/devutils/rake'
require 'jars/installer' require 'jars/installer'
desc "Fetch any jars required for this plugin" desc 'Fetch any jars required for this plugin'
task :install_jars do task :install_jars do
ENV['JARS_HOME'] = Dir.pwd + "/vendor/jar-dependencies/runtime-jars" ENV['JARS_HOME'] = Dir.pwd + '/vendor/jar-dependencies/runtime-jars'
ENV['JARS_VENDOR'] = "false" ENV['JARS_VENDOR'] = 'false'
Jars::Installer.new.vendor_jars!(false) Jars::Installer.new.vendor_jars!(false)
end end

6
Vagrantfile vendored
View File

@ -2,10 +2,10 @@
# vi: set ft=ruby : # vi: set ft=ruby :
Vagrant.configure(2) do |config| Vagrant.configure(2) do |config|
config.vm.box = "debian/jessie64" config.vm.box = 'debian/jessie64'
config.vm.synced_folder ".", "/vagrant", type: :virtualbox config.vm.synced_folder '.', '/vagrant', type: :virtualbox
config.vm.provision "shell", inline: <<-EOP config.vm.provision 'shell', inline: <<-EOP
gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
curl -sSL https://get.rvm.io | bash -s stable --ruby=jruby-1.7 curl -sSL https://get.rvm.io | bash -s stable --ruby=jruby-1.7
usermod -a -G rvm vagrant usermod -a -G rvm vagrant

4
lib/logstash-output-jdbc_jars.rb
View File

@ -1,5 +1,5 @@
# encoding: utf-8 # encoding: utf-8
require 'logstash/environment' require 'logstash/environment'
root_dir = File.expand_path(File.join(File.dirname(__FILE__), "..")) root_dir = File.expand_path(File.join(File.dirname(__FILE__), '..'))
LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor") LogStash::Environment.load_runtime_jars! File.join(root_dir, 'vendor')

164
lib/logstash/outputs/jdbc.rb
View File

@ -1,27 +1,32 @@
# encoding: utf-8 # encoding: utf-8
require "logstash/outputs/base" require 'logstash/outputs/base'
require "logstash/namespace" require 'logstash/namespace'
require "concurrent" require 'concurrent'
require "stud/interval" require 'stud/interval'
require "java" require 'java'
require "logstash-output-jdbc_jars" require 'logstash-output-jdbc_jars'
# Write events to a SQL engine, using JDBC.
#
# It is upto the user of the plugin to correctly configure the plugin. This
# includes correctly crafting the SQL statement, and matching the number of
# parameters correctly.
class LogStash::Outputs::Jdbc < LogStash::Outputs::Base class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
STRFTIME_FMT = "%Y-%m-%d %T.%L".freeze STRFTIME_FMT = '%Y-%m-%d %T.%L'.freeze
# Will never work, but only because it duplicates data (i.e. duplicate keys) # Will never work, but only because it duplicates data (i.e. duplicate keys)
# Will log a warning, but not retry. # Will log a warning, but not retry.
SQL_STATES_IGNORE = [ SQL_STATES_IGNORE = [
### Class: Unqualified Successful Completion ### Class: Unqualified Successful Completion
# Success. This shouldn't get thrown, but JDBC driver quality varies, so who knows. # Success. This shouldn't get thrown, but JDBC driver quality varies, so who knows.
0000, 0000,
### Class: Constraint Violation ### Class: Constraint Violation
# Integrity constraint violation. # Integrity constraint violation.
23000, 23000,
# A violation of the constraint imposed by a unique index or a unique constraint occurred. # A violation of the constraint imposed by a unique index or a unique constraint occurred.
23505 23505
] ].freeze
# Will log an error, but not retry. # Will log an error, but not retry.
SQL_STATES_FATAL = [ SQL_STATES_FATAL = [
@ -39,87 +44,84 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
# Character conversion resulted in truncation. # Character conversion resulted in truncation.
22524, 22524,
### Constraint Violation ### Constraint Violation
# The insert or update value of a foreign key is invalid. # The insert or update value of a foreign key is invalid.
23503, 23503,
# The range of values for the identity column or sequence is exhausted. # The range of values for the identity column or sequence is exhausted.
23522 23522
] ].freeze
config_name "jdbc" config_name 'jdbc'
# Driver class - Reintroduced for https://github.com/theangryangel/logstash-output-jdbc/issues/26 # Driver class - Reintroduced for https://github.com/theangryangel/logstash-output-jdbc/issues/26
config :driver_class, :validate => :string config :driver_class, validate: :string
# Does the JDBC driver support autocommit? # Does the JDBC driver support autocommit?
config :driver_auto_commit, :validate => :boolean, :default => true, :required => true config :driver_auto_commit, validate: :boolean, default: true, required: true
# Where to find the jar # Where to find the jar
# Defaults to not required, and to the original behaviour # Defaults to not required, and to the original behaviour
config :driver_jar_path, :validate => :string, :required => false config :driver_jar_path, validate: :string, required: false
# jdbc connection string # jdbc connection string
config :connection_string, :validate => :string, :required => true config :connection_string, validate: :string, required: true
# jdbc username - optional, maybe in the connection string # jdbc username - optional, maybe in the connection string
config :username, :validate => :string, :required => false config :username, validate: :string, required: false
# jdbc password - optional, maybe in the connection string # jdbc password - optional, maybe in the connection string
config :password, :validate => :string, :required => false config :password, validate: :string, required: false
# [ "insert into table (message) values(?)", "%{message}" ] # [ "insert into table (message) values(?)", "%{message}" ]
config :statement, :validate => :array, :required => true config :statement, validate: :array, required: true
# If this is an unsafe statement, use event.sprintf # If this is an unsafe statement, use event.sprintf
# This also has potential performance penalties due to having to create a # This also has potential performance penalties due to having to create a
# new statement for each event, rather than adding to the batch and issuing # new statement for each event, rather than adding to the batch and issuing
# multiple inserts in 1 go # multiple inserts in 1 go
config :unsafe_statement, :validate => :boolean, :default => false config :unsafe_statement, validate: :boolean, default: false
# Number of connections in the pool to maintain # Number of connections in the pool to maintain
config :max_pool_size, :validate => :number, :default => 5 config :max_pool_size, validate: :number, default: 5
# Connection timeout # Connection timeout
config :connection_timeout, :validate => :number, :default => 10000 config :connection_timeout, validate: :number, default: 10000
# We buffer a certain number of events before flushing that out to SQL. # We buffer a certain number of events before flushing that out to SQL.
# This setting controls how many events will be buffered before sending a # This setting controls how many events will be buffered before sending a
# batch of events. # batch of events.
config :flush_size, :validate => :number, :default => 1000 config :flush_size, validate: :number, default: 1000
# Set initial interval in seconds between retries. Doubled on each retry up to `retry_max_interval` # Set initial interval in seconds between retries. Doubled on each retry up to `retry_max_interval`
config :retry_initial_interval, :validate => :number, :default => 2 config :retry_initial_interval, validate: :number, default: 2
# Maximum time between retries, in seconds # Maximum time between retries, in seconds
config :retry_max_interval, :validate => :number, :default => 128 config :retry_max_interval, validate: :number, default: 128
# Maximum number of sequential failed attempts, before we stop retrying. # Maximum number of sequential failed attempts, before we stop retrying.
# If set to < 1, then it will infinitely retry. # If set to < 1, then it will infinitely retry.
# At the default values this is a little over 10 minutes # At the default values this is a little over 10 minutes
config :max_flush_exceptions, :validate => :number, :default => 10 config :max_flush_exceptions, validate: :number, default: 10
config :max_repeat_exceptions, :obsolete => "This has been replaced by max_flush_exceptions - which behaves slightly differently. Please check the documentation." config :max_repeat_exceptions, obsolete: 'This has been replaced by max_flush_exceptions - which behaves slightly differently. Please check the documentation.'
config :max_repeat_exceptions_time, :obsolete => "This is no longer required" config :max_repeat_exceptions_time, obsolete: 'This is no longer required'
config :idle_flush_time, :obsolete => "No longer necessary under Logstash v5" config :idle_flush_time, obsolete: 'No longer necessary under Logstash v5'
public
def register def register
@logger.info("JDBC - Starting up") @logger.info('JDBC - Starting up')
LogStash::Logger.setup_log4j(@logger) LogStash::Logger.setup_log4j(@logger)
load_jar_files! load_jar_files!
@stopping = Concurrent::AtomicBoolean.new(false) @stopping = Concurrent::AtomicBoolean.new(false)
if (@flush_size > 1000) @logger.warn('JDBC - Flush size is set to > 1000') if @flush_size > 1000
@logger.warn("JDBC - Flush size is set to > 1000")
if @statement.empty?
@logger.error('JDBC - No statement provided. Configuration error.')
end end
if @statement.length < 1 if !@unsafe_statement && @statement.length < 2
@logger.error("JDBC - No statement provided. Configuration error.")
end
if (!@unsafe_statement and @statement.length < 2)
@logger.error("JDBC - Statement has no parameters. No events will be inserted into SQL as you're not passing any event data. Likely configuration error.") @logger.error("JDBC - Statement has no parameters. No events will be inserted into SQL as you're not passing any event data. Likely configuration error.")
end end
@ -133,7 +135,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
end end
def teardown def teardown
@pool.close() @pool.close
super super
end end
@ -143,7 +145,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
private private
def setup_and_test_pool! def setup_and_test_pool!
# Setup pool # Setup pool
@pool = Java::ComZaxxerHikari::HikariDataSource.new @pool = Java::ComZaxxerHikari::HikariDataSource.new
@ -161,47 +163,47 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
validate_connection_timeout = (@connection_timeout / 1000) / 2 validate_connection_timeout = (@connection_timeout / 1000) / 2
# Test connection # Test connection
test_connection = @pool.getConnection() test_connection = @pool.getConnection
unless test_connection.isValid(validate_connection_timeout) unless test_connection.isValid(validate_connection_timeout)
@logger.error("JDBC - Connection is not valid. Please check connection string or that your JDBC endpoint is available.") @logger.error('JDBC - Connection is not valid. Please check connection string or that your JDBC endpoint is available.')
end end
test_connection.close() test_connection.close
end end
def load_jar_files! def load_jar_files!
# Load jar from driver path # Load jar from driver path
unless @driver_jar_path.nil? unless @driver_jar_path.nil?
raise Exception.new("JDBC - Could not find jar file at given path. Check config.") unless File.exists? @driver_jar_path raise Exception, 'JDBC - Could not find jar file at given path. Check config.' unless File.exist? @driver_jar_path
require @driver_jar_path require @driver_jar_path
return return
end end
# Revert original behaviour of loading from vendor directory # Revert original behaviour of loading from vendor directory
# if no path given # if no path given
if ENV['LOGSTASH_HOME'] jarpath = if ENV['LOGSTASH_HOME']
jarpath = File.join(ENV['LOGSTASH_HOME'], "/vendor/jar/jdbc/*.jar") File.join(ENV['LOGSTASH_HOME'], '/vendor/jar/jdbc/*.jar')
else else
jarpath = File.join(File.dirname(__FILE__), "../../../vendor/jar/jdbc/*.jar") File.join(File.dirname(__FILE__), '../../../vendor/jar/jdbc/*.jar')
end end
@logger.debug("JDBC - jarpath", path: jarpath) @logger.debug('JDBC - jarpath', path: jarpath)
jars = Dir[jarpath] jars = Dir[jarpath]
raise Exception.new("JDBC - No jars found in jarpath. Have you read the README?") if jars.empty? raise Exception, 'JDBC - No jars found. Have you read the README?' if jars.empty?
jars.each do |jar| jars.each do |jar|
@logger.debug("JDBC - Loaded jar", :jar => jar) @logger.debug('JDBC - Loaded jar', jar: jar)
require jar require jar
end end
end end
def submit(events) def submit(events)
connection = nil connection = nil
statement = nil statement = nil
events_to_retry = [] events_to_retry = []
begin begin
connection = @pool.getConnection() connection = @pool.getConnection
rescue => e rescue => e
log_jdbc_exception(e) log_jdbc_exception(e)
return events return events
@ -209,19 +211,15 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
events.each do |event| events.each do |event|
begin begin
if @unsafe_statement == true statement = connection.prepareStatement(
statement = connection.prepareStatement(event.sprintf(@statement[0])) (@unsafe_statement == true) ? event.sprintf(@statement[0]) : @statement[0]
else )
statement = connection.prepareStatement(@statement[0])
end
statement = add_statement_event_params(statement, event) if @statement.length > 1 statement = add_statement_event_params(statement, event) if @statement.length > 1
statement.execute
statement.execute()
rescue java.sql.SQLException => e rescue java.sql.SQLException => e
if SQL_STATES_IGNORE.include? e.getSQLState() if SQL_STATES_IGNORE.include? e.getSQLState
@logger.warn('JDBC - Dropping event. Ignore-able exception (duplicate key most likely)', exception: e, event: event) @logger.warn('JDBC - Dropping event. Ignore-able exception (duplicate key most likely)', exception: e, event: event)
elsif SQL_STATES_FATAL.include? e.getSQLState() elsif SQL_STATES_FATAL.include? e.getSQLState
@logger.error('JDBC - Fatal SQL exception. Can never succeed. Dropping event.', exception: e, event: event) @logger.error('JDBC - Fatal SQL exception. Can never succeed. Dropping event.', exception: e, event: event)
else else
log_jdbc_exception(e) log_jdbc_exception(e)
@ -229,16 +227,16 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
end end
rescue => e rescue => e
# Something else happened. # Something else happened.
log_jdbc_exception(e) log_jdbc_exception(e)
events_to_retry.push(event) events_to_retry.push(event)
ensure ensure
statement.close() unless statement.nil? statement.close unless statement.nil?
end end
end end
connection.close() unless connection.nil?
return events_to_retry connection.close unless connection.nil?
events_to_retry
end end
def retrying_submit(actions) def retrying_submit(actions)
@ -248,7 +246,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
attempts = 0 attempts = 0
sleep_interval = @retry_initial_interval sleep_interval = @retry_initial_interval
while submit_actions && submit_actions.length > 0 while submit_actions && !submit_actions.empty?
return if !submit_actions || submit_actions.empty? # If everything's a success we move along return if !submit_actions || submit_actions.empty? # If everything's a success we move along
# We retry whatever didn't succeed # We retry whatever didn't succeed
submit_actions = submit(submit_actions) submit_actions = submit(submit_actions)
@ -279,10 +277,10 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
# See LogStash::Timestamp, below, for the why behind strftime. # See LogStash::Timestamp, below, for the why behind strftime.
statement.setString(idx + 1, event.get(i).strftime(STRFTIME_FMT)) statement.setString(idx + 1, event.get(i).strftime(STRFTIME_FMT))
when LogStash::Timestamp when LogStash::Timestamp
# XXX: Using setString as opposed to setTimestamp, because setTimestamp # XXX: Using setString as opposed to setTimestamp, because setTimestamp
# doesn't behave correctly in some drivers (Known: sqlite) # doesn't behave correctly in some drivers (Known: sqlite)
# #
# Additionally this does not use `to_iso8601`, since some SQL databases # Additionally this does not use `to_iso8601`, since some SQL databases
# choke on the 'T' in the string (Known: Derby). # choke on the 'T' in the string (Known: Derby).
# #
# strftime appears to be the most reliable across drivers. # strftime appears to be the most reliable across drivers.
@ -293,12 +291,10 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
statement.setFloat(idx + 1, event.get(i)) statement.setFloat(idx + 1, event.get(i))
when String when String
statement.setString(idx + 1, event.get(i)) statement.setString(idx + 1, event.get(i))
when true when true, false
statement.setBoolean(idx + 1, true) statement.setBoolean(idx + 1, event.get(i))
when false
statement.setBoolean(idx + 1, false)
else else
if event.get(i).nil? and i =~ /%\{/ if event.get(i).nil? && i =~ /%\{/
statement.setString(idx + 1, event.sprintf(i)) statement.setString(idx + 1, event.sprintf(i))
else else
statement.setString(idx + 1, nil) statement.setString(idx + 1, nil)
@ -312,9 +308,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
def log_jdbc_exception(exception) def log_jdbc_exception(exception)
current_exception = exception current_exception = exception
loop do loop do
@logger.warn("JDBC Exception encountered: Will automatically retry.", :exception => current_exception) @logger.warn('JDBC Exception encountered: Will automatically retry.', exception: current_exception)
current_exception = current_exception.getNextException() current_exception = current_exception.getNextException
break if current_exception == nil break if current_exception.nil?
end end
end end

32
logstash-output-jdbc.gemspec
View File

@ -1,38 +1,40 @@
Gem::Specification.new do |s| Gem::Specification.new do |s|
s.name = 'logstash-output-jdbc' s.name = 'logstash-output-jdbc'
s.version = "1.0.0.pre" s.version = '1.0.0.pre'
s.licenses = [ "Apache License (2.0)" ] s.licenses = ['Apache License (2.0)']
s.summary = "This plugin allows you to output to SQL, via JDBC" s.summary = 'This plugin allows you to output to SQL, via JDBC'
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-jdbc'. This gem is not a stand-alone program" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-jdbc'. This gem is not a stand-alone program"
s.authors = ["the_angry_angel"] s.authors = ['the_angry_angel']
s.email = "karl+github@theangryangel.co.uk" s.email = 'karl+github@theangryangel.co.uk'
s.homepage = "https://github.com/theangryangel/logstash-output-jdbc" s.homepage = 'https://github.com/theangryangel/logstash-output-jdbc'
s.require_paths = [ "lib" ] s.require_paths = ['lib']
# Java only # Java only
s.platform = 'java' s.platform = 'java'
# Files # Files
s.files = Dir.glob("{lib,spec}/**/*.rb") + Dir.glob("vendor/**/*") + %w(LICENSE.txt README.md) s.files = Dir.glob('{lib,spec}/**/*.rb') + Dir.glob('vendor/**/*') + %w(LICENSE.txt README.md)
# Tests # Tests
s.test_files = s.files.grep(%r{^(test|spec|features)/}) s.test_files = s.files.grep(%r{^(test|spec|features)/})
# Special flag to let us know this is actually a logstash plugin # Special flag to let us know this is actually a logstash plugin
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
# Gem dependencies # Gem dependencies
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0" s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
s.add_runtime_dependency 'stud' s.add_runtime_dependency 'stud'
s.add_runtime_dependency "logstash-codec-plain" s.add_runtime_dependency 'logstash-codec-plain'
# Stuck on HikariCP 2.4.2 until https://github.com/brettwooldridge/HikariCP/issues/600 # Stuck on HikariCP 2.4.2 until https://github.com/brettwooldridge/HikariCP/issues/600
# is resolved. # is resolved.
s.requirements << "jar 'com.zaxxer:HikariCP', '2.4.2'" s.requirements << "jar 'com.zaxxer:HikariCP', '2.4.2'"
s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.13'" s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.13'"
s.add_development_dependency "jar-dependencies" s.add_development_dependency 'jar-dependencies'
s.add_development_dependency 'ruby-maven', '~> 3.3' s.add_development_dependency 'ruby-maven', '~> 3.3'
s.add_development_dependency "logstash-devutils" s.add_development_dependency 'logstash-devutils'
s.add_development_dependency 'rubocop'
end end

7
scripts/minutes_to_retries.rb
View File

@ -1,7 +1,6 @@
#!/usr/bin/env ruby -w #!/usr/bin/env ruby -w
seconds_to_reach = 10 * 60 seconds_to_reach = 10 * 60
default_interval = 2
retry_max_interval = 128 retry_max_interval = 128
current_interval = 2 current_interval = 2
@ -9,10 +8,10 @@ total_interval = 0
exceptions_count = 1 exceptions_count = 1
loop do loop do
break if total_interval > seconds_to_reach break if total_interval > seconds_to_reach
exceptions_count += 1 exceptions_count += 1
current_interval = current_interval*2 > retry_max_interval ? retry_max_interval : current_interval*2 current_interval = current_interval * 2 > retry_max_interval ? retry_max_interval : current_interval * 2
total_interval += current_interval total_interval += current_interval
end end

54
spec/jdbc_spec_helper.rb
View File

@ -1,31 +1,26 @@
require "logstash/devutils/rspec/spec_helper" require 'logstash/devutils/rspec/spec_helper'
require "logstash/outputs/jdbc" require 'logstash/outputs/jdbc'
require "stud/temporary" require 'stud/temporary'
require "java" require 'java'
require 'securerandom' require 'securerandom'
RSpec.shared_context 'rspec setup' do RSpec.shared_context 'rspec setup' do
it 'ensure jar is available' do it 'ensure jar is available' do
expect(ENV[jdbc_jar_env]).not_to be_nil, "#{jdbc_jar_env} not defined, required to run tests" expect(ENV[jdbc_jar_env]).not_to be_nil, "#{jdbc_jar_env} not defined, required to run tests"
expect(File.exists?(ENV[jdbc_jar_env])).to eq(true), "#{jdbc_jar_env} defined, but not valid" expect(File.exist?(ENV[jdbc_jar_env])).to eq(true), "#{jdbc_jar_env} defined, but not valid"
end end
end end
RSpec.shared_context 'when initializing' do RSpec.shared_context 'when initializing' do
it 'shouldn\'t register with a missing jar file' do it 'shouldn\'t register with a missing jar file' do
jdbc_settings['driver_jar_path'] = nil jdbc_settings['driver_jar_path'] = nil
plugin = LogStash::Plugin.lookup("output", "jdbc").new(jdbc_settings) plugin = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
expect { plugin.register }.to raise_error expect { plugin.register }.to raise_error
end end
end end
RSpec.shared_context 'when outputting messages' do RSpec.shared_context 'when outputting messages' do
let(:jdbc_test_table) do
let (:jdbc_test_table) do
'logstash_output_jdbc_test' 'logstash_output_jdbc_test'
end end
@ -43,30 +38,28 @@ RSpec.shared_context 'when outputting messages' do
let(:event) { LogStash::Event.new(event_fields) } let(:event) { LogStash::Event.new(event_fields) }
let(:plugin) { let(:plugin) do
# Setup plugin # Setup plugin
output = LogStash::Plugin.lookup("output", "jdbc").new(jdbc_settings) output = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
output.register output.register
if ENV['JDBC_DEBUG'] == '1' output.logger.subscribe(STDOUT) if ENV['JDBC_DEBUG'] == '1'
output.logger.subscribe(STDOUT)
end
# Setup table # Setup table
c = output.instance_variable_get(:@pool).getConnection() c = output.instance_variable_get(:@pool).getConnection
unless jdbc_drop_table.nil? unless jdbc_drop_table.nil?
stmt = c.createStatement() stmt = c.createStatement
stmt.executeUpdate(jdbc_drop_table) stmt.executeUpdate(jdbc_drop_table)
stmt.close() stmt.close
end end
stmt = c.createStatement() stmt = c.createStatement
stmt.executeUpdate(jdbc_create_table) stmt.executeUpdate(jdbc_create_table)
stmt.close() stmt.close
c.close() c.close
output output
} end
it 'should save a event' do it 'should save a event' do
expect { plugin.multi_receive([event]) }.to_not raise_error expect { plugin.multi_receive([event]) }.to_not raise_error
@ -74,18 +67,15 @@ RSpec.shared_context 'when outputting messages' do
sleep 5 sleep 5
# Verify the number of items in the output table # Verify the number of items in the output table
c = plugin.instance_variable_get(:@pool).getConnection() c = plugin.instance_variable_get(:@pool).getConnection
stmt = c.prepareStatement("select count(*) as total from #{jdbc_test_table} where message = ?") stmt = c.prepareStatement("select count(*) as total from #{jdbc_test_table} where message = ?")
stmt.setString(1, event.get('message')) stmt.setString(1, event.get('message'))
rs = stmt.executeQuery() rs = stmt.executeQuery
count = 0 count = 0
while rs.next() count = rs.getInt('total') while rs.next
count = rs.getInt("total") stmt.close
end c.close
stmt.close()
c.close()
expect(count).to eq(1) expect(count).to eq(1)
end end
end end

20
spec/outputs/jdbc_derby_spec.rb
View File

@ -1,10 +1,9 @@
require_relative '../jdbc_spec_helper' require_relative '../jdbc_spec_helper'
describe "logstash-output-jdbc: derby", if: ENV['JDBC_DERBY_JAR'] do describe 'logstash-output-jdbc: derby', if: ENV['JDBC_DERBY_JAR'] do
include_context 'rspec setup'
include_context "rspec setup" include_context 'when initializing'
include_context "when initializing" include_context 'when outputting messages'
include_context "when outputting messages"
let(:jdbc_jar_env) do let(:jdbc_jar_env) do
'JDBC_DERBY_JAR' 'JDBC_DERBY_JAR'
@ -19,12 +18,11 @@ describe "logstash-output-jdbc: derby", if: ENV['JDBC_DERBY_JAR'] do
end end
let(:jdbc_settings) do let(:jdbc_settings) do
{ {
"driver_class" => "org.apache.derby.jdbc.EmbeddedDriver", 'driver_class' => 'org.apache.derby.jdbc.EmbeddedDriver',
"connection_string" => "jdbc:derby:memory:testdb;create=true", 'connection_string' => 'jdbc:derby:memory:testdb;create=true',
"driver_jar_path" => ENV[jdbc_jar_env], 'driver_jar_path' => ENV[jdbc_jar_env],
"statement" => [ "insert into logstash_output_jdbc_test (created_at, message) values(?, ?)", "@timestamp", "message" ] 'statement' => ['insert into logstash_output_jdbc_test (created_at, message) values(?, ?)', '@timestamp', 'message']
} }
end end
end end

22
spec/outputs/jdbc_mysql_spec.rb
View File

@ -1,22 +1,20 @@
require_relative "../jdbc_spec_helper" require_relative '../jdbc_spec_helper'
describe "logstash-output-jdbc: mysql", if: ENV['JDBC_MYSQL_JAR'] do describe 'logstash-output-jdbc: mysql', if: ENV['JDBC_MYSQL_JAR'] do
include_context 'rspec setup'
include_context "rspec setup" include_context 'when initializing'
include_context "when initializing" include_context 'when outputting messages'
include_context "when outputting messages"
let(:jdbc_jar_env) do let(:jdbc_jar_env) do
'JDBC_MYSQL_JAR' 'JDBC_MYSQL_JAR'
end end
let(:jdbc_settings) do let(:jdbc_settings) do
{ {
"driver_class" => "com.mysql.jdbc.Driver", 'driver_class' => 'com.mysql.jdbc.Driver',
"connection_string" => "jdbc:mysql://localhost/logstash_output_jdbc_test?user=root", 'connection_string' => 'jdbc:mysql://localhost/logstash_output_jdbc_test?user=root',
"driver_jar_path" => ENV[jdbc_jar_env], 'driver_jar_path' => ENV[jdbc_jar_env],
"statement" => [ "insert into #{jdbc_test_table} (created_at, message) values(?, ?)", "@timestamp", "message" ] 'statement' => ["insert into #{jdbc_test_table} (created_at, message) values(?, ?)", '@timestamp', 'message']
} }
end end
end end

10
spec/outputs/jdbc_spec.rb
View File

@ -1,13 +1,11 @@
require_relative "../jdbc_spec_helper" require_relative '../jdbc_spec_helper'
describe LogStash::Outputs::Jdbc do describe LogStash::Outputs::Jdbc do
context 'when initializing' do context 'when initializing' do
it 'shouldn\'t register without a config' do it 'shouldn\'t register without a config' do
expect { expect do
LogStash::Plugin.lookup("output", "jdbc").new() LogStash::Plugin.lookup('output', 'jdbc').new
}.to raise_error(LogStash::ConfigurationError) end.to raise_error(LogStash::ConfigurationError)
end end
end end
end end

26
spec/outputs/jdbc_sqlite_spec.rb
View File

@ -1,28 +1,26 @@
require_relative "../jdbc_spec_helper" require_relative '../jdbc_spec_helper'
describe "logstash-output-jdbc: sqlite", if: ENV['JDBC_SQLITE_JAR'] do describe 'logstash-output-jdbc: sqlite', if: ENV['JDBC_SQLITE_JAR'] do
JDBC_SQLITE_FILE = '/tmp/logstash_output_jdbc_test.db'.freeze
JDBC_SQLITE_FILE = "/tmp/logstash_output_jdbc_test.db"
before(:context) do before(:context) do
File.delete(JDBC_SQLITE_FILE) if File.exists? JDBC_SQLITE_FILE File.delete(JDBC_SQLITE_FILE) if File.exist? JDBC_SQLITE_FILE
end end
include_context "rspec setup" include_context 'rspec setup'
include_context "when initializing" include_context 'when initializing'
include_context "when outputting messages" include_context 'when outputting messages'
let(:jdbc_jar_env) do let(:jdbc_jar_env) do
'JDBC_SQLITE_JAR' 'JDBC_SQLITE_JAR'
end end
let(:jdbc_settings) do let(:jdbc_settings) do
{ {
"driver_class" => "org.sqlite.JDBC", 'driver_class' => 'org.sqlite.JDBC',
"connection_string" => "jdbc:sqlite:#{JDBC_SQLITE_FILE}", 'connection_string' => "jdbc:sqlite:#{JDBC_SQLITE_FILE}",
"driver_jar_path" => ENV[jdbc_jar_env], 'driver_jar_path' => ENV[jdbc_jar_env],
"statement" => [ "insert into #{jdbc_test_table} (created_at, message) values(?, ?)", "@timestamp", "message" ] 'statement' => ["insert into #{jdbc_test_table} (created_at, message) values(?, ?)", '@timestamp', 'message']
} }
end end
end end