0.3.0.pre - Preparing for threadsafety

spec/jdbc_spec_helper.rb

@@ -0,0 +1,135 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/jdbc'
+require 'stud/temporary'
+require 'java'
+require 'securerandom'
+
+RSpec.shared_context 'rspec setup' do
+  it 'ensure jar is available' do
+    expect(ENV[jdbc_jar_env]).not_to be_nil, "#{jdbc_jar_env} not defined, required to run tests"
+    expect(File.exist?(ENV[jdbc_jar_env])).to eq(true), "#{jdbc_jar_env} defined, but not valid"
+  end
+end
+
+RSpec.shared_context 'when initializing' do
+  it 'shouldn\'t register with a missing jar file' do
+    jdbc_settings['driver_jar_path'] = nil
+    plugin = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
+    expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+  end
+end
+
+RSpec.shared_context 'when outputting messages' do
+  let(:logger) { double("logger") }
+
+  let(:jdbc_test_table) do
+    'logstash_output_jdbc_test'
+  end
+
+  let(:jdbc_drop_table) do
+    "DROP TABLE #{jdbc_test_table}"
+  end
+
+  let(:jdbc_create_table) do
+    "CREATE table #{jdbc_test_table} (created_at datetime not null, message varchar(512) not null, message_sprintf varchar(512) not null, static_int int not null, static_bit bit not null)"
+  end
+
+  let(:jdbc_statement) do
+    ["insert into #{jdbc_test_table} (created_at, message, message_sprintf, static_int, static_bit) values(?, ?, ?, ?, ?)", '@timestamp', 'message', 'sprintf-%{message}', 1, true]
+  end
+
+  let(:systemd_database_service) do
+    nil
+  end
+
+  let(:event_fields) do
+    { 'message' => "test-message #{SecureRandom.uuid}" }
+  end
+
+  let(:event) { LogStash::Event.new(event_fields) }
+
+  let(:plugin) do
+    # Setup plugin
+    output = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
+    output.register
+    output.logger = logger
+
+    # Setup table
+    c = output.instance_variable_get(:@pool).getConnection
+
+    # Derby doesn't support IF EXISTS. 
+    # Seems like the quickest solution. Bleurgh.
+    begin
+      stmt = c.createStatement
+      stmt.executeUpdate(jdbc_drop_table)
+    rescue
+      # noop
+    ensure
+      stmt.close
+
+      stmt = c.createStatement
+      stmt.executeUpdate(jdbc_create_table)
+      stmt.close
+      c.close
+    end
+
+    output
+  end
+
+  it 'should save a event' do
+    expect { plugin.multi_receive([event]) }.to_not raise_error
+
+    # Verify the number of items in the output table
+    c = plugin.instance_variable_get(:@pool).getConnection
+    stmt = c.prepareStatement("select count(*) as total from #{jdbc_test_table} where message = ?")
+    stmt.setString(1, event['message'])
+    rs = stmt.executeQuery
+    count = 0
+    count = rs.getInt('total') while rs.next
+    stmt.close
+    c.close
+
+    expect(count).to eq(1)
+  end
+
+  it 'should not save event, and log an unretryable exception' do
+    e = LogStash::Event.new({})
+
+    expect(logger).to receive(:error).once.with(/JDBC - Exception. Not retrying/, Hash)
+    expect { plugin.multi_receive([e]) }.to_not raise_error
+  end
+
+  it 'it should retry after a connection loss, and log a warning' do
+    skip "does not run as a service" if systemd_database_service.nil?
+
+    p = plugin
+
+    # Check that everything is fine right now
+    expect { p.multi_receive([event]) }.not_to raise_error
+
+    # Start a thread to stop and restart the service.
+    t = Thread.new(systemd_database_service) { |systemd_database_service|
+      start_stop_cmd = 'sudo /etc/init.d/%<service>s* %<action>s'
+
+      `which systemctl`
+      if $?.success?
+        start_stop_cmd = 'sudo systemctl %<action>s %<service>s'
+      end
+
+      cmd = start_stop_cmd % { action: 'stop', service: systemd_database_service }
+      `#{cmd}`
+      sleep 10
+
+      cmd = start_stop_cmd % { action: 'start', service: systemd_database_service }
+      `#{cmd}`
+    }
+
+    # Wait a few seconds to the service to stop
+    sleep 5
+
+    expect(logger).to receive(:warn).at_least(:once).with(/JDBC - Exception. Retrying/, Hash)
+    expect { p.multi_receive([event]) }.to_not raise_error
+
+    t.join
+  end
+end

spec/outputs/jdbc_derby_spec.rb

@@ -0,0 +1,25 @@
+require_relative '../jdbc_spec_helper'
+
+describe 'logstash-output-jdbc: derby', if: ENV['JDBC_DERBY_JAR'] do
+  include_context 'rspec setup'
+  include_context 'when initializing'
+  include_context 'when outputting messages'
+
+  let(:jdbc_jar_env) do
+    'JDBC_DERBY_JAR'
+  end
+
+  let(:jdbc_create_table) do
+    "CREATE table #{jdbc_test_table} (created_at timestamp not null, message varchar(512) not null, message_sprintf varchar(512) not null, static_int int not null, static_bit boolean not null)"
+  end
+ 
+  let(:jdbc_settings) do
+    {
+      'driver_class' => 'org.apache.derby.jdbc.EmbeddedDriver',
+      'connection_string' => 'jdbc:derby:memory:testdb;create=true',
+      'driver_jar_path' => ENV[jdbc_jar_env],
+      'statement' => jdbc_statement,
+      'max_flush_exceptions' => 1
+    }
+  end
+end

spec/outputs/jdbc_mysql_spec.rb

@@ -0,0 +1,25 @@
+require_relative '../jdbc_spec_helper'
+
+describe 'logstash-output-jdbc: mysql', if: ENV['JDBC_MYSQL_JAR'] do
+  include_context 'rspec setup'
+  include_context 'when initializing'
+  include_context 'when outputting messages'
+
+  let(:jdbc_jar_env) do
+    'JDBC_MYSQL_JAR'
+  end
+
+  let(:systemd_database_service) do
+    'mysql'
+  end
+
+  let(:jdbc_settings) do
+    {
+      'driver_class' => 'com.mysql.jdbc.Driver',
+      'connection_string' => 'jdbc:mysql://localhost/logstash_output_jdbc_test?user=root',
+      'driver_jar_path' => ENV[jdbc_jar_env],
+      'statement' => jdbc_statement,
+      'max_flush_exceptions' => 1
+    }
+  end
+end

spec/outputs/jdbc_spec.rb

@@ -1,94 +1,11 @@
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/jdbc"
-require "stud/temporary"
-require "java"
+require_relative '../jdbc_spec_helper'
 
 describe LogStash::Outputs::Jdbc do
-
-  let(:derby_settings) do
-    { 
-      "driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
-      "connection_string" => "jdbc:derby:memory:testdb;create=true",
-      "driver_jar_path" => ENV['JDBC_DERBY_JAR'],
-      "statement" => [ "insert into log (created_at, message) values(?, ?)", "@timestamp" "message" ]
-    }
-  end
-
-  context 'rspec setup' do
-
-    it 'ensure derby is available' do
-      j = ENV['JDBC_DERBY_JAR']
-      expect(j).not_to be_nil, "JDBC_DERBY_JAR not defined, required to run tests"
-      expect(File.exists?(j)).to eq(true), "JDBC_DERBY_JAR defined, but not valid"
-    end
-    
-  end
-
   context 'when initializing' do
-
     it 'shouldn\'t register without a config' do
-      expect { 
-        LogStash::Plugin.lookup("output", "jdbc").new()
-      }.to raise_error(LogStash::ConfigurationError)
+      expect do
+        LogStash::Plugin.lookup('output', 'jdbc').new
+      end.to raise_error(LogStash::ConfigurationError)
     end
-
-    it 'shouldn\'t register with a missing jar file' do
-      derby_settings['driver_jar_path'] = nil
-      plugin = LogStash::Plugin.lookup("output", "jdbc").new(derby_settings)
-      expect { plugin.register }.to raise_error
-    end
-
-    it 'shouldn\'t register with a missing jar file' do
-      derby_settings['connection_string'] = nil
-      plugin = LogStash::Plugin.lookup("output", "jdbc").new(derby_settings)
-      expect { plugin.register }.to raise_error
-    end
-
   end
-
-  context 'when outputting messages' do
-
-    let(:event_fields) do
-      { message: 'test-message' }
-    end
-    let(:event) { LogStash::Event.new(event_fields) }
-    let(:plugin) {
-      # Setup plugin
-      output = LogStash::Plugin.lookup("output", "jdbc").new(derby_settings)
-      output.register
-      if ENV['JDBC_DEBUG'] == '1'
-        output.logger.subscribe(STDOUT)
-      end
-
-      # Setup table
-      c = output.instance_variable_get(:@pool).getConnection()
-      stmt = c.createStatement()
-      stmt.executeUpdate("CREATE table log (created_at timestamp, message varchar(512))")
-      stmt.close()
-      c.close()
-
-      output
-    }
-
-    it 'should save a event' do
-      expect { plugin.receive(event) }.to_not raise_error
-      
-      # Wait for 1 second, for the buffer to flush
-      sleep 1
-
-      c = plugin.instance_variable_get(:@pool).getConnection()
-      stmt = c.createStatement()
-      rs = stmt.executeQuery("select count(*) as total from log")
-      count = 0
-      while rs.next()
-        count = rs.getInt("total")
-      end
-      stmt.close()
-      c.close()
-
-      expect(count).to be > 0
-    end
-
-  end
-
 end

spec/outputs/jdbc_sqlite_spec.rb

@@ -0,0 +1,27 @@
+require_relative '../jdbc_spec_helper'
+
+describe 'logstash-output-jdbc: sqlite', if: ENV['JDBC_SQLITE_JAR'] do
+  JDBC_SQLITE_FILE = '/tmp/logstash_output_jdbc_test.db'.freeze
+
+  before(:context) do
+    File.delete(JDBC_SQLITE_FILE) if File.exist? JDBC_SQLITE_FILE
+  end
+
+  include_context 'rspec setup'
+  include_context 'when initializing'
+  include_context 'when outputting messages'
+
+  let(:jdbc_jar_env) do
+    'JDBC_SQLITE_JAR'
+  end
+
+  let(:jdbc_settings) do
+    {
+      'driver_class' => 'org.sqlite.JDBC',
+      'connection_string' => "jdbc:sqlite:#{JDBC_SQLITE_FILE}",
+      'driver_jar_path' => ENV[jdbc_jar_env],
+      'statement' => jdbc_statement,
+      'max_flush_exceptions' => 1
+    }
+  end
+end