Release 5.3.0

Provisionally adds whole event encoding. Attempts to give an escape plan for users who may have the same key in their event data (to preserve backwards compatibility this is disabled by default). Address #83

Also adds provisional support for BigDecimal. This is untested.

CHANGELOG.md

@@ -1,6 +1,13 @@
 # Change Log
 All notable changes to this project will be documented in this file, from 0.2.0.
 
+## [5.2.1] - 2017-04-09
+  - Adds Array and Hash to_json support for non-sprintf syntax
+
+## [5.2.0] - 2017-04-01
+  - Upgrades HikariCP to latest
+  - Fixes HikariCP logging integration issues
+
 ## [5.1.0] - 2016-12-17
   - phoenix-thin fixes for issue #60
 

THANKS.md

@@ -0,0 +1,18 @@
+logstash-output-jdbc is a project originally created by Karl Southern 
+(the_angry_angel), but there are a number of people that have contributed 
+or implemented key features over time. We do our best to keep this list 
+up-to-date, but you can also have a look at the nice contributor graphs
+produced by GitHub: https://github.com/theangryangel/logstash-output-jdbc/graphs/contributors
+
+ * [hordijk](https://github.com/hordijk)
+ * [dmitryakadiamond](https://github.com/dmitryakadiamond)
+ * [MassimoSporchia](https://github.com/MassimoSporchia)
+ * [ebuildy](https://github.com/ebuildy)
+ * [kushtrimjunuzi](https://github.com/kushtrimjunuzi)
+ * [josemazo](https://github.com/josemazo)
+ * [aceoliver](https://github.com/aceoliver)
+ * [roflmao](https://github.com/roflmao)
+ * [onesuper](https://github.com/onesuper)
+ * [phr0gz](https://github.com/phr0gz)
+ * [jMonsinjon](https://github.com/jMonsinjon)
+ * [mlkmhd](https://github.com/mlkmhd)

Vagrantfile

@@ -1,20 +1,19 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
+JRUBY_VERSION = "jruby-1.7"
+
 Vagrant.configure(2) do |config|
 
   config.vm.define "debian" do |deb|
-    deb.vm.box = 'debian/jessie64'
+    deb.vm.box = 'debian/stretch64'
     deb.vm.synced_folder '.', '/vagrant', type: :virtualbox
 
     deb.vm.provision 'shell', inline: <<-EOP
-      echo "deb http://ftp.debian.org/debian jessie-backports main" | tee --append /etc/apt/sources.list > /dev/null
-      sed -i 's/main/main contrib non-free/g' /etc/apt/sources.list
       apt-get update
-      apt-get remove openjdk-7-jre-headless -y -q
+      apt-get install openjdk-8-jre ca-certificates-java git curl -y -q
-      apt-get install git openjdk-8-jre curl -y -q
+      curl -sSL https://rvm.io/mpapis.asc | sudo gpg --import -
-      gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+      curl -sSL https://get.rvm.io | bash -s stable --ruby=#{JRUBY_VERSION}
-      curl -sSL https://get.rvm.io | bash -s stable --ruby=jruby-1.7
       usermod -a -G rvm vagrant
     EOP
   end
@@ -27,8 +26,8 @@ Vagrant.configure(2) do |config|
     centos.vm.provision 'shell', inline: <<-EOP
       yum update
       yum install java-1.7.0-openjdk
-      gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+      curl -sSL https://rvm.io/mpapis.asc | sudo gpg --import -
-      curl -sSL https://get.rvm.io | bash -s stable --ruby=jruby-1.7
+      curl -sSL https://get.rvm.io | bash -s stable --ruby=#{JRUBY_VERSION}
       usermod -a -G rvm vagrant
     EOP
   end

examples/sql-server.md

@@ -8,8 +8,25 @@ input
 }
 output {
 	jdbc {
+		driver_jar_path => '/opt/sqljdbc42.jar'
 		connection_string => "jdbc:sqlserver://server:1433;databaseName=databasename;user=username;password=password"
 		statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ]
 	}
+	
+}
+```
+
+Another example, with mixed static strings and parameters, with thanks to [@MassimoSporchia](https://github.com/MassimoSporchia)
+```
+input
+{
+	stdin { }
+}
+output {
+jdbc {
+		driver_jar_path => '/opt/sqljdbc42.jar'
+		connection_string => "jdbc:sqlserver://server:1433;databaseName=databasename;user=username;password=password"
+		statement => [ "INSERT INTO log (host, timestamp, message, comment) VALUES(?, ?, ?, 'static string')", "host", "@timestamp", "message" ]
+	}
 }
 ```

lib/logstash/outputs/jdbc.rb

@@ -5,6 +5,8 @@ require 'concurrent'
 require 'stud/interval'
 require 'java'
 require 'logstash-output-jdbc_jars'
+require 'json'
+require 'bigdecimal'
 
 # Write events to a SQL engine, using JDBC.
 #
@@ -63,7 +65,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   config :unsafe_statement, validate: :boolean, default: false
 
   # Number of connections in the pool to maintain
-  config :max_pool_size, validate: :number, default: 24
+  config :max_pool_size, validate: :number, default: 5
 
   # Connection timeout
   config :connection_timeout, validate: :number, default: 10000
@@ -99,6 +101,12 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   config :max_repeat_exceptions_time, obsolete: 'This is no longer required'
   config :idle_flush_time, obsolete: 'No longer necessary under Logstash v5'
   
+  # Allows the whole event to be converted to JSON
+  config :enable_event_as_json_keyword, validate: :boolean, default: false
+  
+  # The magic key used to convert the whole event to JSON. If you need this, and you have the default in your events, you can use this to change your magic keyword.
+  config :event_as_json_keyword, validate: :string, default: '@event'
+
   def register
     @logger.info('JDBC - Starting up')
 
@@ -200,7 +208,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
     begin
       connection = @pool.getConnection
     rescue => e
-      log_jdbc_exception(e, true)
+      log_jdbc_exception(e, true, nil)
       # If a connection is not available, then the server has gone away
       # We're not counting that towards our retry count.
       return events, false
@@ -214,7 +222,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
         statement = add_statement_event_params(statement, event) if @statement.length > 1
         statement.execute
       rescue => e
-        if retry_exception?(e)
+        if retry_exception?(e, event.to_json())
           events_to_retry.push(event)
         end
       ensure
@@ -261,7 +269,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
 
   def add_statement_event_params(statement, event)
     @statement[1..-1].each_with_index do |i, idx|
-      if i.is_a? String 
+      if @enable_event_as_json_keyword and i.is_a? String and i == @event_as_json_keyword
+        value = event.to_json
+      elsif i.is_a? String
         value = event.get(i)
         if value.nil? and i =~ /%\{/
           value = event.sprintf(i)
@@ -289,10 +299,14 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
         else
           statement.setInt(idx + 1, value)
         end
+      when BigDecimal
+        statement.setBigDecimal(idx + 1, value)
       when Float
         statement.setFloat(idx + 1, value)
       when String
         statement.setString(idx + 1, value)
+      when Array, Hash
+        statement.setString(idx + 1, value.to_json)
       when true, false
         statement.setBoolean(idx + 1, value)
       else
@@ -303,16 +317,21 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
     statement
   end
 
-  def retry_exception?(exception)
+  def retry_exception?(exception, event)
     retrying = (exception.respond_to? 'getSQLState' and (RETRYABLE_SQLSTATE_CLASSES.include?(exception.getSQLState.to_s[0,2]) or @retry_sql_states.include?(exception.getSQLState)))
-    log_jdbc_exception(exception, retrying)
+    log_jdbc_exception(exception, retrying, event)
 
     retrying
   end
 
-  def log_jdbc_exception(exception, retrying)
+  def log_jdbc_exception(exception, retrying, event)
     current_exception = exception
     log_text = 'JDBC - Exception. ' + (retrying ? 'Retrying' : 'Not retrying') + '.'
+    
+    if(event != nil)
+        log_text += ' event: "' + event + '".'
+    end
+    
     log_method = (retrying ? 'warn' : 'error')
 
     loop do

logstash-output-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-jdbc'
-  s.version = '5.1.0'
+  s.version = '5.3.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = 'This plugin allows you to output to SQL, via JDBC'
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-jdbc'. This gem is not a stand-alone program"
@@ -22,17 +22,17 @@ Gem::Specification.new do |s|
   s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2'
   s.add_runtime_dependency 'stud'
   s.add_runtime_dependency 'logstash-codec-plain'
 
-  s.requirements << "jar 'com.zaxxer:HikariCP', '2.4.7'"
+  s.requirements << "jar 'com.zaxxer:HikariCP', '2.7.2'"
-  s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.21'"
+  s.requirements << "jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.6.2'"
 
   s.add_development_dependency 'jar-dependencies'
   s.add_development_dependency 'ruby-maven', '~> 3.3'
 
-  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency "logstash-devutils", "~> 1.3", ">= 1.3.1"
 
   s.add_development_dependency 'rubocop', '0.41.2'
 end

spec/jdbc_spec_helper.rb

@@ -59,11 +59,11 @@ RSpec.shared_context 'when outputting messages' do
   end
 
   let(:jdbc_create_table) do
-    "CREATE table #{jdbc_test_table} (created_at datetime not null, message varchar(512) not null, message_sprintf varchar(512) not null, static_int int not null, static_bit bit not null, static_bigint bigint not null)"
+    "CREATE table #{jdbc_test_table} (created_at datetime not null, message varchar(512) not null, message_sprintf varchar(512) not null, static_int int not null, static_bit bit not null, static_bigint bigint not null, static_float float not null)"
   end
 
   let(:jdbc_statement) do
-    ["insert into #{jdbc_test_table} (created_at, message, message_sprintf, static_int, static_bit, static_bigint) values(?, ?, ?, ?, ?, ?)", '@timestamp', 'message', 'sprintf-%{message}', 1, true, 4000881632477184]
+    ["insert into #{jdbc_test_table} (created_at, message, message_sprintf, static_int, static_bit, static_bigint, static_float) values(?, ?, ?, ?, ?, ?, ?)", '@timestamp', 'message', 'sprintf-%{message}', 1, true, 4000881632477184, 12.1]
   end
 
   let(:systemd_database_service) do

spec/outputs/jdbc_derby_spec.rb

@@ -10,7 +10,7 @@ describe 'logstash-output-jdbc: derby', if: ENV['JDBC_DERBY_JAR'] do
   end
 
   let(:jdbc_create_table) do
-    "CREATE table #{jdbc_test_table} (created_at timestamp not null, message varchar(512) not null, message_sprintf varchar(512) not null, static_int int not null, static_bit boolean not null, static_bigint bigint not null)"
+    "CREATE table #{jdbc_test_table} (created_at timestamp not null, message varchar(512) not null, message_sprintf varchar(512) not null, static_int int not null, static_bit boolean not null, static_bigint bigint not null, static_float float not null)"
   end
  
   let(:jdbc_settings) do