Switches from slf4j-nop to log4j. Uses built in logstash log4j setup. Switches to jar-dependencies (vendor'ed) instead of version controlled jars. Update logstash-api to v2. Does not yet support multi_recieve

This commit is contained in:
Karl Southern 2016-05-13 22:12:57 +01:00
parent d056093ab8
commit df811f3d29
10 changed files with 58 additions and 24 deletions

2
.gitignore vendored
View File

@ -4,3 +4,5 @@ Gemfile.bak
.bundle .bundle
.vagrant .vagrant
.mvn .mvn
vendor
lib/**/*.jar

View File

@ -2,6 +2,10 @@ language: ruby
cache: bundler cache: bundler
rvm: rvm:
- jruby - jruby
before_install:
- bundle install
- bundle exec rake vendor
- bundle exec rake install_jars
before_script: before_script:
- ./scripts/travis-before_script.sh - ./scripts/travis-before_script.sh
- source ./scripts/travis-variables.sh - source ./scripts/travis-variables.sh

View File

@ -35,19 +35,6 @@ For development:
- Add JDBC jar files to vendor/jar/jdbc in your logstash installation - Add JDBC jar files to vendor/jar/jdbc in your logstash installation
- And then configure (examples below) - And then configure (examples below)
## Running tests
For development tests are recommended to run inside a virtual machine (Vagrantfile is included in the repo), as it requires
access to various database engines and could completely destroy any data in a live system.
If you have vagrant available (this is temporary whilst I'm hacking on v5 support. I'll make this more streamlined later):
- `vagrant up`
- `vagrant ssh`
- `cd /vagrant`
- `gem install bundler`
- `cd /vagrant && bundle install`
- `./scripts/travis-before_script.sh && source ./scripts/travis-variables.sh`
- `bundle exec rspec`
## Configuration options ## Configuration options
| Option | Type | Description | Required? | Default | | Option | Type | Description | Required? | Default |
@ -70,3 +57,21 @@ If you have vagrant available (this is temporary whilst I'm hacking on v5 suppor
Example logstash configurations, can now be found in the examples directory. Where possible we try to link every configuration with a tested jar. Example logstash configurations, can now be found in the examples directory. Where possible we try to link every configuration with a tested jar.
If you have a working sample configuration, for a DB thats not listed, pull requests are welcome. If you have a working sample configuration, for a DB thats not listed, pull requests are welcome.
## Development and Running tests
For development tests are recommended to run inside a virtual machine (Vagrantfile is included in the repo), as it requires
access to various database engines and could completely destroy any data in a live system.
If you have vagrant available (this is temporary whilst I'm hacking on v5 support. I'll make this more streamlined later):
- `vagrant up`
- `vagrant ssh`
- `cd /vagrant`
- `gem install bundler`
- `cd /vagrant && bundle install && bundle exec rake vendor && bundle exec rake install_jars`
- `./scripts/travis-before_script.sh && source ./scripts/travis-variables.sh`
- `bundle exec rspec`
## Releasing
- `bundle exec rake install_jars`
- `gem build logstash-output-jdbc.gemspec`
- `gem push`

View File

@ -1 +1,12 @@
task :default do
system("rake -T")
end
require 'jars/installer'
task :install_jars do
ENV['JARS_HOME'] = Dir.pwd + "/vendor/jar-dependencies/runtime-jars"
ENV['JARS_VENDOR'] = "false"
Jars::Installer.new.vendor_jars!(false)
end
require "logstash/devutils/rake" require "logstash/devutils/rake"

View File

@ -83,6 +83,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
def register def register
@logger.info("JDBC - Starting up") @logger.info("JDBC - Starting up")
LogStash::Logger.setup_log4j(@logger)
load_jar_files! load_jar_files!
@exceptions_tracker = RingBuffer.new(@max_flush_exceptions) @exceptions_tracker = RingBuffer.new(@max_flush_exceptions)
@ -250,10 +251,10 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
def add_statement_event_params(statement, event) def add_statement_event_params(statement, event)
@statement[1..-1].each_with_index do |i, idx| @statement[1..-1].each_with_index do |i, idx|
case event[i] case event.get(i)
when Time when Time
# See LogStash::Timestamp, below, for the why behind strftime. # See LogStash::Timestamp, below, for the why behind strftime.
statement.setString(idx + 1, event[i].strftime(STRFTIME_FMT)) statement.setString(idx + 1, event.get(i).strftime(STRFTIME_FMT))
when LogStash::Timestamp when LogStash::Timestamp
# XXX: Using setString as opposed to setTimestamp, because setTimestamp # XXX: Using setString as opposed to setTimestamp, because setTimestamp
# doesn't behave correctly in some drivers (Known: sqlite) # doesn't behave correctly in some drivers (Known: sqlite)
@ -262,19 +263,19 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
# choke on the 'T' in the string (Known: Derby). # choke on the 'T' in the string (Known: Derby).
# #
# strftime appears to be the most reliable across drivers. # strftime appears to be the most reliable across drivers.
statement.setString(idx + 1, event[i].time.strftime(STRFTIME_FMT)) statement.setString(idx + 1, event.get(i).time.strftime(STRFTIME_FMT))
when Fixnum, Integer when Fixnum, Integer
statement.setInt(idx + 1, event[i]) statement.setInt(idx + 1, event.get(i))
when Float when Float
statement.setFloat(idx + 1, event[i]) statement.setFloat(idx + 1, event.get(i))
when String when String
statement.setString(idx + 1, event[i]) statement.setString(idx + 1, event.get(i))
when true when true
statement.setBoolean(idx + 1, true) statement.setBoolean(idx + 1, true)
when false when false
statement.setBoolean(idx + 1, false) statement.setBoolean(idx + 1, false)
else else
if event[i].nil? and i =~ /%\{/ if event.get(i).nil? and i =~ /%\{/
statement.setString(idx + 1, event.sprintf(i)) statement.setString(idx + 1, event.sprintf(i))
else else
statement.setString(idx + 1, nil) statement.setString(idx + 1, nil)

View File

@ -9,6 +9,9 @@ Gem::Specification.new do |s|
s.homepage = "https://github.com/theangryangel/logstash-output-jdbc" s.homepage = "https://github.com/theangryangel/logstash-output-jdbc"
s.require_paths = [ "lib" ] s.require_paths = [ "lib" ]
# Java only
s.platform = 'java'
# Files # Files
s.files = Dir.glob("{lib,vendor,spec}/**/*") + %w(LICENSE.txt README.md) s.files = Dir.glob("{lib,vendor,spec}/**/*") + %w(LICENSE.txt README.md)
@ -17,11 +20,19 @@ Gem::Specification.new do |s|
# Special flag to let us know this is actually a logstash plugin # Special flag to let us know this is actually a logstash plugin
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
# Gem dependencies # Gem dependencies
s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0" s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
s.add_runtime_dependency 'stud' s.add_runtime_dependency 'stud'
s.add_runtime_dependency "logstash-codec-plain" s.add_runtime_dependency "logstash-codec-plain"
# Stuck on HikariCP 2.4.2 until https://github.com/brettwooldridge/HikariCP/issues/600
# is resolved.
s.requirements << "jar 'com.zaxxer:HikariCP', '2.4.2'"
s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.13'"
s.add_development_dependency "jar-dependencies"
s.add_development_dependency 'ruby-maven', '~> 3.3'
s.add_development_dependency "logstash-devutils" s.add_development_dependency "logstash-devutils"
end end

View File

@ -83,7 +83,7 @@ RSpec.shared_context 'when outputting messages' do
# Verify the number of items in the output table # Verify the number of items in the output table
c = plugin.instance_variable_get(:@pool).getConnection() c = plugin.instance_variable_get(:@pool).getConnection()
stmt = c.prepareStatement("select count(*) as total from #{jdbc_test_table} where message = ?") stmt = c.prepareStatement("select count(*) as total from #{jdbc_test_table} where message = ?")
stmt.setString(1, event['message']) stmt.setString(1, event.get('message'))
rs = stmt.executeQuery() rs = stmt.executeQuery()
count = 0 count = 0
while rs.next() while rs.next()