Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b26ac4e141 | ||
|
|
aea343fcd9 | ||
|
|
470e6309b5 | ||
|
|
a7dde52b7b | ||
|
|
53c0f5761d | ||
|
|
4287c01037 | ||
|
|
4f93fa7224 | ||
|
|
b52a7358ff | ||
|
|
beb8f72560 | ||
|
|
6b6973d6ee | ||
|
|
c790a645f2 |
5
.gitignore
vendored
Normal file
5
.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
*.gem
|
||||||
|
Gemfile.lock
|
||||||
|
Gemfile.bak
|
||||||
|
.bundle
|
||||||
|
vendor
|
||||||
36
README.md
36
README.md
@@ -1,15 +1,23 @@
|
|||||||
# logstash-jdbc
|
# logstash-output-jdbc
|
||||||
|
|
||||||
**This is an old version of the plugin, for Logstash v1.4 only. It is no longer receiving backports of fixes and contains several flaws which have been addressed in the Logstash v2 compatible versions, and above. Please upgrade/see the other branches in the source repository.**
|
**This is an old version of the plugin, for Logstash v1.5 only. It is no longer receiving backports of fixes and contains several flaws which have been addressed in the Logstash v2 compatible versions, and above. Please upgrade/see the other branches in the source repository.**
|
||||||
|
|
||||||
JDBC output plugin for Logstash.
|
|
||||||
This plugin is provided as an external plugin and is not part of the Logstash project.
|
This plugin is provided as an external plugin and is not part of the Logstash project.
|
||||||
|
|
||||||
## Warning
|
This plugin allows you to output to SQL databases, using JDBC adapters.
|
||||||
|
See below for tested adapters, and example configurations.
|
||||||
|
|
||||||
This has not yet been extensively tested with all JDBC drivers and may not yet work for you.
|
This has not yet been extensively tested with all JDBC drivers and may not yet work for you.
|
||||||
|
|
||||||
|
This plugin does not bundle any JDBC jar files, and does expect them to be in a
|
||||||
|
particular location. Please ensure you read the 4 installation lines below.
|
||||||
|
|
||||||
|
## Versions
|
||||||
|
- See v1.5 branch for logstash v1.5
|
||||||
|
- See v1.4 branch for logstash 1.4
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
- Copy lib directory contents into your logstash installation.
|
- Run `bin/plugin install logstash-output-jdbc` in your logstash installation directory
|
||||||
- Create the directory vendor/jar/jdbc in your logstash installation (`mkdir -p vendor/jar/jdbc/`)
|
- Create the directory vendor/jar/jdbc in your logstash installation (`mkdir -p vendor/jar/jdbc/`)
|
||||||
- Add JDBC jar files to vendor/jar/jdbc in your logstash installation
|
- Add JDBC jar files to vendor/jar/jdbc in your logstash installation
|
||||||
- Configure
|
- Configure
|
||||||
@@ -93,4 +101,20 @@ output {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
/* vim: set ts=4 sw=4 tw=0 :*/
|
### Mysql
|
||||||
|
With thanks to [@jMonsinjon](https://github.com/jMonsinjon)
|
||||||
|
* Tested with Version 14.14 Distrib 5.5.43, for debian-linux-gnu (x86_64)
|
||||||
|
* Tested using http://dev.mysql.com/downloads/file.php?id=457911 (mysql-connector-java-5.1.36-bin.jar)
|
||||||
|
```
|
||||||
|
input
|
||||||
|
{
|
||||||
|
stdin { }
|
||||||
|
}
|
||||||
|
output {
|
||||||
|
jdbc {
|
||||||
|
driver_class => "com.mysql.jdbc.Driver"
|
||||||
|
connection_string => "jdbc:mysql://HOSTNAME/DATABASE?user=USER&password=PASSWORD"
|
||||||
|
statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|||||||
include Stud::Buffer
|
include Stud::Buffer
|
||||||
|
|
||||||
config_name "jdbc"
|
config_name "jdbc"
|
||||||
milestone 1
|
|
||||||
|
|
||||||
# Driver class
|
# Driver class
|
||||||
config :driver_class, :validate => :string
|
config :driver_class, :validate => :string
|
||||||
@@ -50,11 +49,21 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|||||||
|
|
||||||
public
|
public
|
||||||
def register
|
def register
|
||||||
|
|
||||||
@logger.info("JDBC - Starting up")
|
@logger.info("JDBC - Starting up")
|
||||||
|
|
||||||
jarpath = File.join(File.dirname(__FILE__), "../../../vendor/jar/jdbc/*.jar")
|
if ENV['LOGSTASH_HOME']
|
||||||
@logger.info(jarpath)
|
jarpath = File.join(ENV['LOGSTASH_HOME'], "/vendor/jar/jdbc/*.jar")
|
||||||
Dir[jarpath].each do |jar|
|
else
|
||||||
|
jarpath = File.join(File.dirname(__FILE__), "../../../vendor/jar/jdbc/*.jar")
|
||||||
|
end
|
||||||
|
|
||||||
|
@logger.debug("JDBC - jarpath", path: jarpath)
|
||||||
|
|
||||||
|
jars = Dir[jarpath]
|
||||||
|
raise Exception.new("JDBC - No jars found in jarpath. Have you read the README?") if jars.empty?
|
||||||
|
|
||||||
|
jars.each do |jar|
|
||||||
@logger.debug("JDBC - Loaded jar", :jar => jar)
|
@logger.debug("JDBC - Loaded jar", :jar => jar)
|
||||||
require jar
|
require jar
|
||||||
end
|
end
|
||||||
@@ -99,7 +108,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|||||||
|
|
||||||
@statement[1..-1].each_with_index do |i, idx|
|
@statement[1..-1].each_with_index do |i, idx|
|
||||||
case event[i]
|
case event[i]
|
||||||
when Time
|
when Time, LogStash::Timestamp
|
||||||
# Most reliable solution, cross JDBC driver
|
# Most reliable solution, cross JDBC driver
|
||||||
statement.setString(idx + 1, event[i].iso8601())
|
statement.setString(idx + 1, event[i].iso8601())
|
||||||
when Fixnum, Integer
|
when Fixnum, Integer
|
||||||
@@ -108,6 +117,12 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|||||||
statement.setFloat(idx + 1, event[i])
|
statement.setFloat(idx + 1, event[i])
|
||||||
when String
|
when String
|
||||||
statement.setString(idx + 1, event[i])
|
statement.setString(idx + 1, event[i])
|
||||||
|
when true
|
||||||
|
statement.setBoolean(idx + 1, true)
|
||||||
|
when false
|
||||||
|
statement.setBoolean(idx + 1, false)
|
||||||
|
when nil
|
||||||
|
statement.setString(idx + 1, nil)
|
||||||
else
|
else
|
||||||
statement.setString(idx + 1, event.sprintf(i))
|
statement.setString(idx + 1, event.sprintf(i))
|
||||||
end
|
end
|
||||||
@@ -124,6 +139,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|||||||
# Since the exceutebatch failed this should mean any events failed to be
|
# Since the exceutebatch failed this should mean any events failed to be
|
||||||
# inserted will be re-run. We're going to log it for the lols anyway.
|
# inserted will be re-run. We're going to log it for the lols anyway.
|
||||||
@logger.warn("JDBC - Exception. Will automatically retry", :exception => e)
|
@logger.warn("JDBC - Exception. Will automatically retry", :exception => e)
|
||||||
|
if e.getNextException() != nil
|
||||||
|
@logger.warn("JDBC - Exception. Will automatically retry", :exception => e.getNextException())
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
statement.close()
|
statement.close()
|
||||||
|
|||||||
24
logstash-output-jdbc.gemspec
Normal file
24
logstash-output-jdbc.gemspec
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
Gem::Specification.new do |s|
|
||||||
|
s.name = 'logstash-output-jdbc'
|
||||||
|
s.version = "0.1.1"
|
||||||
|
s.licenses = [ "Apache License (2.0)" ]
|
||||||
|
s.summary = "This plugin allows you to output to SQL, via JDBC"
|
||||||
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
||||||
|
s.authors = ["the_angry_angel"]
|
||||||
|
s.email = "karl+github@theangryangel.co.uk"
|
||||||
|
s.homepage = "https://github.com/theangryangel/logstash-output-jdbc"
|
||||||
|
s.require_paths = [ "lib" ]
|
||||||
|
|
||||||
|
# Files
|
||||||
|
s.files = `git ls-files`.split($\)
|
||||||
|
# Tests
|
||||||
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
||||||
|
|
||||||
|
# Special flag to let us know this is actually a logstash plugin
|
||||||
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
||||||
|
|
||||||
|
# Gem dependencies
|
||||||
|
s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 2.0.0"
|
||||||
|
s.add_runtime_dependency "logstash-codec-plain"
|
||||||
|
s.add_development_dependency "logstash-devutils"
|
||||||
|
end
|
||||||
Reference in New Issue
Block a user