From 6bae1d81e3467d8ae917386e446e42cdd772a610 Mon Sep 17 00:00:00 2001 From: Karl Southern Date: Sat, 17 Dec 2016 13:07:48 +0000 Subject: [PATCH] Apache phoenix-thin support --- CHANGELOG.md | 8 ++++- README.md | 1 + Vagrantfile | 42 ++++++++++++++++------- examples/apache-phoenix-hbase-sql.md | 1 + examples/apache-phoenix-thin-hbase-sql.md | 28 +++++++++++++++ lib/logstash/outputs/jdbc.rb | 9 +++++ logstash-output-jdbc.gemspec | 2 +- 7 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 examples/apache-phoenix-thin-hbase-sql.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f4e3e53..545edd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,13 @@ # Change Log All notable changes to this project will be documented in this file, from 0.2.0. -## [0.3.1] = 2016-08-28 +## [5.1.0] - 2016-12-17 + - phoenix-thin fixes for issue #60 + +## [5.0.0] - 2016-11-03 + - logstash v5 support + +## [0.3.1] - 2016-08-28 - Adds connection_test configuration option, to prevent the connection test from occuring, allowing the error to be suppressed. Useful for cockroachdb deployments. https://github.com/theangryangel/logstash-output-jdbc/issues/53 diff --git a/README.md b/README.md index e24860d..7454848 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ For development: | driver_jar_path | String | File path to jar file containing your JDBC driver. This is optional, and all JDBC jars may be placed in $LOGSTASH_HOME/vendor/jar/jdbc instead. | No | | | connection_string | String | JDBC connection URL | Yes | | | connection_test | Boolean | Run a JDBC connection test. Some drivers do not function correctly, and you may need to disable the connection test to supress an error. Cockroach with the postgres JDBC driver is such an example. | No | Yes | +| connection_test_query | String | Connection test and init query string, required for some JDBC drivers that don't support isValid(). Typically you'd set to this "SELECT 1" | No | | | username | String | JDBC username - this is optional as it may be included in the connection string, for many drivers | No | | | password | String | JDBC password - this is optional as it may be included in the connection string, for many drivers | No | | | statement | Array | An array of strings representing the SQL statement to run. Index 0 is the SQL statement that is prepared, all other array entries are passed in as parameters (in order). A parameter may either be a property of the event (i.e. "@timestamp", or "host") or a formatted string (i.e. "%{host} - %{message}" or "%{message}"). If a key is passed then it will be automatically converted as required for insertion into SQL. If it's a formatted string then it will be passed in verbatim. | Yes | | diff --git a/Vagrantfile b/Vagrantfile index 2f7c815..8f6033b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,17 +2,35 @@ # vi: set ft=ruby : Vagrant.configure(2) do |config| - config.vm.box = 'debian/jessie64' - config.vm.synced_folder '.', '/vagrant', type: :virtualbox - config.vm.provision 'shell', inline: <<-EOP - echo "deb http://ftp.debian.org/debian jessie-backports main" | tee --append /etc/apt/sources.list > /dev/null - sed -i 's/main/main contrib non-free/g' /etc/apt/sources.list - apt-get update - apt-get remove openjdk-7-jre-headless -y -q - apt-get install git openjdk-8-jre curl -y -q - gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 - curl -sSL https://get.rvm.io | bash -s stable --ruby=jruby-1.7 - usermod -a -G rvm vagrant - EOP + config.vm.define "debian" do |deb| + deb.vm.box = 'debian/jessie64' + deb.vm.synced_folder '.', '/vagrant', type: :virtualbox + + deb.vm.provision 'shell', inline: <<-EOP + echo "deb http://ftp.debian.org/debian jessie-backports main" | tee --append /etc/apt/sources.list > /dev/null + sed -i 's/main/main contrib non-free/g' /etc/apt/sources.list + apt-get update + apt-get remove openjdk-7-jre-headless -y -q + apt-get install git openjdk-8-jre curl -y -q + gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 + curl -sSL https://get.rvm.io | bash -s stable --ruby=jruby-1.7 + usermod -a -G rvm vagrant + EOP + end + + config.vm.define "centos" do |centos| + centos.vm.box = 'centos/7' + centos.ssh.insert_key = false # https://github.com/mitchellh/vagrant/issues/7610 + centos.vm.synced_folder '.', '/vagrant', type: :virtualbox + + centos.vm.provision 'shell', inline: <<-EOP + yum update + yum install java-1.7.0-openjdk + gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 + curl -sSL https://get.rvm.io | bash -s stable --ruby=jruby-1.7 + usermod -a -G rvm vagrant + EOP + end + end diff --git a/examples/apache-phoenix-hbase-sql.md b/examples/apache-phoenix-hbase-sql.md index 8bfb3fe..18f6895 100644 --- a/examples/apache-phoenix-hbase-sql.md +++ b/examples/apache-phoenix-hbase-sql.md @@ -1,6 +1,7 @@ # Example: Apache Phoenix (HBase SQL) * Tested with Ubuntu 14.04.03 / Logstash 2.1 / Apache Phoenix 4.6 * HBase and Zookeeper must be both accessible from logstash machine + * Please see apache-phoenix-thin-hbase-sql for phoenix-thin. The examples are different. ``` input { diff --git a/examples/apache-phoenix-thin-hbase-sql.md b/examples/apache-phoenix-thin-hbase-sql.md new file mode 100644 index 0000000..75b171c --- /dev/null +++ b/examples/apache-phoenix-thin-hbase-sql.md @@ -0,0 +1,28 @@ +# Example: Apache Phoenix-Thin (HBase SQL) + +**There are special instructions for phoenix-thin. Please read carefully!** + + * Tested with Logstash 5.1.1 / Apache Phoenix 4.9 + * HBase and Zookeeper must be both accessible from logstash machine + * At time of writing phoenix-client does not include all the required jars (see https://issues.apache.org/jira/browse/PHOENIX-3476), therefore you must *not* use the driver_jar_path configuration option and instead: + - `mkdir -p vendor/jar/jdbc` in your logstash installation path + - copy `phoenix-queryserver-client-4.9.0-HBase-1.2.jar` from the phoenix distribution into this folder + - download the calcite jar from https://mvnrepository.com/artifact/org.apache.calcite/calcite-avatica/1.6.0 and place it into your `vendor/jar/jdbc` directory + * Use the following configuration as a base. The connection_test => false and connection_test_query are very important and should not be omitted. Phoenix-thin does not appear to support isValid and these are necessary for the connection to be added to the pool and be available. + +``` +input +{ + stdin { } +} +output { + jdbc { + connection_test => false + connection_test_query => "select 1" + driver_class => "org.apache.phoenix.queryserver.client.Driver" + connection_string => "jdbc:phoenix:thin:url=http://localhost:8765;serialization=PROTOBUF" + statement => [ "UPSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ] + } + +} +``` diff --git a/lib/logstash/outputs/jdbc.rb b/lib/logstash/outputs/jdbc.rb index a8345aa..8c9cb26 100644 --- a/lib/logstash/outputs/jdbc.rb +++ b/lib/logstash/outputs/jdbc.rb @@ -86,6 +86,10 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base # Run a connection test on start. config :connection_test, validate: :boolean, default: true + # Connection test and init string, required for some JDBC endpoints + # notable phoenix-thin - see logstash-output-jdbc issue #60 + config :connection_test_query, validate: :string, required: false + # Maximum number of sequential failed attempts, before we stop retrying. # If set to < 1, then it will infinitely retry. # At the default values this is a little over 10 minutes @@ -146,6 +150,11 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base validate_connection_timeout = (@connection_timeout / 1000) / 2 + if !@connection_test_query.nil? and @connection_test_query.length > 1 + @pool.setConnectionTestQuery(@connection_test_query) + @pool.setConnectionInitSql(@connection_test_query) + end + return unless @connection_test # Test connection diff --git a/logstash-output-jdbc.gemspec b/logstash-output-jdbc.gemspec index 62257bd..8aef151 100644 --- a/logstash-output-jdbc.gemspec +++ b/logstash-output-jdbc.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-jdbc' - s.version = '5.0.0' + s.version = '5.1.0' s.licenses = ['Apache License (2.0)'] s.summary = 'This plugin allows you to output to SQL, via JDBC' s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-jdbc'. This gem is not a stand-alone program"