Apache phoenix-thin support

CHANGELOG.md

@@ -1,7 +1,13 @@
 # Change Log
 All notable changes to this project will be documented in this file, from 0.2.0.
 
-## [0.3.1] = 2016-08-28
+## [5.1.0] - 2016-12-17
+  - phoenix-thin fixes for issue #60
+
+## [5.0.0] - 2016-11-03
+  - logstash v5 support
+
+## [0.3.1] - 2016-08-28
   - Adds connection_test configuration option, to prevent the connection test from occuring, allowing the error to be suppressed.
     Useful for cockroachdb deployments. https://github.com/theangryangel/logstash-output-jdbc/issues/53 
 

README.md

@@ -44,6 +44,7 @@ For development:
 | driver_jar_path | String | File path to jar file containing your JDBC driver. This is optional, and all JDBC jars may be placed in $LOGSTASH_HOME/vendor/jar/jdbc instead. | No | |
 | connection_string | String | JDBC connection URL | Yes | |
 | connection_test | Boolean | Run a JDBC connection test. Some drivers do not function correctly, and you may need to disable the connection test to supress an error. Cockroach with the postgres JDBC driver is such an example. | No | Yes |
+| connection_test_query | String | Connection test and init query string, required for some JDBC drivers that don't support isValid(). Typically you'd set to this "SELECT 1" | No |  |
 | username | String | JDBC username - this is optional as it may be included in the connection string, for many drivers | No | |
 | password | String | JDBC password - this is optional as it may be included in the connection string, for many drivers | No | |
 | statement | Array | An array of strings representing the SQL statement to run. Index 0 is the SQL statement that is prepared, all other array entries are passed in as parameters (in order). A parameter may either be a property of the event (i.e. "@timestamp", or "host") or a formatted string (i.e. "%{host} - %{message}" or "%{message}"). If a key is passed then it will be automatically converted as required for insertion into SQL. If it's a formatted string then it will be passed in verbatim. | Yes |  |

Vagrantfile

@@ -2,17 +2,35 @@
 # vi: set ft=ruby :
 
 Vagrant.configure(2) do |config|
-  config.vm.box = 'debian/jessie64'
-  config.vm.synced_folder '.', '/vagrant', type: :virtualbox
 
-  config.vm.provision 'shell', inline: <<-EOP
+  config.vm.define "debian" do |deb|
-    echo "deb http://ftp.debian.org/debian jessie-backports main" | tee --append /etc/apt/sources.list > /dev/null
+    deb.vm.box = 'debian/jessie64'
-    sed -i 's/main/main contrib non-free/g' /etc/apt/sources.list
+    deb.vm.synced_folder '.', '/vagrant', type: :virtualbox
-    apt-get update
+
-    apt-get remove openjdk-7-jre-headless -y -q
+    deb.vm.provision 'shell', inline: <<-EOP
-    apt-get install git openjdk-8-jre curl -y -q
+      echo "deb http://ftp.debian.org/debian jessie-backports main" | tee --append /etc/apt/sources.list > /dev/null
-    gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+      sed -i 's/main/main contrib non-free/g' /etc/apt/sources.list
-    curl -sSL https://get.rvm.io | bash -s stable --ruby=jruby-1.7
+      apt-get update
-    usermod -a -G rvm vagrant
+      apt-get remove openjdk-7-jre-headless -y -q
-  EOP
+      apt-get install git openjdk-8-jre curl -y -q
+      gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+      curl -sSL https://get.rvm.io | bash -s stable --ruby=jruby-1.7
+      usermod -a -G rvm vagrant
+    EOP
+  end
+
+  config.vm.define "centos" do |centos|
+    centos.vm.box = 'centos/7'
+    centos.ssh.insert_key = false # https://github.com/mitchellh/vagrant/issues/7610
+    centos.vm.synced_folder '.', '/vagrant', type: :virtualbox
+
+    centos.vm.provision 'shell', inline: <<-EOP
+      yum update
+      yum install java-1.7.0-openjdk
+      gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+      curl -sSL https://get.rvm.io | bash -s stable --ruby=jruby-1.7
+      usermod -a -G rvm vagrant
+    EOP
+  end
+
 end

examples/apache-phoenix-hbase-sql.md

@@ -1,6 +1,7 @@
 # Example: Apache Phoenix (HBase SQL)
   * Tested with Ubuntu 14.04.03 / Logstash 2.1 / Apache Phoenix 4.6
   * <!> HBase and Zookeeper must be both accessible from logstash machine <!>
+  * Please see apache-phoenix-thin-hbase-sql for phoenix-thin. The examples are different.
 ```
 input
 {

examples/apache-phoenix-thin-hbase-sql.md

@@ -0,0 +1,28 @@
+# Example: Apache Phoenix-Thin (HBase SQL)
+
+**There are special instructions for phoenix-thin. Please read carefully!**
+
+  * Tested with Logstash 5.1.1 / Apache Phoenix 4.9
+  * HBase and Zookeeper must be both accessible from logstash machine
+  * At time of writing phoenix-client does not include all the required jars (see https://issues.apache.org/jira/browse/PHOENIX-3476), therefore you must *not* use the driver_jar_path configuration option and instead:
+    - `mkdir -p vendor/jar/jdbc` in your logstash installation path
+    - copy `phoenix-queryserver-client-4.9.0-HBase-1.2.jar` from the phoenix distribution into this folder
+    - download the calcite jar from https://mvnrepository.com/artifact/org.apache.calcite/calcite-avatica/1.6.0 and place it into your `vendor/jar/jdbc` directory
+  * Use the following configuration as a base. The connection_test => false and connection_test_query are very important and should not be omitted. Phoenix-thin does not appear to support isValid and these are necessary for the connection to be added to the pool and be available.
+
+```
+input
+{
+    stdin { }
+}
+output {
+    jdbc {
+        connection_test => false
+        connection_test_query => "select 1"
+        driver_class => "org.apache.phoenix.queryserver.client.Driver"
+        connection_string => "jdbc:phoenix:thin:url=http://localhost:8765;serialization=PROTOBUF"
+        statement => [ "UPSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ]
+    }
+
+}
+```

lib/logstash/outputs/jdbc.rb

@@ -86,6 +86,10 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   # Run a connection test on start.
   config :connection_test, validate: :boolean, default: true
 
+  # Connection test and init string, required for some JDBC endpoints
+  # notable phoenix-thin - see logstash-output-jdbc issue #60
+  config :connection_test_query, validate: :string, required: false
+
   # Maximum number of sequential failed attempts, before we stop retrying.
   # If set to < 1, then it will infinitely retry.
   # At the default values this is a little over 10 minutes
@@ -146,6 +150,11 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
 
     validate_connection_timeout = (@connection_timeout / 1000) / 2
 
+    if !@connection_test_query.nil? and @connection_test_query.length > 1
+      @pool.setConnectionTestQuery(@connection_test_query)
+      @pool.setConnectionInitSql(@connection_test_query)
+    end
+
     return unless @connection_test
 
     # Test connection

logstash-output-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-jdbc'
-  s.version = '5.0.0'
+  s.version = '5.1.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = 'This plugin allows you to output to SQL, via JDBC'
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-jdbc'. This gem is not a stand-alone program"