initial commit

.gitignore

@@ -0,0 +1,2 @@
+.*.swp
+vars/global_vars.yml

README.md

@@ -0,0 +1,57 @@
+ansible-playbook-kibana
+=======================
+
+This repo is a set of playbooks which will deploy Logstash + ElasticSearch + Kibana all on a single server.
+
+### Setup
+
+To get started you will need to setup `./vars/global_vars.yml`. To do this I recommend just copying `./vars/global_vars.yml.sample` and filling in the variables to whatever you want.
+
+```shell
+cp ./vars/global_vars.yml.sample ./vars/global_vars.yml
+vi ./vars/global_vars.yml
+```
+
+You'll also need to setup ansible's inventory to define the `central-logging` host.
+
+```shell
+sudo vi /etc/ansible/hosts
+```
+and add
+
+```ini
+[central-logging]
+foo.com
+```
+
+### Running Ansible
+
+There are a few ways to use this set of playbooks -- you can either run the deployment of each service individually, or you can run them all in one go.
+
+
+
+##### All Services
+
+```shell
+ansible-playbook playbooks/all.yml
+```
+
+
+##### Just ElasticSearch
+
+```shell
+ansible-playbook playbooks/elasticsearch.yml
+```
+
+##### Just LogStash
+
+```shell
+ansible-playbook playbooks/logstash.yml
+```
+
+##### Just Kibana3
+
+```shell
+ansible-playbook playbooks/kibana.yml
+```
+

files/kibana/kibana.conf.j2

@@ -0,0 +1,28 @@
+server {
+  listen 80;
+  server_name logs.notprod.pl;
+  root /var/www/kibana;
+
+  # Set image format types to expire in a very long time
+  location ~* ^.+\.(jpg|jpeg|gif|png|ico)$ {
+      access_log off;
+      expires max;
+  }
+
+  # Set css and js to expire in a very long time
+  location ~* ^.+\.(css|js)$ {
+      access_log off;
+      expires max;
+  }
+
+  # Catchall for everything else
+  location / {
+    root /var/www/kibana;
+    index index.html;
+    expires 1d;
+    try_files $uri/ $uri;
+    if (-f $request_filename) {
+      break;
+    }
+  }
+}

files/logstash/logstash-init

@@ -0,0 +1,25 @@
+# logstash-indexer.conf# logstash - indexer instance
+#
+
+description     "logstash indexer instance"
+
+start on virtual-filesystems
+stop on runlevel [06]
+
+respawn
+respawn limit 5 30
+limit nofile 65550 65550
+
+env HOME=/opt/logstash
+env JAVA_OPTS='-Xms512m -Xmx512m'
+
+chdir /opt/logstash
+setuid root
+console log
+
+# for versions 1.1.1 - 1.1.4 the internal web service crashes when touched
+# and the current workaround is to just not run it and run Kibana instead
+
+script
+        exec java -jar /opt/logstash/share/logstash.jar agent -f /opt/logstash/etc/logstash.conf --log /var/log/logstash/logstash-indexer.out
+end script

files/logstash/logstash.conf

@@ -0,0 +1,26 @@
+# logstash configuration
+
+# Define inputs
+input {
+    syslog {
+        debug => false
+        host => "0.0.0.0"
+        port => 5514
+        type => "linux-syslog"
+    }
+}
+filter {
+    grok {
+        type => "linux-syslog"
+        pattern => "%{SYSLOGLINE}"
+    }
+}
+# Define outputs
+output {
+    # send events to stdout for easy debugging
+    # stdout { debug => true debug_format => "json" }
+
+    elasticsearch {
+        host => "127.0.0.1"
+    }
+}

handlers/elasticsearch.yml

@@ -0,0 +1,3 @@
+---
+- name: restart elasticsearch
+  action: service name=elasticsearch state=restarted

handlers/logstash.yml

@@ -0,0 +1,3 @@
+---
+- name: restart logstash
+  action: service name=logstash state=restarted

playbooks/all.yml

@@ -0,0 +1,16 @@
+---
+- hosts: central-logging
+  user: ubuntu
+  sudo: yes
+
+  vars_files:
+    - ../vars/global_vars.yml
+
+  tasks:
+    - include: ../tasks/elasticsearch-setup.yml
+    - include: ../tasks/logstash-setup.yml
+    - include: ../tasks/kibana-setup.yml
+
+  handlers:
+    - include: ../handlers/elasticsearch.yml
+    - include: ../handlers/logstash.yml

playbooks/elasticsearch.yml

@@ -0,0 +1,13 @@
+---
+- hosts: central-logging
+  user: ubuntu
+  sudo: yes
+
+  vars_files:
+    - ../vars/global_vars.yml
+
+  tasks:
+    - include: ../tasks/elasticsearch-setup.yml
+
+  handlers:
+    - include: ../handlers/elasticsearch.yml

playbooks/kibana.yml

@@ -0,0 +1,10 @@
+---
+- hosts: central-logging
+  user: ubuntu
+  sudo: yes
+
+  vars_files:
+    - ../vars/global_vars.yml
+
+  tasks:
+    - include: ../tasks/kibana-setup.yml

playbooks/logstash.yml

@@ -0,0 +1,13 @@
+---
+- hosts: central-logging
+  user: ubuntu
+  sudo: yes
+
+  vars_files:
+    - ../vars/global_vars.yml
+
+  tasks:
+    - include: ../tasks/logstash-setup.yml
+
+  handlers:
+    - include: ../handlers/logstash.yml

tasks/elasticsearch-setup.yml

@@ -0,0 +1,16 @@
+---
+- name: ensure apt cache is up to date
+  apt: update_cache=yes
+- name: ensure python-software-properties is installed
+  apt: pkg=python-software-properties
+- name: ensure app apt dependencies are installed
+  apt: pkg=$item
+  with_items:
+    - openjdk-7-jre
+
+- name: ensure we have the elasticsearch .deb
+  get_url: url=https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${es_version}.deb dest=/tmp/elasticsearch-${es_version}.deb mode=0644
+
+- name: ensure elasticsearch is installed
+  shell: dpkg -i /tmp/elasticsearch-${es_version}.deb creates=/etc/elasticsearch
+  notify: restart elasticsearch

tasks/kibana-setup.yml

@@ -0,0 +1,21 @@
+---
+- name: ensure apt cache is up to date
+  apt: update_cache=yes
+- name: ensure python-software-properties is installed
+  apt: pkg=python-software-properties
+- name: ensure app apt dependencies are installed
+  apt: pkg=$item state=installed
+  with_items:
+    - git
+    - nginx
+
+- name: ensure we have the specified logstash release
+  git: repo=https://github.com/elasticsearch/kibana.git dest=/var/www/kibana update=yes
+- name: ensure kibana nginx site is in place
+  template: src=../files/kibana/kibana.conf.j2 dest=/etc/nginx/sites-available/kibana mode=0755
+- name: ensure default Nginx site is not enabled
+  file: state=absent path=/etc/nginx/sites-enabled/default
+- name: ensure kibana nginx site is enabled
+  file: state=link src=/etc/nginx/sites-available/kibana path=/etc/nginx/sites-enabled/kibana
+- name: ensure nginx config is reloaded by restarting nginx
+  service: name=nginx state=restarted

tasks/logstash-setup.yml

@@ -0,0 +1,35 @@
+---
+- name: ensure apt cache is up to date
+  apt: update_cache=yes
+- name: ensure python-software-properties is installed
+  apt: pkg=python-software-properties
+- name: ensure app apt dependencies are installed
+  apt: pkg=$item state=installed
+  with_items:
+    - openjdk-7-jre
+    - redis-server
+
+- name: ensure /opt/logstash exists
+  file: path=/opt/logstash state=directory owner=root group=root mode=0755
+
+- name: ensure subdirectories exist
+  file: path=/opt/logstash/$item owner=root group=root mode=0755 state=directory
+  with_items:
+    - bin
+    - etc
+    - share
+
+- name: ensure logstash config is in place
+  copy: src=../files/logstash/logstash.conf dest=/opt/logstash/etc/logstash.conf owner=root group=root mode=0644
+
+- name: ensure logstash upstart job is in place
+  copy: src=../files/logstash/logstash-init dest=/etc/init/logstash.conf owner=root group=root mode=0755
+
+- name: ensure logstash has a logging dir at /var/log/logstash
+  file: path=/var/log/logstash owner=root group=root mode=0755 state=directory
+
+- name: ensure we have the specified logstash release
+  get_url: url=${logstash_url} dest=/opt/logstash/share/${logstash_file}
+
+- name: ensure symlink with no version exists at /opt/logstash/share/logstash.jar
+  file: src=/opt/logstash/share/${logstash_file} dest=/opt/logstash/share/logstash.jar state=link

vars/global_vars.yml.sample

@@ -0,0 +1,6 @@
+---
+es_version: 0.90.2
+logstash_version: 1.1.13
+logstash_file: logstash-1.1.13-monolithic.jar
+logstash_url: https://logstash.objects.dreamhost.com/release/logstash-1.1.13-monolithic.jar
+kibana_domain: foo.com