From dd46dda79cf513585b0d2c632e7d36345745aadd Mon Sep 17 00:00:00 2001
From: jakedahn <jake@markupisart.com>
Date: Sat, 6 Jul 2013 20:21:12 -0700
Subject: [PATCH] initial commit

---
 .gitignore                    |  2 ++
 README.md                     | 57 +++++++++++++++++++++++++++++++++++
 files/kibana/kibana.conf.j2   | 28 +++++++++++++++++
 files/logstash/logstash-init  | 25 +++++++++++++++
 files/logstash/logstash.conf  | 26 ++++++++++++++++
 handlers/elasticsearch.yml    |  3 ++
 handlers/logstash.yml         |  3 ++
 playbooks/all.yml             | 16 ++++++++++
 playbooks/elasticsearch.yml   | 13 ++++++++
 playbooks/kibana.yml          | 10 ++++++
 playbooks/logstash.yml        | 13 ++++++++
 tasks/elasticsearch-setup.yml | 16 ++++++++++
 tasks/kibana-setup.yml        | 21 +++++++++++++
 tasks/logstash-setup.yml      | 35 +++++++++++++++++++++
 vars/global_vars.yml.sample   |  6 ++++
 15 files changed, 274 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 README.md
 create mode 100644 files/kibana/kibana.conf.j2
 create mode 100644 files/logstash/logstash-init
 create mode 100644 files/logstash/logstash.conf
 create mode 100644 handlers/elasticsearch.yml
 create mode 100644 handlers/logstash.yml
 create mode 100644 playbooks/all.yml
 create mode 100644 playbooks/elasticsearch.yml
 create mode 100644 playbooks/kibana.yml
 create mode 100644 playbooks/logstash.yml
 create mode 100644 tasks/elasticsearch-setup.yml
 create mode 100644 tasks/kibana-setup.yml
 create mode 100644 tasks/logstash-setup.yml
 create mode 100644 vars/global_vars.yml.sample

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..dcee5de
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+.*.swp
+vars/global_vars.yml
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..fe8799f
--- /dev/null
+++ b/README.md
@@ -0,0 +1,57 @@
+ansible-playbook-kibana
+=======================
+
+This repo is a set of playbooks which will deploy Logstash + ElasticSearch + Kibana all on a single server.
+
+### Setup
+
+To get started you will need to setup `./vars/global_vars.yml`. To do this I recommend just copying `./vars/global_vars.yml.sample` and filling in the variables to whatever you want.
+
+```shell
+cp ./vars/global_vars.yml.sample ./vars/global_vars.yml
+vi ./vars/global_vars.yml
+```
+
+You'll also need to setup ansible's inventory to define the `central-logging` host.
+
+```shell
+sudo vi /etc/ansible/hosts
+```
+and add
+
+```ini
+[central-logging]
+foo.com
+```
+
+### Running Ansible
+
+There are a few ways to use this set of playbooks -- you can either run the deployment of each service individually, or you can run them all in one go.
+
+
+
+##### All Services
+
+```shell
+ansible-playbook playbooks/all.yml
+```
+
+
+##### Just ElasticSearch
+
+```shell
+ansible-playbook playbooks/elasticsearch.yml
+```
+
+##### Just LogStash
+
+```shell
+ansible-playbook playbooks/logstash.yml
+```
+
+##### Just Kibana3
+
+```shell
+ansible-playbook playbooks/kibana.yml
+```
+
diff --git a/files/kibana/kibana.conf.j2 b/files/kibana/kibana.conf.j2
new file mode 100644
index 0000000..b172c14
--- /dev/null
+++ b/files/kibana/kibana.conf.j2
@@ -0,0 +1,28 @@
+server {
+  listen 80;
+  server_name logs.notprod.pl;
+  root /var/www/kibana;
+
+  # Set image format types to expire in a very long time
+  location ~* ^.+\.(jpg|jpeg|gif|png|ico)$ {
+      access_log off;
+      expires max;
+  }
+
+  # Set css and js to expire in a very long time
+  location ~* ^.+\.(css|js)$ {
+      access_log off;
+      expires max;
+  }
+
+  # Catchall for everything else
+  location / {
+    root /var/www/kibana;
+    index index.html;
+    expires 1d;
+    try_files $uri/ $uri;
+    if (-f $request_filename) {
+      break;
+    }
+  }
+}
diff --git a/files/logstash/logstash-init b/files/logstash/logstash-init
new file mode 100644
index 0000000..2f42f13
--- /dev/null
+++ b/files/logstash/logstash-init
@@ -0,0 +1,25 @@
+# logstash-indexer.conf# logstash - indexer instance
+#
+
+description     "logstash indexer instance"
+
+start on virtual-filesystems
+stop on runlevel [06]
+
+respawn
+respawn limit 5 30
+limit nofile 65550 65550
+
+env HOME=/opt/logstash
+env JAVA_OPTS='-Xms512m -Xmx512m'
+
+chdir /opt/logstash
+setuid root
+console log
+
+# for versions 1.1.1 - 1.1.4 the internal web service crashes when touched
+# and the current workaround is to just not run it and run Kibana instead
+
+script
+        exec java -jar /opt/logstash/share/logstash.jar agent -f /opt/logstash/etc/logstash.conf --log /var/log/logstash/logstash-indexer.out
+end script
diff --git a/files/logstash/logstash.conf b/files/logstash/logstash.conf
new file mode 100644
index 0000000..a72fe9d
--- /dev/null
+++ b/files/logstash/logstash.conf
@@ -0,0 +1,26 @@
+# logstash configuration
+
+# Define inputs
+input {
+    syslog {
+        debug => false
+        host => "0.0.0.0"
+        port => 5514
+        type => "linux-syslog"
+    }
+}
+filter {
+    grok {
+        type => "linux-syslog"
+        pattern => "%{SYSLOGLINE}"
+    }
+}
+# Define outputs
+output {
+    # send events to stdout for easy debugging
+    # stdout { debug => true debug_format => "json" }
+
+    elasticsearch {
+        host => "127.0.0.1"
+    }
+}
diff --git a/handlers/elasticsearch.yml b/handlers/elasticsearch.yml
new file mode 100644
index 0000000..d6c32af
--- /dev/null
+++ b/handlers/elasticsearch.yml
@@ -0,0 +1,3 @@
+---
+- name: restart elasticsearch
+  action: service name=elasticsearch state=restarted
diff --git a/handlers/logstash.yml b/handlers/logstash.yml
new file mode 100644
index 0000000..16c7e77
--- /dev/null
+++ b/handlers/logstash.yml
@@ -0,0 +1,3 @@
+---
+- name: restart logstash
+  action: service name=logstash state=restarted
diff --git a/playbooks/all.yml b/playbooks/all.yml
new file mode 100644
index 0000000..5e02e8b
--- /dev/null
+++ b/playbooks/all.yml
@@ -0,0 +1,16 @@
+---
+- hosts: central-logging
+  user: ubuntu
+  sudo: yes
+
+  vars_files:
+    - ../vars/global_vars.yml
+
+  tasks:
+    - include: ../tasks/elasticsearch-setup.yml
+    - include: ../tasks/logstash-setup.yml
+    - include: ../tasks/kibana-setup.yml
+
+  handlers:
+    - include: ../handlers/elasticsearch.yml
+    - include: ../handlers/logstash.yml
diff --git a/playbooks/elasticsearch.yml b/playbooks/elasticsearch.yml
new file mode 100644
index 0000000..267a69a
--- /dev/null
+++ b/playbooks/elasticsearch.yml
@@ -0,0 +1,13 @@
+---
+- hosts: central-logging
+  user: ubuntu
+  sudo: yes
+
+  vars_files:
+    - ../vars/global_vars.yml
+
+  tasks:
+    - include: ../tasks/elasticsearch-setup.yml
+
+  handlers:
+    - include: ../handlers/elasticsearch.yml
diff --git a/playbooks/kibana.yml b/playbooks/kibana.yml
new file mode 100644
index 0000000..397c918
--- /dev/null
+++ b/playbooks/kibana.yml
@@ -0,0 +1,10 @@
+---
+- hosts: central-logging
+  user: ubuntu
+  sudo: yes
+
+  vars_files:
+    - ../vars/global_vars.yml
+
+  tasks:
+    - include: ../tasks/kibana-setup.yml
diff --git a/playbooks/logstash.yml b/playbooks/logstash.yml
new file mode 100644
index 0000000..96884cf
--- /dev/null
+++ b/playbooks/logstash.yml
@@ -0,0 +1,13 @@
+---
+- hosts: central-logging
+  user: ubuntu
+  sudo: yes
+
+  vars_files:
+    - ../vars/global_vars.yml
+
+  tasks:
+    - include: ../tasks/logstash-setup.yml
+
+  handlers:
+    - include: ../handlers/logstash.yml
diff --git a/tasks/elasticsearch-setup.yml b/tasks/elasticsearch-setup.yml
new file mode 100644
index 0000000..554f64e
--- /dev/null
+++ b/tasks/elasticsearch-setup.yml
@@ -0,0 +1,16 @@
+---
+- name: ensure apt cache is up to date
+  apt: update_cache=yes
+- name: ensure python-software-properties is installed
+  apt: pkg=python-software-properties
+- name: ensure app apt dependencies are installed
+  apt: pkg=$item
+  with_items:
+    - openjdk-7-jre
+
+- name: ensure we have the elasticsearch .deb
+  get_url: url=https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${es_version}.deb dest=/tmp/elasticsearch-${es_version}.deb mode=0644
+
+- name: ensure elasticsearch is installed
+  shell: dpkg -i /tmp/elasticsearch-${es_version}.deb creates=/etc/elasticsearch
+  notify: restart elasticsearch
diff --git a/tasks/kibana-setup.yml b/tasks/kibana-setup.yml
new file mode 100644
index 0000000..a690823
--- /dev/null
+++ b/tasks/kibana-setup.yml
@@ -0,0 +1,21 @@
+---
+- name: ensure apt cache is up to date
+  apt: update_cache=yes
+- name: ensure python-software-properties is installed
+  apt: pkg=python-software-properties
+- name: ensure app apt dependencies are installed
+  apt: pkg=$item state=installed
+  with_items:
+    - git
+    - nginx
+
+- name: ensure we have the specified logstash release
+  git: repo=https://github.com/elasticsearch/kibana.git dest=/var/www/kibana update=yes
+- name: ensure kibana nginx site is in place
+  template: src=../files/kibana/kibana.conf.j2 dest=/etc/nginx/sites-available/kibana mode=0755
+- name: ensure default Nginx site is not enabled
+  file: state=absent path=/etc/nginx/sites-enabled/default
+- name: ensure kibana nginx site is enabled
+  file: state=link src=/etc/nginx/sites-available/kibana path=/etc/nginx/sites-enabled/kibana
+- name: ensure nginx config is reloaded by restarting nginx
+  service: name=nginx state=restarted
diff --git a/tasks/logstash-setup.yml b/tasks/logstash-setup.yml
new file mode 100644
index 0000000..7f7f850
--- /dev/null
+++ b/tasks/logstash-setup.yml
@@ -0,0 +1,35 @@
+---
+- name: ensure apt cache is up to date
+  apt: update_cache=yes
+- name: ensure python-software-properties is installed
+  apt: pkg=python-software-properties
+- name: ensure app apt dependencies are installed
+  apt: pkg=$item state=installed
+  with_items:
+    - openjdk-7-jre
+    - redis-server
+
+- name: ensure /opt/logstash exists
+  file: path=/opt/logstash state=directory owner=root group=root mode=0755
+
+- name: ensure subdirectories exist
+  file: path=/opt/logstash/$item owner=root group=root mode=0755 state=directory
+  with_items:
+    - bin
+    - etc
+    - share
+
+- name: ensure logstash config is in place
+  copy: src=../files/logstash/logstash.conf dest=/opt/logstash/etc/logstash.conf owner=root group=root mode=0644
+
+- name: ensure logstash upstart job is in place
+  copy: src=../files/logstash/logstash-init dest=/etc/init/logstash.conf owner=root group=root mode=0755
+
+- name: ensure logstash has a logging dir at /var/log/logstash
+  file: path=/var/log/logstash owner=root group=root mode=0755 state=directory
+
+- name: ensure we have the specified logstash release
+  get_url: url=${logstash_url} dest=/opt/logstash/share/${logstash_file}
+
+- name: ensure symlink with no version exists at /opt/logstash/share/logstash.jar
+  file: src=/opt/logstash/share/${logstash_file} dest=/opt/logstash/share/logstash.jar state=link
diff --git a/vars/global_vars.yml.sample b/vars/global_vars.yml.sample
new file mode 100644
index 0000000..8d605ee
--- /dev/null
+++ b/vars/global_vars.yml.sample
@@ -0,0 +1,6 @@
+---
+es_version: 0.90.2
+logstash_version: 1.1.13
+logstash_file: logstash-1.1.13-monolithic.jar
+logstash_url: https://logstash.objects.dreamhost.com/release/logstash-1.1.13-monolithic.jar
+kibana_domain: foo.com