Full upgrade for Vagrant (#202)

Vagrantfile

@@ -6,7 +6,7 @@ VAGRANTFILE_API_VERSION = "2"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "ubuntu/xenial64"
 
   config.vm.provider "virtualbox" do |v|
     v.name = "ELK_vagrant"
@@ -17,11 +17,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box_check_update = true
 
   config.vm.network "forwarded_port", guest: 5000, host: 5000
-
   config.vm.network "forwarded_port", guest: 5601, host: 5601
-
   config.vm.network "forwarded_port", guest: 9200, host: 9200
-
   config.vm.network "forwarded_port", guest: 9300, host: 9300
 
   #SSH
@@ -32,16 +29,18 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   #Provision
   config.vm.provision "shell", inline: <<-SHELL
     sudo touch /var/lib/cloud/instance/locale-check.skip
-    sudo apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-    sudo sh -c 'echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" > /etc/apt/sources.list.d/docker.list'
-    sudo apt-cache policy docker-engine
-    sudo apt-get update
-    sudo apt-get upgrade -y
-    sudo apt-get install -y docker-engine python-pip
+    echo "vm.max_map_count=262144" | sudo tee /etc/sysctl.d/10-elasticsearch.conf
+    sudo sysctl -p --system
+    sudo apt update
+    sudo apt -y full-upgrade
+    sudo apt -y install apt-transport-https ca-certificates curl software-properties-common
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    sudo apt update
+    sudo apt install -y docker-ce python-pip
     sudo pip install docker-compose
     sudo usermod -aG docker vagrant
-    sudo -u vagrant pip install docker-compose
-    sudo docker-compose -f /vagrant/docker-compose.yml up -d
+    cd /vagrant && docker-compose up -d
   SHELL
 
 end

docker-compose.yml

 version: '2'
 
 services:
+
   elasticsearch:
-    build: elasticsearch/
+    build:
+      context: elasticsearch/
+    volumes:
+      - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
     ports:
       - "9200:9200"
       - "9300:9300"
     environment:
-      ES_JAVA_OPTS: "-Xms1g -Xmx1g"
+      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
     networks:
-      - docker_elk
+      - elk
+
   logstash:
-    build: logstash/
-    command: -f /etc/logstash/conf.d/
+    build:
+      context: logstash/
     volumes:
-      - ./logstash/config:/etc/logstash/conf.d
+      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
+      - ./logstash/pipeline:/usr/share/logstash/pipeline:ro
     ports:
       - "5000:5000"
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
     networks:
-      - docker_elk
+      - elk
     depends_on:
       - elasticsearch
+
   kibana:
-    build: kibana/
+    build:
+      context: kibana/
     volumes:
-      - ./kibana/config/:/etc/kibana/
+      - ./kibana/config/:/usr/share/kibana/config:ro
     ports:
       - "5601:5601"
     networks:
-      - docker_elk
+      - elk
     depends_on:
       - elasticsearch
 
 networks:
-  docker_elk:
-    driver: bridge
+
+  elk:
+    driver: bridge
\ No newline at end of file

elasticsearch/Dockerfile

-FROM elasticsearch:5
+# https://github.com/elastic/elasticsearch-docker
+FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.1.0
 
-ENV ES_JAVA_OPTS="-Des.path.conf=/etc/elasticsearch"
-
-CMD ["-E", "network.host=0.0.0.0", "-E", "discovery.zen.minimum_master_nodes=1"]
+# Add your elasticsearch plugins setup here
+# Example: RUN elasticsearch-plugin install analysis-icu

elasticsearch/config/.placeholder

-Ensure the existence of the parent folder.

elasticsearch/config/elasticsearch.yml

+---
+## Default Elasticsearch configuration from elasticsearch-docker.
+## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml
+#
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0
+
+# minimum_master_nodes need to be explicitly set when bound on a public IP
+# set to 1 to allow single node clusters
+# Details: https://github.com/elastic/elasticsearch/pull/17288
+discovery.zen.minimum_master_nodes: 1
+
+## Use single node discovery in order to disable production mode and avoid bootstrap checks
+## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
+#
+discovery.type: single-node

extensions/README.md

+Third-party extensions that enable extra integrations with the ELK stack.

extensions/logspout/Dockerfile

+# uses ONBUILD instructions described here:
+# https://github.com/gliderlabs/logspout/tree/master/custom
+
+FROM gliderlabs/logspout:master
+ENV SYSLOG_FORMAT rfc3164

extensions/logspout/README.md

+# Logspout extension
+
+Logspout collects all Docker logs using the Docker logs API, and forwards them to Logstash without any additional
+configuration.
+
+## Usage
+
+If you want to include the Logspout extension, run Docker Compose from the root of the repository with an additional
+command line argument referencing the `logspout-compose.yml` file:
+
+```bash
+$ docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml up
+```
+
+In your Logstash pipeline configuration, enable the `udp` input and set the input codec to `json`:
+
+```
+input {
+  udp {
+    port  => 5000
+    codec => json
+  }
+}
+```
+
+## Documentation
+
+https://github.com/looplab/logspout-logstash

extensions/logspout/build.sh

+#!/bin/sh
+
+# unmodified from:
+# https://github.com/gliderlabs/logspout/blob/67ee3831cbd0594361bb3381380c65bdbeb3c20f/custom/build.sh
+
+set -e
+apk add --update go git mercurial build-base
+mkdir -p /go/src/github.com/gliderlabs
+cp -r /src /go/src/github.com/gliderlabs/logspout
+cd /go/src/github.com/gliderlabs/logspout
+export GOPATH=/go
+go get
+go build -ldflags "-X main.Version=$1" -o /bin/logspout
+apk del go git mercurial build-base
+rm -rf /go /var/cache/apk/* /root/.glide
+
+# backwards compatibility
+ln -fs /tmp/docker.sock /var/run/docker.sock

extensions/logspout/logspout-compose.yml

+version: '2'
+
+services:
+  logspout:
+    build:
+      context: extensions/logspout
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    environment:
+      ROUTE_URIS: logstash://logstash:5000
+      LOGSTASH_TAGS: docker-elk
+    networks:
+      - elk
+    depends_on:
+      - logstash
+    restart: on-failure

extensions/logspout/modules.go

+package main
+
+// installs the Logstash adapter for Logspout, and required dependencies
+// https://github.com/looplab/logspout-logstash
+import (
+	_ "github.com/looplab/logspout-logstash"
+	_ "github.com/gliderlabs/logspout/transports/udp"
+	_ "github.com/gliderlabs/logspout/transports/tcp"
+)

kibana/Dockerfile

-FROM kibana:5
+# https://github.com/elastic/kibana-docker
+FROM docker.elastic.co/kibana/kibana-oss:6.1.0
+
+# Add your kibana plugins setup here
+# Example: RUN kibana-plugin install <name|url>

kibana/config/kibana.yml

-# Kibana is served by a back end server. This setting specifies the port to use.
-server.port: 5601
-
-# This setting specifies the IP address of the back end server.
-server.host: "0.0.0.0"
-
-# Enables you to specify a path to mount Kibana at if you are running behind a proxy. This setting
-# cannot end in a slash.
-# server.basePath: ""
-
-# The maximum payload size in bytes for incoming server requests.
-# server.maxPayloadBytes: 1048576
-
-# The Kibana server's name.  This is used for display purposes.
-# server.name: "your-hostname"
-
-# The URL of the Elasticsearch instance to use for all your queries.
-elasticsearch.url: "http://elasticsearch:9200"
-
-# When this setting’s value is true Kibana uses the hostname specified in the server.host
-# setting. When the value of this setting is false, Kibana uses the hostname of the host
-# that connects to this Kibana instance.
-# elasticsearch.preserveHost: true
-
-# Kibana uses an index in Elasticsearch to store saved searches, visualizations and
-# dashboards. Kibana creates a new index if the index doesn’t already exist.
-# kibana.index: ".kibana"
-
-# The default application to load.
-# kibana.defaultAppId: "discover"
-
-# If your Elasticsearch is protected with basic authentication, these settings provide
-# the username and password that the Kibana server uses to perform maintenance on the Kibana
-# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
-# is proxied through the Kibana server.
-# elasticsearch.username: "user"
-# elasticsearch.password: "pass"
-
-# Paths to the PEM-format SSL certificate and SSL key files, respectively. These
-# files enable SSL for outgoing requests from the Kibana server to the browser.
-# server.ssl.cert: /path/to/your/server.crt
-# server.ssl.key: /path/to/your/server.key
-
-# Optional settings that provide the paths to the PEM-format SSL certificate and key files.
-# These files validate that your Elasticsearch backend uses the same key files.
-# elasticsearch.ssl.cert: /path/to/your/client.crt
-# elasticsearch.ssl.key: /path/to/your/client.key
-
-# Optional setting that enables you to specify a path to the PEM file for the certificate
-# authority for your Elasticsearch instance.
-# elasticsearch.ssl.ca: /path/to/your/CA.pem
-
-# To disregard the validity of SSL certificates, change this setting’s value to false.
-# elasticsearch.ssl.verify: true
-
-# Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
-# the elasticsearch.requestTimeout setting.
-# elasticsearch.pingTimeout: 1500
-
-# Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
-# must be a positive integer.
-# elasticsearch.requestTimeout: 30000
-
-# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
-# headers, set this value to [] (an empty list).
-# elasticsearch.requestHeadersWhitelist: [ authorization ]
-
-# Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
-# elasticsearch.shardTimeout: 0
-
-# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.
-# elasticsearch.startupTimeout: 5000
-
-# Specifies the path where Kibana creates the process ID file.
-# pid.file: /var/run/kibana.pid
-
-# Enables you specify a file where Kibana stores log output.
-# logging.dest: stdout
-
-# Set the value of this setting to true to suppress all logging output.
-# logging.silent: false
-
-# Set the value of this setting to true to suppress all logging output other than error messages.
-# logging.quiet: false
-
-# Set the value of this setting to true to log all events, including system usage information
-# and all requests.
-# logging.verbose: false
-
-# Set the interval in milliseconds to sample system and process performance
-# metrics. Minimum is 100ms. Defaults to 10000.
-# ops.interval: 10000
+---
+## Default Kibana configuration from kibana-docker.
+## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml
+#
+server.name: kibana
+server.host: "0"
+elasticsearch.url: http://elasticsearch:9200

logstash/Dockerfile

-FROM logstash:5
+# https://github.com/elastic/logstash-docker
+FROM docker.elastic.co/logstash/logstash-oss:6.1.0
 
 # Add your logstash plugins setup here
 # Example: RUN logstash-plugin install logstash-filter-json

logstash/config/logstash.yml

+---
+## Default Logstash configuration from logstash-docker.
+## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-oss.yml
+#
+http.host: "0.0.0.0"
+path.config: /usr/share/logstash/pipeline
\ No newline at end of file