diff --git a/.gitignore b/.gitignore index 4ebc4b7..0adb285 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ *.log *.ova *.retry + +.project +manifest.json +**/*.box \ No newline at end of file diff --git a/build_packer.sh b/build_packer.sh new file mode 100755 index 0000000..2c65141 --- /dev/null +++ b/build_packer.sh @@ -0,0 +1,31 @@ +#!/bin/bash -eux + +# set the session to be noninteractive +export DEBIAN_FRONTEND="noninteractive" + +### set region +export AWS_REGION="us-east-1" + +### list first VPC id +export BUILD_VPC_ID=$(aws ec2 describe-vpcs \ + --query 'Vpcs[0].[VpcId]' \ + --output text); +echo $BUILD_VPC_ID; + +### list first subnet id, within VPC +export BUILD_SUBNET_ID=$(aws ec2 describe-subnets \ + --filters "Name=vpc-id,Values=$BUILD_VPC_ID" \ + --query 'Subnets[0].[SubnetId]' \ + --output text); +echo $BUILD_SUBNET_ID; + +### build Packer AMI + +packer validate packer.json + +packer inspect packer.json + +packer build packer.json + +export BUILD_AMI_ID=$(jq '.builds[-1].artifact_id' -r manifest.json | cut -d':' -f2); +echo $BUILD_AMI_ID; diff --git a/default_configs/elasticsearch.yml b/default_configs/elasticsearch.yml new file mode 100644 index 0000000..cbfc567 --- /dev/null +++ b/default_configs/elasticsearch.yml @@ -0,0 +1,88 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please consult the documentation for further information on configuration options: +# https://www.elastic.co/guide/en/elasticsearch/reference/index.html +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +#cluster.name: my-application +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +#node.name: node-1 +# +# Add custom attributes to the node: +# +#node.attr.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +#path.data: /path/to/data +# +# Path to log files: +# +#path.logs: /path/to/logs +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +bootstrap.memory_lock: true +# +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# Set the bind address to a specific IP (IPv4 or IPv6): +# +#network.host: 192.168.0.1 +# +# Set a custom port for HTTP: +# +#http.port: 9200 +# +# For more information, consult the network module documentation. +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when new node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +#discovery.zen.ping.unicast.hosts: ["host1", "host2"] +# +# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1): +# +discovery.zen.minimum_master_nodes: 1 +# +# For more information, consult the zen discovery module documentation. +# +# ---------------------------------- Gateway ----------------------------------- +# +# Block initial recovery after a full cluster restart until N nodes are started: +# +#gateway.recover_after_nodes: 3 +# +# For more information, consult the gateway module documentation. +# +# ---------------------------------- Various ----------------------------------- +# +# Require explicit names when deleting indices: +# +action.destructive_requires_name: true diff --git a/default_configs/filebeat.yml b/default_configs/filebeat.yml new file mode 100644 index 0000000..7ef4d21 --- /dev/null +++ b/default_configs/filebeat.yml @@ -0,0 +1,33 @@ +filebeat.prospectors: + +#- input_type: log +# paths: +# - /var/log/syslog +# document_type: syslog + +- input_type: log + paths: + - /var/log/kibana/kibana.log + document_type: kibana + json.message_key: log + +- input_type: log + paths: + - /var/lib/docker/containers/*/*.log + document_type: docker + json.message_key: log + + +filebeat.modules: +- module: system +- module: nginx + + +#output.logstash: +# hosts: ["localhost:5044"] + + +output.elasticsearch: + hosts: ["localhost:9200"] + username: "elastic" + password: "changeme" diff --git a/default_configs/heartbeat.yml b/default_configs/heartbeat.yml new file mode 100644 index 0000000..cc39334 --- /dev/null +++ b/default_configs/heartbeat.yml @@ -0,0 +1,11 @@ +heartbeat.monitors: +- type: http + urls: ["http://localhost:80"] + schedule: '@every 10s' + check.response.status: 200 + + +output.elasticsearch: + hosts: ["localhost:9200"] + username: "elastic" + password: "changeme" diff --git a/default_configs/kibana.yml b/default_configs/kibana.yml new file mode 100644 index 0000000..39eb87d --- /dev/null +++ b/default_configs/kibana.yml @@ -0,0 +1,106 @@ +# Kibana is served by a back end server. This setting specifies the port to use. +#server.port: 5601 + +# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values. +# The default is 'localhost', which usually means remote machines will not be able to connect. +# To allow connections from remote users, set this parameter to a non-loopback address. +#server.host: "localhost" + +# Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects +# the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests +# to Kibana. This setting cannot end in a slash. +#server.basePath: "" + +# The maximum payload size in bytes for incoming server requests. +#server.maxPayloadBytes: 1048576 + +# The Kibana server's name. This is used for display purposes. +#server.name: "your-hostname" + +# The URL of the Elasticsearch instance to use for all your queries. +#elasticsearch.url: "http://localhost:9200" + +# When this setting's value is true Kibana uses the hostname specified in the server.host +# setting. When the value of this setting is false, Kibana uses the hostname of the host +# that connects to this Kibana instance. +#elasticsearch.preserveHost: true + +# Kibana uses an index in Elasticsearch to store saved searches, visualizations and +# dashboards. Kibana creates a new index if the index doesn't already exist. +#kibana.index: ".kibana" + +# The default application to load. +#kibana.defaultAppId: "discover" + +# If your Elasticsearch is protected with basic authentication, these settings provide +# the username and password that the Kibana server uses to perform maintenance on the Kibana +# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which +# is proxied through the Kibana server. +#elasticsearch.username: "user" +#elasticsearch.password: "pass" + +# Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively. +# These settings enable SSL for outgoing requests from the Kibana server to the browser. +#server.ssl.enabled: false +#server.ssl.certificate: /path/to/your/server.crt +#server.ssl.key: /path/to/your/server.key + +# Optional settings that provide the paths to the PEM-format SSL certificate and key files. +# These files validate that your Elasticsearch backend uses the same key files. +#elasticsearch.ssl.certificate: /path/to/your/client.crt +#elasticsearch.ssl.key: /path/to/your/client.key + +# Optional setting that enables you to specify a path to the PEM file for the certificate +# authority for your Elasticsearch instance. +#elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ] + +# To disregard the validity of SSL certificates, change this setting's value to 'none'. +#elasticsearch.ssl.verificationMode: full + +# Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of +# the elasticsearch.requestTimeout setting. +#elasticsearch.pingTimeout: 1500 + +# Time in milliseconds to wait for responses from the back end or Elasticsearch. This value +# must be a positive integer. +#elasticsearch.requestTimeout: 30000 + +# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side +# headers, set this value to [] (an empty list). +#elasticsearch.requestHeadersWhitelist: [ authorization ] + +# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten +# by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration. +#elasticsearch.customHeaders: {} + +# Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable. +#elasticsearch.shardTimeout: 0 + +# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying. +#elasticsearch.startupTimeout: 5000 + +# Specifies the path where Kibana creates the process ID file. +#pid.file: /var/run/kibana.pid + +# Enables you specify a file where Kibana stores log output. +#logging.dest: stdout + +# Set the value of this setting to true to suppress all logging output. +#logging.silent: false + +# Set the value of this setting to true to suppress all logging output other than error messages. +#logging.quiet: false + +# Set the value of this setting to true to log all events, including system usage information +# and all requests. +#logging.verbose: false + +# Set the interval in milliseconds to sample system and process performance +# metrics. Minimum is 100ms. Defaults to 5000. +#ops.interval: 5000 + +# The default locale. This locale can be used in certain circumstances to substitute any missing +# translations. +#i18n.defaultLocale: "en" +server.host: "0.0.0.0" +logging.dest: /var/log/kibana/kibana.log diff --git a/default_configs/metricbeat.yml b/default_configs/metricbeat.yml new file mode 100644 index 0000000..048801c --- /dev/null +++ b/default_configs/metricbeat.yml @@ -0,0 +1,47 @@ +metricbeat.modules: + + +- module: system + metricsets: + - cpu + - load + - core + - diskio + - filesystem + - fsstat + - memory + - network + - process + - socket + enabled: true + period: 10s + processes: ['.*'] + cgroups: true + +- module: redis + metricsets: ["info"] + enabled: true + period: 10s + hosts: ["127.0.0.1:6379"] + +- module: nginx + metricsets: ["stubstatus"] + enabled: true + period: 10s + hosts: ["http://127.0.0.1"] + +- module: mongodb + metricsets: ["status"] + hosts: ["localhost:27017"] + +- module: docker + metricsets: ["cpu", "info", "memory", "network", "diskio", "container"] + hosts: ["unix:///var/run/docker.sock"] + enabled: true + period: 10s + + +output.elasticsearch: + hosts: ["http://localhost:9200"] + username: "elastic" + password: "changeme" diff --git a/default_configs/packetbeat.yml b/default_configs/packetbeat.yml new file mode 100644 index 0000000..90ea6c2 --- /dev/null +++ b/default_configs/packetbeat.yml @@ -0,0 +1,35 @@ +packetbeat.interfaces.device: any + + +packetbeat.flows: + timeout: 30s + period: 10s + + +packetbeat.protocols.icmp: + enabled: true + + +packetbeat.protocols.dns: + ports: [53] + include_authorities: true + include_additionals: true + + +packetbeat.protocols.http: + ports: [80, 5601] + + +packetbeat.protocols.redis: + ports: [6379] + + +packetbeat.protocols.mongodb: + ports: [27017] + + +output.redis: + hosts: ["localhost:6379"] + datatype: "list" + key: "packetbeat" + db: 0 diff --git a/elastic-stack/0_install.yml b/elastic-stack/0_install.yml index c347081..1555a11 100644 --- a/elastic-stack/0_install.yml +++ b/elastic-stack/0_install.yml @@ -46,21 +46,23 @@ apt: name=python-pip state=latest + # https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-using-the-repository - name: Get the Docker key on 64bit apt_key: - keyserver: "hkp://p80.pool.sks-keyservers.net:80" - id: 58118E89F3A912897C070ADBF76221572C52609D + url: "https://download.docker.com/linux/ubuntu/gpg" + id: 0EBFCD88 state: present when: "platform == 'amd64'" - name: Add the Docker repository apt_repository: - repo: "deb https://apt.dockerproject.org/repo ubuntu-xenial main" + repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable" state: present when: "platform == 'amd64'" + # https://docs.docker.com/release-notes/docker-ce/ - name: Install Docker - apt: name=docker-engine state=latest update_cache=yes + apt: name=docker-ce state=latest update_cache=yes when: "platform == 'amd64'" - name: Make sure Docker autostarts and is started diff --git a/elastic-stack/ansible.sh b/elastic-stack/ansible.sh new file mode 100644 index 0000000..d0d2d67 --- /dev/null +++ b/elastic-stack/ansible.sh @@ -0,0 +1,15 @@ +#!/bin/bash -eux + +# set the session to be noninteractive +export DEBIAN_FRONTEND="noninteractive" + +# Install Ansible repository. +apt -y install software-properties-common +apt-add-repository ppa:ansible/ansible + +# Install Ansible. +apt -y update +apt -y install ansible + +mkdir /elastic-stack/ +chmod 0777 /elastic-stack \ No newline at end of file diff --git a/elastic-stack/upgrade_os.sh b/elastic-stack/upgrade_os.sh new file mode 100644 index 0000000..1417e2f --- /dev/null +++ b/elastic-stack/upgrade_os.sh @@ -0,0 +1,19 @@ +#!/bin/bash -eux + +# set the session to be noninteractive +export DEBIAN_FRONTEND="noninteractive" + +# Disable the release upgrader +sed -i.bak 's/^Prompt=.*$/Prompt=never/' /etc/update-manager/release-upgrades + +# Disabling periodic apt upgrades" +echo 'APT::Periodic::Enable "0";' >> /etc/apt/apt.conf.d/10periodic + +# Upgrade OS and kernel +apt -y update +apt-get -y --force-yes upgrade +apt-get -y dist-upgrade --force-yes + +# Remove old packages and configs +apt-get autoremove -y +apt-get purge -y $(dpkg --list |grep '^rc' |awk '{print $2}') diff --git a/elastic-stack/vars.yml b/elastic-stack/vars.yml index 5f63765..602c249 100644 --- a/elastic-stack/vars.yml +++ b/elastic-stack/vars.yml @@ -1,2 +1,2 @@ -elastic_version: 5.5.2 +elastic_version: 5.6.3 elastic_download: https://artifacts.elastic.co diff --git a/packer.json b/packer.json new file mode 100644 index 0000000..4a044b9 --- /dev/null +++ b/packer.json @@ -0,0 +1,72 @@ +{ + "variables": { + "vpc": "{{env `BUILD_VPC_ID`}}", + "subnet": "{{env `BUILD_SUBNET_ID`}}", + "aws_region": "{{env `AWS_REGION`}}", + "ami_name": "packer-{{isotime \"2006-01-02_15-04-06\"}}" + }, + "builders": [{ + "name": "Packer AMI Builder", + "type": "amazon-ebs", + "region": "{{user `aws_region`}}", + "source_ami_filter": { + "filters": { + "name": "ubuntu/images/hvm-ssd/ubuntu-xenial*", + "virtualization-type": "hvm", + "root-device-type": "ebs" + }, + "owners": ["099720109477"], + "most_recent": true + }, + "instance_type": "t2.micro", + "ssh_username": "ubuntu", + "ami_name": "{{user `ami_name` | clean_ami_name}}", + "tags": { + "Name": "{{user `ami_name`}}" + }, + "run_tags": { + "Name": "{{user `ami_name`}}" + }, + "run_volume_tags": { + "Name": "{{user `ami_name`}}" + }, + "snapshot_tags": { + "Name": "{{user `ami_name`}}" + }, + "ami_description": "Packer AMI Builder demo", + "associate_public_ip_address": "true", + "vpc_id": "{{user `vpc`}}", + "subnet_id": "{{user `subnet`}}" + }], + "provisioners": [ + { + "type": "shell", + "script": "elastic-stack/upgrade_os.sh", + "execute_command": "sudo -S bash {{.Path}}" + }, + { + "type": "shell", + "script": "elastic-stack/ansible.sh", + "execute_command": "sudo -S bash {{.Path}}" + }, + { + "type": "ansible-local", + "playbook_file": "./elastic-stack/0_install.yml", + "playbook_dir": "./elastic-stack/", + "staging_directory": "/elastic-stack/" + }, + { + "type": "shell", + "inline": [ + "rm .ssh/authorized_keys ; sudo rm /root/.ssh/authorized_keys" + ] + } + ], + "post-processors": [ + { + "type": "manifest", + "output": "manifest.json", + "strip_path": true + } + ] +} \ No newline at end of file diff --git a/readme.md b/readme.md index 8dbf681..0bc002e 100644 --- a/readme.md +++ b/readme.md @@ -53,6 +53,23 @@ If Vagrant and Ansible sound too complicated, there is also the final result: An * Or use SSH with the same credentials: * Windows: Use [http://www.putty.org](http://www.putty.org) and connect to `vagrant@127.0.0.1` on port 2222. * Mac and Linux: `$ ssh vagrant@127.0.0.1 -p 2222 -o PreferredAuthentications=password` + + + +## Packer AWS AMI + +If you want to deploy this to Amazon, you can use Packer to build the AMI image, using the latest official Ubuntu 16.04 Xenial LTS release from Canonical. You will need: +* Packer https://www.packer.io/downloads.html +* AWS CLI http://docs.aws.amazon.com/cli/latest/userguide/installing.html +* AWS IAM Role to deploy EC2 instances +* jq https://stedolan.github.io/jq/ + +```shell +./build_packer.sh +``` + +The example script is configured to the "us-east-1" region, it will use your locally installed AWS CLI to find the first / default VPC ID, the first subnet within that VPC, build the AMI, save the results to manifest.json file, and use jq to extract the AMI id. +You can easily integrate this into a Jenkins pipeline to automate the build of your ELK stack, and optionally use Terraform to deploy the AMI. @@ -79,3 +96,92 @@ $ java -jar /opt/injector.jar 100000 1000 ## Logstash Demo You can play around with a Logstash example by calling `$ sudo /usr/share/logstash/bin/logstash --path.settings /etc/logstash -f /elastic-stack/raffle/raffle.conf` (it can take some time) and you will find the result in the `raffle` index. + + + +## Release Notes + +Elastic +- https://www.elastic.co/downloads/past-releases + +Elasticsearch +- https://github.com/elastic/elasticsearch/releases +- https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html + +Logstash +- https://github.com/elastic/logstash/releases +- https://www.elastic.co/guide/en/logstash/current/releasenotes.html + +Kibana +- https://github.com/elastic/kibana/releases +- https://www.elastic.co/guide/en/kibana/current/release-notes.html + +Beats +- https://github.com/elastic/beats/releases + +X-Pack +- https://www.elastic.co/guide/en/x-pack/current/xpack-introduction.html +- https://www.elastic.co/guide/en/x-pack/current/xpack-change-list.html + + + +## Log files +```shell + +# filebeat +/var/log/filebeat/filebeat + +# metricbeat +/var/log/metricbeat/metricbeat + +# heartbeat +/var/log/heartbeat/heartbeat + +# logstash +/var/log/logstash/logstash-plain.log +/var/log/logstash/logstash.stdout +/var/log/logstash/logstash.log + +# elasticsearch +/var/log/elasticsearch/elasticsearch.log +/var/log/elasticsearch/elasticsearch.log.* +/var/log/elasticsearch/elasticsearch_deprecation.log +/var/log/elasticsearch/elasticsearch_index_search_slowlog.log +/var/log/elasticsearch/elasticsearch_index_indexing_slowlog.log.log + +# kibana +/var/log/kibana.log +/var/log/kibana/kibana.stderr +/var/log/kibana/kibana.stdout +``` + + + +## Application folders +```shell + +# filebeat +/etc/filebeat +/usr/share/filebeat + +# metricbeat +/etc/metricbeat +/usr/share/metricbeat + +# heartbeat +/etc/heartbeat +/usr/share/heartbeat + +# logstash +/etc/logstash +/usr/share/logstash + +# elasticsearch +/etc/elasticsearch/ +/usr/share/elasticsearch/ + +# kibana +/etc/kibana +/usr/share/kibana +/opt/kibana +``` \ No newline at end of file