install logstash | zuhdi.org

Install Logstash

Environment

  • Debian 9.7 x64 (RAM 4GB)
  • OpenJDK 1.8.0
  • Elasticsearch 7.1.1
  • Logstash 7.1.1
  • Filebeat 7.1.1

Pre-requisite Install Elasticsearch

Excerpt

1. Install OpenJDK 8

root@athos:~# apt-get update && apt-get -y upgrade && apt-get -y dist-upgrade

root@athos:~# apt-get -y --no-install-recommends install openjdk-8-jre-headless

root@athos:~# java -version
openjdk version "1.8.0_212"
OpenJDK Runtime Environment (build 1.8.0_212-8u212-b03-2~deb9u1-b03)
OpenJDK 64-Bit Server VM (build 25.212-b03, mixed mode)

2. Install Logstash

root@athos:~# wget -qO- https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
OK

echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" > \
  /etc/apt/sources.list.d/elastic-7.x.list

root@athos:~# echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" > \
>   /etc/apt/sources.list.d/elastic-7.x.list

root@athos:~# apt-get -y --no-install-recommends install apt-transport-https

root@athos:~# apt-get update && apt-get -y --no-install-recommends install logstash

root@athos:~# /usr/share/logstash/bin/logstash --version
logstash 7.1.1

3. Test Run

/usr/share/logstash/bin/logstash \
  -e 'input { stdin { } } output { stdout {} }'

root@athos:~# /usr/share/logstash/bin/logstash \
>   -e 'input { stdin { } } output { stdout {} }'
(truncated)
The stdin plugin is now waiting for input:
[INFO ] 2019-05-30 04:26:00.850 [Api Webserver] agent 
  - Successfully started Logstash API endpoint {:port=>9600}
Hello, World!
{
      "@version" => "1",
    "@timestamp" => 2019-05-30T04:27:53.253Z,
       "message" => "Hello, World!",
          "host" => "athos"
}
[INFO ] 2019-05-30 04:28:24.819 [LogStash::Runner] runner - Logstash shut down.

4. Configure Filebeat (Test)

root@athos:~# apt-get -y --no-install-recommends install curl

wget -q \
  https://download.elastic.co/demos/logstash/gettingstarted/logstash-tutorial.log.gz

root@athos:~# wget -q \
>   https://download.elastic.co/demos/logstash/gettingstarted/logstash-tutorial.log.gz

root@athos:~# gunzip -c logstash-tutorial.log.gz > sample.log

curl -sSL -O \
  https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.1.1-linux-x86_64.tar.gz

root@athos:~# curl -sSL -O \
> https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.1.1-linux-x86_64.tar.gz

root@athos:~# tar -xf filebeat-7.1.1-linux-x86_64.tar.gz

root@athos:~# ~/filebeat-7.1.1-linux-x86_64/filebeat version
filebeat version 7.1.1 (amd64), libbeat 7.1.1 
  [3358d9a5a09e3c6709a2d3aaafde628ea34e8419 built 2019-05-23 13:21:33 +0000 UTC]

cat <<EOF > filebeat.yml
filebeat.inputs:
- type: log
  paths:
    - /root/sample.log
output.logstash:
  hosts: ["localhost:5044"]
EOF

root@athos:~# cat <<EOF > filebeat.yml
> filebeat.inputs:
> - type: log
>   paths:
>     - /root/sample.log
> output.logstash:
>   hosts: ["localhost:5044"]
> EOF

root@athos:~# ~/filebeat-7.1.1-linux-x86_64/filebeat -e -c filebeat.yml -d "publish"
2019-05-30T04:32:57.903Z        INFO    crawler/crawler.go:72   Loading Inputs: 1
2019-05-30T04:32:57.903Z        INFO    log/input.go:148        
  Configured paths: [/root/sample.log]
2019-05-30T04:32:57.903Z        INFO    input/input.go:114      
  Starting input of type: log; ID: 13188218236747279507
(truncated)

5. Configure Logstash For Filebeat Input (Test)

cat <<EOF > pipeline.conf
input {
    beats {
        port => "5044"
    }
}
output {
    stdout { codec => rubydebug }
}
EOF

root@logstash:~# cat <<EOF > pipeline.conf
> input {
>     beats {
>         port => "5044"
>     }
> }
> output {
>     stdout { codec => rubydebug }
> }
> EOF

/usr/share/logstash/bin/logstash -f pipeline.conf \
  --config.test_and_exit

root@athos:~# /usr/share/logstash/bin/logstash -f pipeline.conf \
>   --config.test_and_exit
Configuration OK
[INFO ] 2019-05-30 04:35:20.595 [LogStash::Runner] runner 
  - Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash

/usr/share/logstash/bin/logstash -f pipeline.conf \
  --config.reload.automatic

root@athos:~# /usr/share/logstash/bin/logstash -f pipeline.conf \
>   --config.reload.automatic
(truncated)
[INFO ] 2019-05-30 04:36:19.148 [[main]-pipeline-manager] beats 
  - Beats inputs: Starting input listener {:address=>"0.0.0.0:5044"}
[INFO ] 2019-05-30 04:36:19.340 [[main]<beats] Server - Starting server on port: 5044
{
         "agent" => {
             "version" => "7.1.0",
            "hostname" => "athos",
        "ephemeral_id" => "a0176db0-1da1-4945-8d4d-82e3167fc7b3",
                "type" => "filebeat",
                  "id" => "4b63d78c-2724-48b7-afe2-8ee6d0763ce3"
    },
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ],
          "host" => {
        "name" => "athos"
    },
       "message" => "83.149.9.216 - - [04/Jan/2015:05:13:42 +0000] (truncated)",
    "@timestamp" => 2019-05-29T05:20:20.361Z,
         "input" => {
        "type" => "log"
    },
      "@version" => "1",
           "log" => {
          "file" => {
            "path" => "/root/sample.log"
        },
        "offset" => 0
    },
           "ecs" => {
        "version" => "1.0.0"
    }
}
^C
[WARN ] 2019-05-30 04:38:10.041 [SIGINT handler] runner - SIGINT received. Shutting down.
[INFO ] 2019-05-30 06:06:20.018 [LogStash::Runner] runner - Logstash shut down.

6. systemd

root@athos:~# systemctl enable --now logstash.service
Created symlink /etc/systemd/system/multi-user.target.wants/logstash.service 
  → /etc/systemd/system/logstash.service.

root@athos:~# systemctl status logstash.service
● logstash.service - logstash
   Loaded: loaded (/etc/systemd/system/logstash.service; enabled; vendor preset: enabled)
   Active: active (running) since Thu 2019-05-30 04:58:07 UTC; 22s ago
 Main PID: 9121 (java)
    Tasks: 18 (limit: 4915)
   CGroup: /system.slice/logstash.service
           └─9121 /usr/bin/java -Xms1g -Xmx1g -XX:+UseConcMarkSweepGC (truncated)

7. Apache HTTP Server Log

root@athos:~# apt-get -y --no-install-recommends install apache2

cat <<EOF > /etc/logstash/conf.d/apache2.conf
input {
  file {
    path => "/var/log/apache2/access.log"
  }
}

output {
  elasticsearch { 
    hosts => "localhost"
    index => "apache2"
  }
}
EOF

root@athos:~# cat <<EOF > /etc/logstash/conf.d/apache2.conf
> input {
>   file {
>     path => "/var/log/apache2/access.log"
>   }
> }
>
> output {
>   elasticsearch {
>     hosts => "localhost"
>     index => "apache2"
>   }
> }
> EOF

/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/apache2.conf \
  --config.test_and_exit

root@athos:~# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/apache2.conf \
>   --config.test_and_exit
Configuration OK
[INFO ] 2019-05-30 05:39:29.605 [LogStash::Runner] runner 
  - Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash

root@athos:~# ls -lF /var/log/apache2/
total 12
-rw-r----- 1 root adm 6399 May 30 05:12 access.log
-rw-r----- 1 root adm  279 May 30 04:41 error.log
-rw-r----- 1 root adm    0 May 30 04:41 other_vhosts_access.log

root@athos:~# usermod -aG adm logstash
sent invalidate(passwd) request, exiting
sent invalidate(group) request, exiting
sent invalidate(passwd) request, exiting
sent invalidate(group) request, exiting

root@athos:~# id logstash
uid=999(logstash) gid=999(logstash) groups=999(logstash),4(adm)

root@athos:~# systemctl restart logstash.service

root@athos:~# curl -X GET "localhost:9200/_cat/indices?v"
health status index   uuid                   pri rep docs.count store.size pri.store.size
yellow open   apache2 woswnL2FTp2ulEDeKxaNXg   1   1          1      6.5kb          6.5kb

Hugo. Malte Kiefer & Zuhdi Najib.