1、logstash简介
2、logstash安装及配置
[root@foundation50 7.6]# scp logstash-7.6.1.rpm server4: 拷贝下载的软件到server4，版本和elasticsearch保持一致
[root@foundation50 docs]# cd hadoop/
[root@foundation50 hadoop]# scp jdk-8u181-linux-x64.rpm server4: 拷贝下载的软件jdk到server4上
[root@server4 ~]# rpm -ivh jdk-8u181-linux-x64.rpm 安装jdk
[root@server4 ~]# rpm -ivh logstash-7.6.1.rpm 安装logstash
2.1标准输入和标准输出
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -e 'input { stdin { } } output { stdout {} }' /usr/share/logstash/bin/logstash 为logstash安装位置， stdin表示标准输入，output表示标准输出
stash API endpoint {:port=>9600}
westos 输入啥内容就会输出什么内容
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"host" => "server4",
"@version" => "1",
"message" => "westos",
"@timestamp" => 2022-05-17T10:50:50.438Z
}
hello 输入啥内容就会输出什么内容
{
"host" => "server4",
"@version" => "1",
"message" => "hello",
"@timestamp" => 2022-05-17T10:52:41.980Z
}
这是一个简单的演示，意义不大,通常使用以下方式：
[root@server4 ~]# cd /etc/logstash/conf.d
[root@server4 conf.d]# vim test.conf
input {
stdin {}
}
output {
stdout {}
}
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf -f表示指定test.conf文件
stash API endpoint {:port=>9600}
westos 输入什么就输出什么
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"host" => "server4",
"message" => "westos",
"@timestamp" => 2022-05-17T11:28:16.507Z,
"@version" => "1"
}
2.2标准输入到文件
[root@server4 conf.d]# vim test.conf
input {
stdin {}
}
output {
file {
path => "/tmp/logstash.txt" 表示输出的文件路经
codec => line { format => "custom format: %{message}"} codec表示格式化行，在每行开头加custom %{message}表示输入的内容
}
}
stash API endpoint {:port=>9600}
hello
[INFO ] 2022-05-17 19:40:00.632 [[main]>worker1] file - Opening file {:path=>"/tmp/logstash.txt"}
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf
[root@server4 conf.d]# cat /tmp/logstash.txt
custom format: hello 刚才输入的内容保存到文件里了
[root@server4 conf.d]# vim test.conf
input {
stdin {}
}
output {
stdout {} 可以添加多个模块 ，这样不仅可以在终端里输出，也可以把输出保存在文件里
file {
path => "/tmp/logstash.txt"
codec => line { format => "custom format: %{message}"}
}
}
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf
stash API endpoint {:port=>9600}
westos 输入westos
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"message" => "westos", 既可以输出到终端
"host" => "server4",
"@timestamp" => 2022-05-17T14:09:16.038Z,
"@version" => "1"
}
[INFO ] 2022-05-17 22:09:16.527 [[main]>worker0] file - Opening file {:path=>"/tmp/logstash.txt"} 也可以保存到文件中
2.3标准输入到es主机
[root@server4 ~]# cd /etc/logstash/conf.d
[root@server4 conf.d]# vim test.conf
input {
stdin {}
}
output {
stdout {}
#file {
# path => "/tmp/logstash.txt"
# codec => line { format => "custom format: %{message}"}
#}
elasticsearch {
hosts => ["172.25.50.1:9200"] 输出到es集群的任何一个主机
index => "messagelog-%{+YYYY.MM.dd}" 定制索引名称，每天会更新一份
}
}
访问172.25.50.1:9200
如何把文件输出到ES里面
[root@server4 conf.d]# ll /var/log/messages 将日志文件输出到es上
-rw------- 1 root root 452334 May 17 23:01 /var/log/messages
[root@server4 conf.d]# chmod 644 /var/log/messages 给普通用户读的权限
查看插件用法
进入官网
[root@server4 conf.d]# vim test.conf
input {
file {
path => "/var/log/messages" 指定文件路经
start_position => "beginning" 开始位置，从开头开始
}
}
output {
stdout {}
#file {
# path => "/tmp/logstash.txt"
# codec => line { format => "custom format: %{message}"}
#}
elasticsearch {
hosts => ["172.25.50.1:9200"]
index => "messagelog-%{+YYYY.MM.dd}"
}
}
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf 运行
当我们把elasticsearch上的索引删除了
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf
当我们重新运行，此时不会把message内容再次写到ES上，只是将文件里更新的内容写道ES上，因为假如当logstash重启如果把文
件里的数据重复写道ES上就会造成数据冗余，此时我们要根据进度号，找到目前读到那了，然后接着往下面读
[root@server4 conf.d]# cd /usr/share/logstash/
[root@server4 logstash]# cd data/
[root@server4 data]# cd plugins/
[root@server4 plugins]# ls
inputs
[root@server4 plugins]# cd inputs/
[root@server4 inputs]# ls
file
[root@server4 inputs]# cd file/
[root@server4 file]# l.
. .. .sincedb_452905a167cf4509fd08acb964fdb20c
[root@server4 file]# cat .sincedb_452905a167cf4509fd08acb964fdb20c
51098409 0 64768 452399 1652806389.204203 /var/log/messages 文件一共分为六段，分别代表的意思如下图：
[root@server4 file]# rm -fr .sincedb_452905a167cf4509fd08acb964fdb20c 当我们需要把文件内容重新读一编,就需要将此文件删除
[root@server4 file]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf 此时再次运行，将会把message文件的数据重新导一份到ES上
[root@server4 ~]# cd /usr/share/logstash/data/plugins/inputs/file/
[root@server4 file]# l.
. .. .sincedb_452905a167cf4509fd08acb964fdb20c
[root@server4 file]# cat .sincedb_452905a167cf4509fd08acb964fdb20c 再次查看，就会生成新的文件，纪录读取进度（偏移量）
51098409 0 64768 452600 1652807840.39266 /var/log/messages
[root@server4 file]# logger hello world 在日志文件messages里面写一条内容
[root@server4 file]# cat .sincedb_452905a167cf4509fd08acb964fdb20c 查看文件，发现文件变化了，偏移量变成了452642
51098409 0 64768 452642 1652808015.583087 /var/log/messages
2.4 Syslog输入插件—实现简单的日志采集及管理
[root@server4 conf.d]# vim test.conf
input {
#file {
#path => "/var/log/messages"
#start_position => "beginning"
#}
syslog {} 当添加这个插件表示logstash可以伪装成一个日志收集器
}
output {
stdout {}
#file {
# path => "/tmp/logstash.txt"
# codec => line { format => "custom format: %{message}"}
#}
elasticsearch {
hosts => ["172.25.50.1:9200"]
index => "syslog-%{+YYYY.MM.dd}"
}
}
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf 运行
[root@server4 ~]# netstat -antlpu | grep :514 查看端口，已经开启，此时server4就是一个日志采集服务器
tcp6 0 0 :::514 :::* LISTEN 23378/java
udp 0 0 0.0.0.0:514 0.0.0.0:* 23378/java
此时我们将server1上的日志远程同步到server4上
[root@server1 ~]# vim /etc/rsyslog.conf 编辑
[root@server1 ~]# systemctl restart rsyslog.service 重启服务
[root@server1 ~]# logger hello server1 此时添加一条日志信息
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf 运行，可以看出server1会将添加的日志信息，发送到远端日志采集服务器
同样server2，进行同样配置
[root@server2 elasticsearch]# vim /etc/rsyslog.conf
[root@server2 elasticsearch]# systemctl restart rsyslog.service 重启服务
[root@server2 elasticsearch]# logger hello server2
2.5 多行过滤插件
[root@server4 conf.d]# vim demo.conf
input {
stdin {
codec => multiline { multiline表示多行
pattern => "EOF" 表示从那输到那算一个整体，当匹配EOF关键字时输入结束
negate => "true" 表示是否匹配到,true表示匹配到
what => "previous" 表示向上合并还是向下合并，previous表示向上合并，next表示向下合并
}
}
}
output {
stdout{}
}
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/demo.conf 运行
stash API endpoint {:port=>9600}
1 多行输入
2
3
4
5
EOF 当输入EOF结束
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"@version" => "1",
"host" => "server4",
"tags" => [
[0] "multiline"
],
"message" => "1\n2\n3\n4\n5", 将上述5行内容汇总成一行
"@timestamp" => 2022-05-17T21:49:09.851Z
}
[root@server2 ~]# cd /var/log/elasticsearch/
[root@server2 elasticsearch]# gunzip my-es-2022-05-17-1.log.gz 解压,此日志里面含有多行，用于实验
[root@server2 elasticsearch]# scp my-es-2022-05-17-1.log server4: /var/log 将日志拷贝到server4上
[root@server4 ~]# ll /var/log/my-es-2022-05-17-1.log 查看权限644，有读的权限
-rw-r--r-- 1 root root 118249 May 18 06:14 /var/log/my-es-2022-05-17-1.log
input {
file {
path => "/var/log/my-es-2022-05-17-1.log"
start_position => "beginning"
codec => multiline { 添加多行插件，应为日志里面有多行
pattern => "^\[" 表示以中括号开头的为结束（这个需要在日志文件观察，什么时侯多行结束）
negate => "true"
what => "previous"
}
}
#syslog {}
}
output {
stdout {}
#file {
# path => "/tmp/logstash.txt"
# codec => line { format => "custom format: %{message}"}
#}
elasticsearch {
hosts => ["172.25.50.1:9200"]
index => "eslog-%{+YYYY.MM.dd}"
}
}
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf 运行
2.6 grok过滤插件
[root@server4 elasticsearch-head-master]# cd /etc/logstash/conf.d/
[root@server4 conf.d]# vim demo.conf
input {
stdin{}
}
filter {
grok {
match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" } 与之匹配的日志表达式
}
}
output {
stdout{}
}
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/demo.conf 运行
stash API endpoint {:port=>9600}
55.3.244.1 GET /index.html 15824 0.043 输入一段日志信息
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"@timestamp" => 2022-05-18T10:50:00.609Z, 将这段信息进行预处理。切片处理
"message" => " 55.3.244.1 GET /index.html 15824 0.043",
"duration" => "0.043",
"@version" => "1",
"bytes" => "15824",
"request" => "/index.html",
"client" => "55.3.244.1",
"method" => "GET",
"host" => "server4"
}
示例二：
[root@server4 conf.d]# yum install httpd -y 安装apache
[root@server4 conf.d]# systemctl start httpd.service 启动
[root@server4 conf.d]# echo www.westos.org > /var/www/html/index.html 创建一个首页
[root@server4 conf.d]# cat /var/log/httpd/access_log 查看httpd日志
172.25.50.250 - - [18/May/2022:19:18:41 +0800] "GET / HTTP/1.1" 200 15 "-" "curl/7.61.1"
现在我们要写匹配的日志表达式我们不会写，但是本机上有一些内置的日志表达式可以参考，如下：
[root@server4 conf.d]# cd /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-patterns-core-4.1.2/patterns/
[root@server4 patterns]# ls
aws exim httpd maven nagios ruby
bacula firewalls java mcollective postgresql squid
bind grok-patterns junos mcollective-patterns rails
bro haproxy linux-syslog mongodb redis
[root@server4 patterns]# cat httpd 查看
HTTPDUSER %{EMAILADDRESS}|%{USER}
HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
# Log formats htppd日志表达式，有两种格式
HTTPD_COMMONLOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent} 当前用的是此日志格式，我们直接取变量即可
查看apache是那种格式
[root@server4 patterns]# vim /etc/httpd/conf/httpd.conf
/var/log/httpd/access_log 查看日志文件以及目录有没查看的权限
[root@server4 conf.d]# ll /var/log/httpd/access_log
-rw-r--r-- 1 root root 168 May 18 19:18 /var/log/httpd/access_log 文件有权限644
[root@server4 conf.d]# ll -d /var/log/
drwxr-xr-x. 9 root root 4096 May 18 23:14 /var/log/ 有权限
[root@server4 conf.d]# ll -d /var/log/httpd
drwx------ 2 root root 41 May 18 18:57 /var/log/httpd httpd目录没有权限，虽然文件权限，但是目录没有权限，无法查看
[root@server4 conf.d]# chmod 755 /var/log/httpd/ 设置权限
[root@server4 conf.d]# mv demo.conf apache.conf
[root@server4 conf.d]# vim apache.conf
input {
file{
path => "/var/log/httpd/access_log" 指定文件路经
start_position => "beginning" 从开头开始
}
}
filter {
grok {
match => { "message" => "%{HTTPD_COMBINEDLOG}" } 日志匹配格式
}
}
output {
stdout{}
elasticsearch { 输出到es上
hosts => ["172.25.50.1:9200"]
index => "apachelog-%{+YYYY.MM.dd}"
}
}
[root@server4 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/apache.conf 运行
stash API endpoint {:port=>9600}
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"@version" => "1", 可以看出将日志切割成一个一个的
"httpversion" => "1.1",
"referrer" => "\"-\"",
"message" => "::1 - - [18/May/2022:19:18:12 +0800] \"GET / HTTP/1.1\" 200 15 \"-\" \"curl/7.29.0\"",
"bytes" => "15",
"agent" => "\"curl/7.29.0\"",
"timestamp" => "18/May/2022:19:18:12 +0800",
"@timestamp" => 2022-05-18T16:30:20.964Z,
"ident" => "-",
"host" => "server4",
"clientip" => "::1",
"verb" => "GET",
"response" => "200",
"path" => "/var/log/httpd/access_log",
"auth" => "-",
"request" => "/"
}
版权声明：本文为qq_43114229原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。