Filebeat
Filebeat介绍与部署
Filebeat附带预构建的模块，这些模块包含收集、解析、充实和可视化各种日志文件格式数据所需的配置，每个Filebeat模块由一个或多个文件集组成，这些文件集包含摄取节点管道、Elasticsearch模板、Filebeat勘探者配置和Kibana仪表盘。
Filebeat模块很好的入门，它是轻量级单用途的日志收集工具，用于在没有安装java的服务器上专门收集日志，可以将日志转发到logstash、elasticsearch或redis等场景中进行下一步处理。
Filebeat部署
官方文档:https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html
官网下载地址:https://www.elastic.co/downloads/beats/filebeat
安装
# 下拉官方rpm包
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.6.16-x86_64.rpm
# 安装filebeat
yum localinstall -y filebeat-5.6.16-x86_64.rpm
# 备份配置文件
cp /etc/filebeat/filebeat.yml{,.bak}
filebeat收集日志到本地文件
# 编辑配置文件
vim /etc/filebeat/filebeat.yml
filebeat.prospectors:
- input_type: log
paths:
- /var/log/nginx/8081_access_json.log
# 不收集的行
exclude_lins: ["^DBG","^$"]
# 日志类型
document_type: 8081_log
- input_type: log
paths:
- /var/log/nginx/8082_access_json.log
# 不收集的行
exclude_lins: ["^DBG","^$"]
# 日志类型
document_type: 8082_log
output.file:
path: "/tmp"
filename: "water.txt"
# 启动服务
systemctl start filebeat
# 检查进程
[root@elk03 ~]# ps -ef | grep filebeat
root 9502 1 0 10:13 ? 00:00:00 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat
root 9525 8809 0 10:17 pts/0 00:00:00 grep --color=auto filebeat
# 刷新nginx页面生成日志信息
# 查看日志信息
[root@elk03 ~]# cat /tmp/water.txt |grep blog.yys.com
{"@timestamp":"2023-07-17T02:13:57.593Z","beat":{"hostname":"elk03","name":"elk03","version":"5.6.16"},"input_type":"log","message":"{\"@timestamp\":\"2023-07-17T10:13:54+08:00\",\"host\":\"10.0.0.104\",\"clientip\":\"10.0.0.104\",\"size\":0,\"responsetime\":0.000,\"upstreamtime\":\"-\",\"upstreamhost\":\"-\",\"http_host\":\"blog.yys.com\",\"url\":\"/index.html\",\"domain\":\"blog.yys.com\",\"xff\":\"10.0.0.1\",\"referer\":\"-\",\"status\":\"304\"}","offset":11457,"source":"/var/log/nginx/8081_access_json.log","type":"8081_log"}
[root@elk03 ~]# cat /tmp/water.txt |grep www.xxx.com
{"@timestamp":"2023-07-17T02:13:50.592Z","beat":{"hostname":"elk03","name":"elk03","version":"5.6.16"},"input_type":"log","message":"{\"@timestamp\":\"2023-07-14T09:57:14+08:00\",\"host\":\"10.0.0.104\",\"clientip\":\"10.0.0.104\",\"size\":16,\"responsetime\":0.000,\"upstreamtime\":\"-\",\"upstreamhost\":\"-\",\"http_host\":\"www.xxx.com\",\"url\":\"/index.html\",\"domain\":\"www.xxx.com\",\"xff\":\"10.0.0.1\",\"referer\":\"-\",\"status\":\"200\"}","offset":271,"source":"/var/log/nginx/8081_access_json.log","type":"8081_log"}
filebeat收集日志输出到ES
# 编写配置文件
vim /etc/filebeat/filebeat.yml
filebeat.prospectors:
- input_type: log
paths:
- /var/log/nginx/8081_access_json.log
# 不收集的行
exclude_lins: ["^DBG","^$"]
# 日志类型
document_type: 8081_log
- input_type: log
paths:
- /var/log/nginx/8082_access_json.log
# 不收集的行
exclude_lins: ["^DBG","^$"]
# 日志类型
document_type: 8082_log
output.logstash:
hosts: ["10.0.0.105:6666"]
# 是否开启输出到logstash 默认就是true
enabled: true
# 工作进程数
worker: 1
# 压缩级别 3
compression_level: 3
# 多个输出的时候开启负载
# loadbalance: true
# 重启filebeat
systemctl restart filebeat
# 检查进程
[root@elk03 ~]# ps -ef |grep filebeat
root 9557 1 0 10:24 ? 00:00:00 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat
root 9567 8809 0 10:24 pts/0 00:00:00 grep --color=auto filebeat
elk04的配置文件
# 编写配置文件
vim /etc/logstash/conf.d/bit.yyy.conf
input{
beats{
port => '6666'
codec => 'json'
}
}
filter{
json{
source => 'message'
remove_field => ['message']
}
}
output{
elasticsearch{
hosts => ['10.0.0.103:9200']
index => '%{type}-%{+yyyy.MM.dd}'
codec => 'json'
}
}
[root@elk04 ~]# logstash -f /etc/logstash/conf.d/bit.yyy.conf
filebeat将日志输出到redis
# 编写配置文件
vim /etc/filebeat/filebeat.yml
filebeat.prospectors:
- input_type: log
paths:
- /var/log/nginx/8081_access_json.log
# 不收集的行
exclude_lins: ["^DBG","^$"]
# 日志类型
document_type: 8081_log
- input_type: log
paths:
- /var/log/nginx/8082_access_json.log
# 不收集的行
exclude_lins: ["^DBG","^$"]
# 日志类型
document_type: 8082_log
output.redis:
hosts: ["10.0.0.51:6379"]
# redis里key的名字
key: 'nginx_fw_log'
# 设置第6库
db: 6
# 设置超时时间
timeout: 5
# 如果设置了redis密码
# passwd: xxx
# 如果没有进程
systemctl stop filebeat
rm -fr /var/lib/filebeat/registry
systemctl start filebeat
# 启动服务
systemctl start filebeat
# redis查看是否导入成功
[root@db01 ~]# redis-cli
127.0.0.1:6379> select 6
OK
127.0.0.1:6379[6]> keys *
1) "nginx_fw_log"