# 架构图
# ELK 加入Kafka 消息队列
在 elk-3 上面安装filebeat，通过filebeat 模板 抓取 system 日志
# filebeat
# 加载 system 系统模板
/etc/filebeat/m
filebeat modules enable system
修改
false true
1
2
3
4
2
3
4
# filebeat 配置文件
vim /etc/filebeat/filebeat.yml
##关闭通过log方式获取
- type: log
enabled: false
#service: nginx_log
# Paths that should be crawled and fetched. Glob based paths.
paths:
- /var/log/nginx/access.log
- /var/log/messages
#- c:\programdata\elasticsearch\logs\*
## 开启模板监控
filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml
# Set to true to enable config reloading
reload.enabled: true
# Period on which files under path should be checked for changes
#reload.period: 10s
## 增加 output kafka输出 注释掉 elasticsearch和logstash输出
#output.elasticsearch:
# # Array of hosts to connect to.
# hosts: ["localhost:9200"]
output.kafka:
hosts: ["elk-1:9092"]
topic: "filebeat"
codec.json:
pretty: false
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
#password: "changeme"
#----------------------------- Logstash output --------------------------------
#output.logstash:
# #The Logstash hosts
# hosts: ["127.0.0.1:5044"]
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# logstash
创建 文件
vim /opt/logstash-7.2.0/logstash_kafka.conf
input {
kafka {
bootstrap_servers => "elk-1:9092"
topics => ["filebeat"]
codec => json
}
}
output {
if [@metadata][pipeline] {
elasticsearch {
hosts => "http://localhost:9200"
manage_template => false
index => "system-%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
# pipeline => "%{[@metadata][pipeline]}"
}
} else {
elasticsearch {
hosts => "http://localhost:9200"
manage_template => false
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
}
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 启动logstash
cd /opt/logstash-7.2.0
nohup ./bin/logstash -f logstash_kafka.conf &
1
2
3
4
2
3
4
# kabana
创建索引
