# 架构图
# 一、logstash agent 端 安装配置
# 1.logstash下载安装配置（web_nginx agent）
wget https://download.elastic.co/logstash/logstash/logstash-2.2.2.tar.gz
tar -xf logstash-2.2.2.tar.gz
cd logstash
>>安装JAVA
yum install java-1.8.0-openjdk.x86_64
1
2
3
4
5
2
3
4
5
# 2.添加logstash配置文件
vim shipper.conf
input {
file{
type => "nginx-www-access"
path => "/app/nginx/logs/*access.log"
exclude => "vlog_access.log"
start_position => "end"
}
}
filter {
if [type] == 'nginx-www-access' or [type] == 'flash-vlog' {
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
grok {
match => { "message" => "%{NGINXACCESS}" }
}
mutate {
gsub => ["x_forwarded_for", ",.*", ""]
}
if [x_forwarded_for] == '-' {
mutate {
replace => { "x_forwarded_for" => "%{clientip}" }
}
}
mutate {
convert => [ "upstream_response_time", "float"]
}
mutate {
convert => [ "request_time", "float"]
}
geoip {
source => "x_forwarded_for"
target => "geoip"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
}
}
output {
redis{
host => "IP"
data_type => "list"
key => "logstash:web"
port => "6379"
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# 添加logstash nginx日志格式配置
mkdir -p logstash_dir/patterns
cd logstash_dir/patterns
vim nginx
**添加如下内容**
NGINXACCESS %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} "%{DATA:x_forwarded_for}" %{DATA:request_body} %{IPORHOST:httphost} "%{DATA:cookie}" (?:%{NUMBER:upstream_response_time:float}|-) (?:%{NUMBER:request_time:float}|-)
**修改nginx 配置文件**
log_format short '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forward
ed_for" $request_body $host "$http_cookie" ' '$upstream_response_time $request_time ';
重新加载 nginx 配置文件
1
2
3
4
5
6
7
8
9
10
11
12
13
2
3
4
5
6
7
8
9
10
11
12
13
# 二、安装redis
# 2.1安装配置redis
wget 下载源码包
解压
cd dir
make test
make prefix=/dir/ install
1
2
3
4
5
2
3
4
5
# 2.2配置文件修改
daemonize yes save 900 1 save 300 10 save 60 10000
# 三、服务器端ELK安装和配置
# 3.1 logstash-server
# 3.1.1 安装
wget https://download.elastic.co/logstash/logstash/logstash-2.2.2.tar.gz
tar -xf logstash-2.2.2.tar.gz
cd logstash
>>安装JAVA
yum install java-1.8.0-openjdk.x86_64
1
2
3
4
5
2
3
4
5
# 3.1.2 配置文件更改
input {
redis {
host => 'IP'
data_type => 'list'
port => "6379"
key => 'logstash:web'
type => 'redis-input'
#threads => 5
threads => 10
}
}
output {
#stdout { }
elasticsearch {
hosts => "IP:9200"
}
}
./bin/logstash -f
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 3.2 install elasticsearch
# 3.2.1 安装
wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.2.1/elasticsearch-2.2.1.tar.gz
tar -xf elasticsearch
cd elasticsearch
1
2
3
2
3
# 3.2.2 配置文件更改
vim config/elasticsearch.yml
修改如下：
# network.host: 192.168.0.1 更改为本机IP
# http.port: 9200 端口默认为9200 iptables增加端口认证
./bin/elasticsearch
1
2
3
4
5
2
3
4
5
# 3.3 安装配置 kibana
# 3.3.1 安装
wget https://download.elastic.co/kibana/kibana/kibana-4.4.2-linux-x64.tar.gz
tar -xf kibana
cd kibana
1
2
3
2
3
# 3.3.2 配置文件更改
修改配置文件
vim config/kibana.yml
# elasticsearch.url: "http://localhost:9200" 更改为elasticsearch IP
# kibana.index: ".kibana" 注释取消
# server.port: 5601
./bin/kibana
1
2
3
4
5
6
2
3
4
5
6