nginx-logstash

发表于

nginx日志格式定义

1
2
3
log_format glog '$remote_addr [$time_local] $request_method $scheme $http_host "$request_uri" '
                '$status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time '
                        '$hostname "$request_body" $req_id';

lua 定义请求id

1
2
3
4
5
6
7
set_by_lua_block $req_id {
	if ngx.var.http_x_nl_request_id == nil then
		return io.open("/proc/sys/kernel/random/uuid"):read()
	else
		return ngx.var.http_x_nl_request_id
	end
}

logstash配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
input {
	stdin {
	}
}

filter {
	grok {
		match => { "message" => "%{IPORHOST:clientip} \[%{HTTPDATE:timestamp}\] %{WORD:verb} %{URIPROTO:scheme} %{URIHOST:domain} \"%{URIPATH:uri_path}(?:%{URIPARAM:uri_args})?\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:\"(?:%{URI:referrer}|-)\"|%{QS:referrer}) %{QS:agent} %{BASE10NUM:request_duration} %{HOSTNAME:hostname} \"%{DATA:post_data}\" %{UUID:uuid}"}
	}
	date {
		match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z"]  
		target => "@timestamp"    
		remove_field => ["timestamp"]
	}
	mutate {
		split => ["uri_args", "?"]
	}
	mutate {
		join => ["uri_args", ""]
	}
	kv {
            source => "uri_args"
            field_split => "&"
            target => "args"
	}

	urldecode{
		field=>[post_data]
	}

	useragent {
		source => "agent"
		target => "ua"
	}

	geoip {
		source => "clientip"
	}
}
output {
	elasticsearch {
		hosts => [ "192.168.1.103:9200" ]
		index => "logstash-test1-%{+YYYY.MM.dd}"
    		document_id => "%{uuid}"
		template_overwrite => true
	}
	stdout { codec => rubydebug }
}

logstash-7.0.1/bin/logstash -f logstash-7.0.1/config/conf.d/nginx.conf –config.reload.automatic