测试Nginx日志数据

80.82.77.139 - - [18/Dec/2019:03:44:32 +0800] "GET /.well-known/security.txt HTTP/1.1" 404 162 "-" "-"
110.249.201.135 - - [18/Dec/2019:03:44:36 +0800] "GET /index.php/archives/295/ HTTP/2.0" 200 7373 "-" "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3453.1882 Mobile Safari/537.36; Bytespider;https://zhanzhang.toutiao.com/"
111.225.149.81 - - [18/Dec/2019:04:04:21 +0800] "GET /index.php/category/EasySwoole/ HTTP/2.0" 200 4055 "-" "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.6874.1274 Mobile Safari/537.36; Bytespider;https://zhanzhang.toutiao.com/"
111.225.149.49 - - [18/Dec/2019:04:10:03 +0800] "GET /index.php/archives/276/ HTTP/2.0" 200 7037 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.9608.1452 Mobile Safari/537.36; Bytespider;https://zhanzhang.toutiao.com/"
193.188.22.123 - - [18/Dec/2019:04:12:20 +0800] "\x03\x00\x00/*\xE0\x00\x00\x00\x00\x00Cookie: mstshash=Administr" 400 166 "-" "-"
111.225.148.184 - - [18/Dec/2019:04:14:56 +0800] "GET /index.php/archives/50/ HTTP/2.0" 200 6405 "-" "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.9366.1845 Mobile Safari/537.36; Bytespider;https://zhanzhang.toutiao.com/"
110.249.201.234 - - [18/Dec/2019:04:41:54 +0800] "GET /index.php/archives/220/ HTTP/2.0" 200 4234 "-" "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.5593.1727 Mobile Safari/537.36; Bytespider;https://zhanzhang.toutiao.com/"
110.249.202.210 - - [18/Dec/2019:04:44:13 +0800] "GET /index.php/archives/52/ HTTP/2.0" 200 6113 "-" "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.7140.1012 Mobile Safari/537.36; Bytespider;https://zhanzhang.toutiao.com/"
110.249.202.47 - - [18/Dec/2019:04:51:42 +0800] "GET /index.php/archives/178/ HTTP/2.0" 200 6738 "-" "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.6790.1511 Mobile Safari/537.36; Bytespider;https://zhanzhang.toutiao.com/"
111.225.148.243 - - [18/Dec/2019:04:56:18 +0800] "GET /index.php/archives/293/ HTTP/2.0" 200 10377 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.7914.1170 Mobile Safari/537.36; Bytespider;https://zhanzhang.toutiao.com/"
#
103.219.184.78 - - [18/Dec/2019:16:09:39 +0800] "GET /index.php/archives/102/ HTTP/2.0" 200 6421 "https://blog.qvbilam.xin/index.php/archives/89/" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36"
210.22.143.82 - - [18/Dec/2019:16:13:17 +0800] "GET /index.php/archives/102/ HTTP/2.0" 200 6460 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36"
210.22.143.82 - - [18/Dec/2019:16:13:41 +0800] "GET / HTTP/2.0" 200 4464 "https://blog.qvbilam.xin/index.php/archives/102/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36"
210.22.143.82 - - [18/Dec/2019:16:14:33 +0800] "GET /index.php/page/2/ HTTP/2.0" 200 4112 "https://blog.qvbilam.xin/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36"
210.22.143.82 - - [18/Dec/2019:16:14:34 +0800] "GET /index.php/archives/298/ HTTP/2.0" 200 8626 "https://blog.qvbilam.xin/index.php/page/2/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36"
115.196.131.122 - - [18/Dec/2019:16:16:12 +0800] "POST /index.php/archives/325/ HTTP/2.0" 302 0 "https://blog.qvbilam.xin/index.php/archives/325/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
115.196.131.122 - - [18/Dec/2019:16:16:12 +0800] "GET /index.php/search/nginx/ HTTP/2.0" 200 3991 "https://blog.qvbilam.xin/index.php/archives/325/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
115.196.131.122 - - [18/Dec/2019:16:16:15 +0800] "GET /index.php/archives/321/ HTTP/2.0" 200 9291 "https://blog.qvbilam.xin/index.php/search/nginx/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
115.196.131.122 - - [18/Dec/2019:16:16:15 +0800] "GET /Users/qvbilam/Library/Application HTTP/2.0" 404 180 "https://blog.qvbilam.xin/index.php/archives/321/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"

ES创建动态模板索引

# 请求
PUT _template/blog_nginx_log
{
    "template":"blog_nginx_log*",
    "mappings":{
        "properties":{
            "request":{
                "type":"text",
                "analyzer":"ik_max_word",
                "search_analyzer":"ik_max_word"
            },
            "geoip":{
                "properties":{
                    "location":{
                        "type":"geo_point",
                        "ignore_malformed":"true"
                    }
                }
            }
        }
    }
}

Filbeat输出到Redis

# vim blog_nginx_log.yml
filebeat.inputs:
- type: log
  paths:
    - /Users/qvbilam/Sites/test/filebeat/log/blog_nginx.log

output.redis:
  hosts: ["127.0.0.1:6379"]
  key: "blog_nginx_log"
  db: 0                                         # 默认
  datatype: list  # 默认
# 启动Filbeat.
cd ~/Sites/test/filebeat
filebeat -e -c conf/blog_nginx_log.yml

查看Redis

Logstash处理Redis数据

# vim blog_nginx_log.conf
input {
  redis {
    data_type => "list"
    key => "blog_nginx_log"
    host => "127.0.0.1"
    port => 6379
    threads => 5
    db => 0
  }
}

filter{
  grok{
    match => {
      "message" => "%{COMBINEDAPACHELOG}"
    }
  }
  mutate{    # 删除字段
    remove_field => ["headers","timestamp","message","host","ecs","@version","httpversion"]
  }
  urldecode{  # 解码
    field => 'request'
  }
  geoip{   # ip解析
    source => 'clientip'
  }
  useragent{  # agent解析
    source => 'agent'
    target => "useragent"
  }
}

output{
  stdout{ codec => rubydebug }
    elasticsearch{
      hosts => ["127.0.0.1:8101"]
      index => "blog_nginx_log_%{+YYYY.MM.dd}"
    }
}
# 启动
cd ~/Sites/study/logstash
logstash -f blog_nginx_log.conf

Kibana

点击查看创建索引模式



点击查看创建可视化



点击查看结果展示



Last modification：February 18th, 2020 at 10:13 pm
© 允许规范转载