|NO.Z.00015|——————————|Deployment|——|Hadoop&ElasticSearch集中式日志分析系统.v15|——|Elasticsearch.v15|日志分析实战.V

一、Logstash读取Kafka

### --- Logstash配置文件

~~~     官网地址：https://www.elastic.co/guide/en/logstash/7.3/plugins-inputs-kafka.html

### --- 上传IP地址库

~~~     # 上传IP地址库到Datas目录下
[root@hadoop02 ~]# ll /opt/yanqi/servers/es/datas/GeoLite2-City.mmdb 
/opt/yanqi/servers/es/datas/GeoLite2-City.mmdb

二、Logstash读取kafka配置

### --- Logstash配置文件编写

~~~     # 编写logstash读取kafka数据的配置文件
[root@hadoop02 ~]# vim /opt/yanqi/servers/es/Logstash/config/logstash_kafka_es.conf
~~~     # 写入如下参数
input {
    kafka {
        bootstrap_servers => "hadoop01:9092,hadoop02:9092,hadoop03:9092"
        topics => ["nginx_access_log"]
        codec => "json"
    }
}

filter {
  if [app] == "www" {
    if [type] == "nginx-access" {
      json {
          source => "message"
          remove_field => ["message"]
      }
      geoip {
          source => "remote_addr"
          target => "geoip"
          database => "/opt/yanqi/servers/es/datas/GeoLite2-City.mmdb"
          add_field => ["[geoip][coordinates]", "%{[geoip][longitude]}"]
          add_field => ["[geoip][coordinates]", "%{[geoip][latitude]}"]
      }
      mutate {
          convert => ["[geoip][coordinates]", "float"]
      }
    }
  }
}

output {
  elasticsearch {
      hosts => ["http://hadoop01:9200","http://hadoop02:9200","http://hadoop03:9200"]
      index => "logstash-%{type}-%{+YYYY.MM.dd}"
  }
  stdout{codec => rubydebug }
}

### --- 验证配置文件完整性

~~~     # 验证配置文件完整性
[root@hadoop02 ~]# /opt/yanqi/servers/es/Logstash/bin/logstash \
-f /opt/yanqi/servers/es/Logstash/config/logstash_kafka_es.conf -t
~~~     # 输出参数
Configuration OK
Config Validation Result: OK. Exiting Logstash

### --- 启动logstash任务

~~~     # 启动logstash服务并读取kafka数据
[root@hadoop02 ~]# /opt/yanqi/servers/es/Logstash/bin/logstash \
-f /opt/yanqi/servers/es/Logstash/config/logstash_kafka_es.conf
~~~     # 输出参数
~~~     # 详见附录二：

二、在es下查看是否有对应的索引创建出来
三、查看索引下是否有数据

附录一：修改Nginx.conf

### --- 修改nginx输出日志IP地址：
### --- 若是在vmware虚拟环境下的地址访问Nginx，它是不会解析出来地理坐标的，修改nginx配置文件，输出日志的IP地址为公网的某一个地址

[root@hadoop02 ~]# vim /usr/local/nginx/conf/nginx.conf
~~~     # 第26行：修改输出日志的IP地址为如下IP地址
          '"remote_addr": "$remote_addr", '     # 默认配置参数
          '"remote_addr": "172.26.12.37", '     # 修改后的配置参数，地址为公网地址即可

### --- 重新加载reload.nginx服务

[root@hadoop02 ~]# /usr/local/nginx/sbin/nginx -s reload

附录二：启动logstash服务并读取kafka数据：输出参数

{
                    "host" => {
        "name" => "hadoop02"
    },
          "request_method" => "GET",
                   "input" => {
        "type" => "log"
    },
                     "app" => "www",
                    "type" => "nginx-access",
             "remote_user" => "-",
                  "status" => "200",
             "request_uri" => "/",
           "http_referrer" => "-",
                "@version" => "1",
                   "geoip" => {
                    "ip" => "170.233.46.141",
          "country_name" => "Brazil",
        "continent_code" => "SA",
         "country_code2" => "BR",
              "location" => {
            "lon" => -43.2192,
            "lat" => -22.8305
        },
              "latitude" => -22.8305,
           "coordinates" => [
            [0] -43.2192,
            [1] -22.8305
        ],
         "country_code3" => "BR",
             "longitude" => -43.2192
    },
            "request_time" => "0.000",
              "@timestamp" => 2021-11-27T08:50:40.000Z,
         "body_bytes_sent" => "612",
    "http_x_forwarded_for" => "-",
                     "ecs" => {
        "version" => "1.0.1"
    },
         "http_user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
             "remote_addr" => "170.233.46.141",
                   "agent" => {
             "version" => "7.3.0",
                  "id" => "dfbd0913-f1fa-4b28-a120-2e9575d387fa",
                "type" => "filebeat",
            "hostname" => "hadoop02",
        "ephemeral_id" => "7e92416c-308d-4fc5-b1e7-947663f60095"
    },
                     "log" => {
        "offset" => 32935,
          "file" => {
            "path" => "/usr/local/nginx/logs/access.log"
        }
    }
}

Walter Savage Landor:strove with none,for none was worth my strife.Nature I loved and, next to Nature, Art:I warm'd both hands before the fire of life.It sinks, and I am ready to depart ——W.S.Landor

dov006-elk

|NO.Z.00015|——————————|Deployment|——|Hadoop&ElasticSearch集中式日志分析系统.v15|——|Elasticsearch.v15|日志分析实战.V

相关

标签