ELK日志实战

一、Logstash

filter{
mutate{
remove_field => ”@version”		//剔除@version字段
remove_field => ”@timestamp”	//剔除@timestamp字段
convert => {“ID” => “integer”}    //将ID字段类型转为integer类型
}
date{//将CAP_DATE字段由字符串类型转为date类型，使es能按时间进行查询
match => [“CAP_DATE”,”YYYY-MM-dd HH:mm:ss”]
target => “CAP_DATE”
}
}
output{
//指定es的ip（此处与logstash在同一机器）、index、document_type、documemt_id对应字段
elasticsearch{
hosts => [“127.0.0.1”]
index => “jy”
document_type => “C_PICRECORD”
document_id => “%{ID}”
template_overwrite => true
}
}

input{
	file {
		path => ["/usr/local/test-mongo.csv"]	
		start_position => "beginning"
	}
}
filter {
	mutate {
		split => ["message" ,"','"]
	}
	mutate {
		add_field => ["TEMP_ID","%{message[0]}"]
		add_field => ["DEV_ID","%{message[1]}"]
		add_field => ["DEV_CHN_NUM","%{message[2]}"]
		add_field => ["DEV_NAME","%{message[3]}"]
		add_field => ["DEV_CHN_NAME","%{message[4]}"]
		add_field => ["CAR_NUM","%{message[5]}"]
		add_field => ["CAR_NUM_TYPE","%{message[6]}"]
		add_field => ["CAR_NUM_COLOR","%{message[7]}"]
		add_field => ["CAR_SPEED","%{message[8]}"]
		add_field => ["CAR_TYPE","%{message[9]}"]
		add_field => ["CAR_COLOR","%{message[10]}"]
		add_field => ["CAR_DIRECT","%{message[11]}"]
		add_field => ["CAR_LENGTH","%{message[12]}"]
		add_field => ["WAY_NUM","%{message[13]}"]
		add_field => ["CAP_TIME_STAMP","%{message[14]}"]
		add_field => ["CAP_DATE","%{message[15]}"]
		add_field => ["CAR_IMG_URL","%{message[16]}"]
		add_field => ["CAR_IMG1_URL","%{message[17]}"]
		add_field => ["CAR_IMG2_URL","%{message[18]}"]
		add_field => ["CAR_IMG3_URL","%{message[19]}"]
		add_field => ["CAR_IMG4_URL","%{message[20]}"]
		add_field => ["CAR_IMG5_URL","%{message[21]}"]
		add_field => ["CAR_NUM_X","%{message[22]}"]
		add_field => ["CAR_NUM_Y","%{message[23]}"]
		add_field => ["CAR_NUM_W","%{message[24]}"]
		add_field => ["CAR_NUM_H","%{message[25]}"]
		add_field => ["CAR_LOGO_TYPE","%{message[26]}"]
		add_field => ["DRIVER_SEATBELT","%{message[27]}"]
		add_field => ["DRIVER_PHONEING","%{message[28]}"]
		add_field => ["CAR_NUM_CORRELATIVE","%{message[29]}"]
		add_field => ["CAP_TYPE","%{message[30]}"]
		add_field => ["CAR_NUM_CONFIDENCE","%{message[31]}"]
		add_field => ["CAR_LOGO_CONFIDENCE","%{message[32]}"]
		add_field => ["DRIVER_SMOKING","%{message[33]}"]
		add_field => ["DRIVER_TIRED","%{message[34]}"]
		add_field => ["DRIVER_VISOR","%{message[35]}"]
		add_field => ["DEV_CODE","%{message[36]}"]
		add_field => ["DEV_CHN_ID","%{message[37]}"]
		add_field => ["ORG_ID BIGINT","%{message[38]}"]
        add_field => ["ORG_CODE","%{message[39]}"]
		add_field => ["ORG_NAME","%{message[40]}"]
		add_field => ["LONGITUDE","%{message[41]}"]
        add_field => ["LATITUDE","%{message[42]}"]
		add_field => ["GROUPID","%{message[43]}"]
		add_field => ["PASSENGER_SEATBELT","%{message[44]}"]
		add_field => ["SEATBELT_AREA","%{message[45]}"]
		add_field => ["PHONE_AREA","%{message[46]}"]
		add_field => ["VEHICLE_MARK_AREA","%{message[47]}"]
		add_field => ["CAR_LOGO_CHILDTYPE","%{message[48]}"]
		add_field => ["CAR_LOGO_YEAR","%{message[49]}"]
		add_field => ["IS_ANALYZE","%{message[50]}"]
		add_field => ["PAPER_MARK","%{message[51]}"]
		add_field => ["DROP_MARK","%{message[52]}"]
		add_field => ["SUN_MARK","%{message[53]}"]
		add_field => ["ANNUALTAG_MARK","%{message[54]}"]
		add_field => ["TEMP_OBJ_BIN_FEATURES","%{message[55]}"]
	}
	mutate {
		split=>["TEMP_ID","'"]
		add_field => ["ID","%{[TEMP_ID][1]}"]
	}
	mutate {
		remove_field => ["message","TEMP_ID","host","path","@timestamp","@version"]
	}
}
output {
        mongodb {
                collection => "csvcollection"
                database => "test"
                uri => "mongodb://test:test@172.16.21.170:27017/test"
				bulk => true
				bulk_interval => 2
				bulk_size => 900
				isodate => true
				retry_delay => 3
				generateId => false
		}
}

二、ES pipeline

PUT _ingest/pipeline/log-pipeline
{
  "description": "solr日志处理管道",
  "processors": [{
        "grok": {
          "field": "message",
          "patterns": [
            "%{MYTIME:@timestamp}\\s*%{LOGLEVEL:level}%{MYMESSAGE:message}"
            ],
          "pattern_definitions" : {
            "MYTIME": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}",
            "MYMESSAGE":"([\\s\\S]*)"
          },
          "on_failure" : [
            {
              "set" : {
                "field" : "level",
                "value" : "UNKNOWN"
              }}]},
"set":{
          "field": "hostname",
          "value": "{{host.name}}",
          "on_failure" : [{
              "set" : {
                "field" : "hostname",
                "value" : "UNKNOWN"}}
          ]}},{
        "date" : {
          "field" : "@timestamp",
          "target_field" : "@timestamp",
          "formats" : ["yyyy-MM-dd HH:mm:ss.SSS","yyyy-MM-dd HH:mm:ss,SSS"],
          "ignore_failure" : true
        }},{
        "remove": {
          "field": ["input", "beat","fields","prospector","host","offset","log"],
          "ignore_missing": true
        }}]}

三、索引模板

PUT _template/template_log
{
  "index_patterns": ["*-log*"],
  "settings":{
    "number_of_shards":1,
    "number_of_replicas": 1,
    "index.lifecycle.name": "log-policy"
  },
  "mappings" : {
    "doc" : {
      "properties" : {
        "@timestamp" : {
          "type" : "date"
        },
        "hostname" : {
          "type" : "keyword",
          "ignore_above" : 256
        },
       "level" : {
          "type" : "keyword",
          "ignore_above" : 256
        },
        "message" : {
          "type" : "text"
        },
        "source" : {
          "type" : "keyword",
          "ignore_above" : 256
        }
      }
    }
  }
}

四、索引生命周期

PUT /_ilm/policy/log-policy
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
        }
      },
      "warm": {
        "min_age": "2d",
        "actions": {
          "forcemerge": {
            "max_num_segments": 1
          }
        }
      },
      "cold": {
        "min_age": "7d",
        "actions": {
          "allocate": {
            "number_of_replicas": 0
          }
        }
      },
      "delete": {
        "min_age": "30d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}