Wiki source code of Getting pinglog(s) in a seperate index
Version 1.1 by Normann P. Nielsen on 2026/11/15 15:41
Hide last authors
| author | version | line-number | content |
|---|---|---|---|
| |
1.1 | 1 | To create an index, an index mapping is needed - In general, I think for collecting filebased logs - the filebeat template suits me. |
| 2 | |||
| 3 | Make a copy of filebeat.json from the zip package at [[https://download.elastic.co/beats/dashboards/beats-dashboards-1.1.0.zip]] and change filebeat.json name and the content likewise. | ||
| 4 | |||
| 5 | Then upload and create index. | ||
| 6 | |||
| 7 | |||
| 8 | {{code}} | ||
| 9 | root@elkserver:curl -XPUT http://localhost:9200/.kibana/index-pattern/pinglog-* -d @pinglog.json | ||
| 10 | {"_index":".kibana","_type":"index-pattern","_id":"pinglog-*","_version":2,"_shards":{"total":2,"successful":1,"failed":0},"created":false} | ||
| 11 | root@elkserver: | ||
| 12 | {{/code}} | ||
| 13 | |||
| 14 | Then, copy filebeat-index-template.json to pinglog-index-template.json (and change the content likewise) | ||
| 15 | |||
| 16 | |||
| 17 | {{code}} | ||
| 18 | root@elkserver:~# curl -XPUT 'http://localhost:9200/_template/pinglog?pretty' -d@pinglog-index-template.json | ||
| 19 | { | ||
| 20 | "acknowledged" : true | ||
| 21 | } | ||
| 22 | root@elkserver:~# | ||
| 23 | {{/code}} | ||
| 24 | |||
| 25 | The collection on afserver still is like on [[ELK - ElasticSearch, LogStash, Kibana>>Main.information-technology.elk-elasticsearch-logstash-kibana]] | ||
| 26 | |||
| 27 | |||
| 28 | {{code language="/etc/filebeat/filebeat.yml"}} | ||
| 29 | paths: | ||
| 30 | - /var/log/pingkaf.txt | ||
| 31 | document_type: pinglog | ||
| 32 | input_type: log | ||
| 33 | {{/code}} | ||
| 34 | |||
| 35 | |||
| 36 | |||
| 37 | This is shipped to Logstash, where output is configured for ElasticSearch- notice the if for type "pinglog": | ||
| 38 | |||
| 39 | |||
| 40 | {{code language="/etc/logstash/conf.d/30-elasticsearch-output.conf"}} | ||
| 41 | output { | ||
| 42 | if [type] == "pinglog" { | ||
| 43 | elasticsearch { | ||
| 44 | hosts => ["localhost:9200"] | ||
| 45 | sniffing => true | ||
| 46 | manage_template => false | ||
| 47 | index => "pinglog-%{+YYYY.MM.dd}" | ||
| 48 | document_type => "%{[@metadata][type]}" | ||
| 49 | } | ||
| 50 | } | ||
| 51 | else | ||
| 52 | { | ||
| 53 | elasticsearch { | ||
| 54 | hosts => ["localhost:9200"] | ||
| 55 | sniffing => true | ||
| 56 | manage_template => false | ||
| 57 | index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" | ||
| 58 | document_type => "%{[@metadata][type]}" | ||
| 59 | } | ||
| 60 | } | ||
| 61 | } | ||
| 62 | {{/code}} | ||
| 63 | |||
| 64 | As I have ensured logs are pretty much key=value for the values I need to collect, I use [[kv>>https://www.elastic.co/guide/en/logstash/current/plugins-filters-kv.html]] to get the fields, and a [[mutate>>https://www.elastic.co/guide/en/logstash/current/plugins-filters-mutate.html]] to ensure the string value for "pingtime=" becomes a float (otherwise it cant be used in a [[Visualization>>https://www.elastic.co/guide/en/kibana/current/visualize.html]]): | ||
| 65 | |||
| 66 | |||
| 67 | {{code language="/etc/logstash/conf.d/02-beats-input.conf"}} | ||
| 68 | input { | ||
| 69 | beats { | ||
| 70 | port => 5044 | ||
| 71 | ssl => true | ||
| 72 | ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt" | ||
| 73 | ssl_key => "/etc/pki/tls/private/logstash-forwarder.key" | ||
| 74 | } | ||
| 75 | } | ||
| 76 | filter { | ||
| 77 | if [type] == "apache" { | ||
| 78 | grok { | ||
| 79 | match => { "message" => "%{COMBINEDAPACHELOG}" } | ||
| 80 | } | ||
| 81 | date { | ||
| 82 | match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ] | ||
| 83 | } | ||
| 84 | geoip { | ||
| 85 | source => "clientip" | ||
| 86 | target => "geoip" | ||
| 87 | database => "/etc/logstash/GeoLiteCity.dat" | ||
| 88 | add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] | ||
| 89 | add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] | ||
| 90 | } | ||
| 91 | mutate { | ||
| 92 | convert => [ "[geoip][coordinates]", "float"] | ||
| 93 | } | ||
| 94 | } | ||
| 95 | else | ||
| 96 | { | ||
| 97 | if [type] == "pinglog" { | ||
| 98 | kv {} | ||
| 99 | mutate { | ||
| 100 | convert => { "pingtime" => "float" } | ||
| 101 | } | ||
| 102 | } | ||
| 103 | } | ||
| 104 | } | ||
| 105 | {{/code}} | ||
| 106 | |||
| 107 | |||
| 108 | |||
| 109 | This should bring pinglogs in the index "pinglogs" |