Merge pull request #20 from ratibor78/multilog
Added the multi website log parsing availability
This commit is contained in:
Alexey Nizhegolenko
GitHub
commit
215a9bd14c
4 changed files with 128 additions and 46 deletions
117
geomap.json
117
geomap.json
|
|
@ -2,58 +2,71 @@
|
|||
"__inputs": [
|
||||
{
|
||||
"name": "DS_INFLUXDB",
|
||||
"label": "Influxdb",
|
||||
"label": "InfluxDB",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "influxdb",
|
||||
"pluginName": "InfluxDB"
|
||||
}
|
||||
],
|
||||
"__elements": [],
|
||||
"__requires": [
|
||||
{
|
||||
"type": "grafana",
|
||||
"id": "grafana",
|
||||
"name": "Grafana",
|
||||
"version": "5.2.3"
|
||||
"version": "9.0.2"
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "grafana-worldmap-panel",
|
||||
"name": "Worldmap Panel",
|
||||
"version": "0.1.2"
|
||||
"version": "0.3.3"
|
||||
},
|
||||
{
|
||||
"type": "datasource",
|
||||
"id": "influxdb",
|
||||
"name": "InfluxDB",
|
||||
"version": "5.0.0"
|
||||
"version": "1.0.0"
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "table",
|
||||
"name": "Table",
|
||||
"version": "5.0.0"
|
||||
"id": "table-old",
|
||||
"name": "Table (old)",
|
||||
"version": ""
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"datasource": {
|
||||
"type": "datasource",
|
||||
"uid": "grafana"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Dashboard for showing GEOIP information from Nginx logs, for the GeoStat script. ",
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"gnetId": 8342,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"iteration": 1539373533704,
|
||||
"iteration": 1658128917676,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"circleMaxSize": "8",
|
||||
|
|
@ -63,7 +76,10 @@
|
|||
"rgba(237, 129, 40, 0.89)",
|
||||
"rgba(50, 172, 45, 0.97)"
|
||||
],
|
||||
"datasource": "${DS_INFLUXDB}",
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "${DS_INFLUXDB}"
|
||||
},
|
||||
"decimals": 0,
|
||||
"esMetric": "Count",
|
||||
"gridPos": {
|
||||
|
|
@ -95,6 +111,10 @@
|
|||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "${DS_INFLUXDB}"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
"params": [
|
||||
|
|
@ -133,6 +153,12 @@
|
|||
"key": "host",
|
||||
"operator": "=~",
|
||||
"value": "/^$host$/"
|
||||
},
|
||||
{
|
||||
"condition": "AND",
|
||||
"key": "website",
|
||||
"operator": "=~",
|
||||
"value": "/^$website$/"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -147,7 +173,10 @@
|
|||
},
|
||||
{
|
||||
"columns": [],
|
||||
"datasource": "${DS_INFLUXDB}",
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "${DS_INFLUXDB}"
|
||||
},
|
||||
"fontSize": "100%",
|
||||
"gridPos": {
|
||||
"h": 13,
|
||||
|
|
@ -167,7 +196,7 @@
|
|||
"styles": [
|
||||
{
|
||||
"alias": "Country",
|
||||
"colorMode": null,
|
||||
"align": "auto",
|
||||
"colors": [
|
||||
"rgba(245, 54, 54, 0.9)",
|
||||
"rgba(237, 129, 40, 0.89)",
|
||||
|
|
@ -183,7 +212,7 @@
|
|||
},
|
||||
{
|
||||
"alias": "",
|
||||
"colorMode": null,
|
||||
"align": "auto",
|
||||
"colors": [
|
||||
"rgba(245, 54, 54, 0.9)",
|
||||
"rgba(237, 129, 40, 0.89)",
|
||||
|
|
@ -200,7 +229,7 @@
|
|||
},
|
||||
{
|
||||
"alias": "Count",
|
||||
"colorMode": null,
|
||||
"align": "auto",
|
||||
"colors": [
|
||||
"rgba(245, 54, 54, 0.9)",
|
||||
"rgba(237, 129, 40, 0.89)",
|
||||
|
|
@ -217,6 +246,10 @@
|
|||
],
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "${DS_INFLUXDB}"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
"params": [
|
||||
|
|
@ -255,59 +288,90 @@
|
|||
"key": "host",
|
||||
"operator": "=~",
|
||||
"value": "/^$host$/"
|
||||
},
|
||||
{
|
||||
"condition": "AND",
|
||||
"key": "website",
|
||||
"operator": "=~",
|
||||
"value": "/^$website$/"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"title": "Country SUM",
|
||||
"transform": "table",
|
||||
"type": "table"
|
||||
"type": "table-old"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 16,
|
||||
"refresh": "5s",
|
||||
"schemaVersion": 36,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "default",
|
||||
"value": "default"
|
||||
},
|
||||
"hide": 0,
|
||||
"label": null,
|
||||
"includeAll": false,
|
||||
"multi": false,
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "influxdb",
|
||||
"queryValue": "",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"allValue": null,
|
||||
"current": {},
|
||||
"datasource": "${DS_INFLUXDB}",
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "${DS_INFLUXDB}"
|
||||
},
|
||||
"definition": "",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": null,
|
||||
"multi": false,
|
||||
"name": "host",
|
||||
"options": [],
|
||||
"query": "SHOW TAG VALUES FROM \"geodata\" WITH KEY = \"host\"",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
},
|
||||
{
|
||||
"current": {},
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "${DS_INFLUXDB}"
|
||||
},
|
||||
"definition": "SHOW TAG VALUES FROM \"geodata\" WITH KEY = \"website\"",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"multi": false,
|
||||
"name": "website",
|
||||
"options": [],
|
||||
"query": "SHOW TAG VALUES FROM \"geodata\" WITH KEY = \"website\"",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"from": "now-5m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
|
|
@ -336,7 +400,8 @@
|
|||
]
|
||||
},
|
||||
"timezone": "",
|
||||
"title": "GeoMap",
|
||||
"title": "Nginx GEOIP Statistic",
|
||||
"uid": "IYo4xyJmz",
|
||||
"version": 6
|
||||
"version": 5,
|
||||
"weekStart": ""
|
||||
}
|
||||
51
geoparser.py
51
geoparser.py
|
|
@ -3,6 +3,8 @@
|
|||
# Parts added by Remko Lodder, 2019.
|
||||
# Added: IPv6 matching, make query based on geoip2 instead of
|
||||
# geoip, which is going away r.s.n.
|
||||
# Added possibility of processing more than one Nginx log file,
|
||||
# by adding threading support. 2022 July by Alexey Nizhegolenko
|
||||
|
||||
import os
|
||||
import re
|
||||
|
|
@ -15,6 +17,7 @@ import geoip2.database
|
|||
import configparser
|
||||
from influxdb import InfluxDBClient
|
||||
from IPy import IP as ipadd
|
||||
import threading
|
||||
|
||||
|
||||
class SyslogBOMFormatter(logging.Formatter):
|
||||
|
|
@ -30,7 +33,7 @@ root = logging.getLogger(__name__)
|
|||
root.setLevel(os.environ.get("LOGLEVEL", "INFO"))
|
||||
root.addHandler(handler)
|
||||
|
||||
def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE): # NOQA
|
||||
def logparse(LOGPATH, WEBSITE, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE): # NOQA
|
||||
# Preparing variables and params
|
||||
IPS = {}
|
||||
COUNT = {}
|
||||
|
|
@ -43,8 +46,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
|
|||
re_IPV6 = re.compile('(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))') # NOQA
|
||||
|
||||
GI = geoip2.database.Reader(GEOIPDB)
|
||||
|
||||
# Main loop to parse access.log file in tailf style with sending metrcs
|
||||
# Main loop that parses log file in tailf style with sending metrics out
|
||||
with open(LOGPATH, "r") as FILE:
|
||||
STR_RESULTS = os.stat(LOGPATH)
|
||||
ST_SIZE = STR_RESULTS[6]
|
||||
|
|
@ -55,7 +57,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
|
|||
LINE = FILE.readline()
|
||||
INODENEW = os.stat(LOGPATH).st_ino
|
||||
if INODE != INODENEW:
|
||||
break
|
||||
return
|
||||
if not LINE:
|
||||
time.sleep(1)
|
||||
FILE.seek(WHERE)
|
||||
|
|
@ -74,6 +76,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
|
|||
COUNT['count'] = 1
|
||||
GEOHASH['geohash'] = HASH
|
||||
GEOHASH['host'] = HOSTNAME
|
||||
GEOHASH['website'] = WEBSITE
|
||||
GEOHASH['country_code'] = INFO.country.iso_code
|
||||
GEOHASH['country_name'] = INFO.country.name
|
||||
GEOHASH['city_name'] = INFO.city.name
|
||||
|
|
@ -81,8 +84,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
|
|||
IPS['fields'] = COUNT
|
||||
IPS['measurement'] = MEASUREMENT
|
||||
METRICS.append(IPS)
|
||||
|
||||
# Sending json data to InfluxDB
|
||||
# Sending json data itto InfluxDB
|
||||
try:
|
||||
CLIENT.write_points(METRICS)
|
||||
except Exception:
|
||||
|
|
@ -90,14 +92,14 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
|
|||
|
||||
|
||||
def main():
|
||||
# Preparing for reading config file
|
||||
# Preparing for reading the config file
|
||||
PWD = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
|
||||
CONFIG = configparser.ConfigParser()
|
||||
CONFIG.read('%s/settings.ini' % PWD)
|
||||
CONFIG.read(f'{PWD}/settings.ini')
|
||||
|
||||
# Getting params from config
|
||||
GEOIPDB = CONFIG.get('GEOIP', 'geoipdb')
|
||||
LOGPATH = CONFIG.get('NGINX_LOG', 'logpath')
|
||||
LOGPATH = CONFIG.get('NGINX_LOGS', 'logpath').split()
|
||||
INFLUXHOST = CONFIG.get('INFLUXDB', 'host')
|
||||
INFLUXPORT = CONFIG.get('INFLUXDB', 'port')
|
||||
INFLUXDBDB = CONFIG.get('INFLUXDB', 'database')
|
||||
|
|
@ -107,14 +109,29 @@ def main():
|
|||
|
||||
# Parsing log file and sending metrics to Influxdb
|
||||
while True:
|
||||
# Get inode from log file
|
||||
INODE = os.stat(LOGPATH).st_ino
|
||||
# Run main loop and grep a log file
|
||||
if os.path.exists(LOGPATH):
|
||||
logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE) # NOQA
|
||||
else:
|
||||
logging.info('Nginx log file %s not found', LOGPATH)
|
||||
print('Nginx log file %s not found' % LOGPATH)
|
||||
logs = []
|
||||
thread_names = []
|
||||
for logitem in LOGPATH:
|
||||
logs.append(logitem.split(":"))
|
||||
for website, log in logs:
|
||||
# Get inode from log file
|
||||
if os.path.exists(log):
|
||||
INODE = os.stat(log).st_ino
|
||||
else:
|
||||
logging.info('Nginx log file %s not found', log)
|
||||
print('Nginx log file %s not found' % log)
|
||||
return
|
||||
# Run the main loop and grep data in separate threads
|
||||
t = website
|
||||
if os.path.exists(log):
|
||||
t = threading.Thread(target=logparse, args=[log, website, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE], daemon=True, name=website) # NOQA
|
||||
for thread in threading.enumerate():
|
||||
thread_names.append(thread.name)
|
||||
if website not in thread_names:
|
||||
t.start()
|
||||
else:
|
||||
logging.info('Nginx log file %s not found', log)
|
||||
print('Nginx log file %s not found' % log)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
BIN
geostat.png
BIN
geostat.png
Binary file not shown.
|
Before Width: | Height: | Size: 188 KiB After Width: | Height: | Size: 137 KiB |
|
|
@ -1,6 +1,6 @@
|
|||
[NGINX_LOG]
|
||||
[NGINX_LOGS]
|
||||
#Path for the log file (Nginx)
|
||||
logpath = /var/log/nginx/access.log
|
||||
logpath = website1:/var/log/website1/access.log website2:/var/log/website2/access.log
|
||||
|
||||
[GEOIP]
|
||||
#Path for the GEOIP DB file
|
||||
|
|
|
|||
Loading…
Reference in a new issue