diff --git a/geomap.json b/geomap.json index 849f041..bd4f0e0 100644 --- a/geomap.json +++ b/geomap.json @@ -2,58 +2,71 @@ "__inputs": [ { "name": "DS_INFLUXDB", - "label": "Influxdb", + "label": "InfluxDB", "description": "", "type": "datasource", "pluginId": "influxdb", "pluginName": "InfluxDB" } ], + "__elements": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "5.2.3" + "version": "9.0.2" }, { "type": "panel", "id": "grafana-worldmap-panel", "name": "Worldmap Panel", - "version": "0.1.2" + "version": "0.3.3" }, { "type": "datasource", "id": "influxdb", "name": "InfluxDB", - "version": "5.0.0" + "version": "1.0.0" }, { "type": "panel", - "id": "table", - "name": "Table", - "version": "5.0.0" + "id": "table-old", + "name": "Table (old)", + "version": "" } ], "annotations": { "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] }, + "description": "Dashboard for showing GEOIP information from Nginx logs, for the GeoStat script. ", "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, + "gnetId": 8342, "graphTooltip": 0, "id": null, - "iteration": 1539373533704, + "iteration": 1658128917676, "links": [], + "liveNow": false, "panels": [ { "circleMaxSize": "8", @@ -63,7 +76,10 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "${DS_INFLUXDB}", + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB}" + }, "decimals": 0, "esMetric": "Count", "gridPos": { @@ -95,6 +111,10 @@ }, "targets": [ { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB}" + }, "groupBy": [ { "params": [ @@ -133,6 +153,12 @@ "key": "host", "operator": "=~", "value": "/^$host$/" + }, + { + "condition": "AND", + "key": "website", + "operator": "=~", + "value": "/^$website$/" } ] } @@ -147,7 +173,10 @@ }, { "columns": [], - "datasource": "${DS_INFLUXDB}", + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB}" + }, "fontSize": "100%", "gridPos": { "h": 13, @@ -167,7 +196,7 @@ "styles": [ { "alias": "Country", - "colorMode": null, + "align": "auto", "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", @@ -183,7 +212,7 @@ }, { "alias": "", - "colorMode": null, + "align": "auto", "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", @@ -200,7 +229,7 @@ }, { "alias": "Count", - "colorMode": null, + "align": "auto", "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", @@ -217,6 +246,10 @@ ], "targets": [ { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB}" + }, "groupBy": [ { "params": [ @@ -255,59 +288,90 @@ "key": "host", "operator": "=~", "value": "/^$host$/" + }, + { + "condition": "AND", + "key": "website", + "operator": "=~", + "value": "/^$website$/" } ] } ], "title": "Country SUM", "transform": "table", - "type": "table" + "type": "table-old" } ], - "refresh": "30s", - "schemaVersion": 16, + "refresh": "5s", + "schemaVersion": 36, "style": "dark", "tags": [], "templating": { "list": [ { "current": { + "selected": false, "text": "default", "value": "default" }, "hide": 0, - "label": null, + "includeAll": false, + "multi": false, "name": "datasource", "options": [], "query": "influxdb", + "queryValue": "", "refresh": 1, "regex": "", + "skipUrlSync": false, "type": "datasource" }, { - "allValue": null, "current": {}, - "datasource": "${DS_INFLUXDB}", + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB}" + }, + "definition": "", "hide": 0, "includeAll": false, - "label": null, "multi": false, "name": "host", "options": [], "query": "SHOW TAG VALUES FROM \"geodata\" WITH KEY = \"host\"", "refresh": 1, "regex": "", + "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", - "tags": [], "tagsQuery": "", "type": "query", "useTags": false + }, + { + "current": {}, + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB}" + }, + "definition": "SHOW TAG VALUES FROM \"geodata\" WITH KEY = \"website\"", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "website", + "options": [], + "query": "SHOW TAG VALUES FROM \"geodata\" WITH KEY = \"website\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" } ] }, "time": { - "from": "now-6h", + "from": "now-5m", "to": "now" }, "timepicker": { @@ -336,7 +400,8 @@ ] }, "timezone": "", - "title": "GeoMap", + "title": "Nginx GEOIP Statistic", "uid": "IYo4xyJmz", - "version": 6 -} + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/geoparser.py b/geoparser.py index f25cfa1..5aeeacb 100755 --- a/geoparser.py +++ b/geoparser.py @@ -3,6 +3,8 @@ # Parts added by Remko Lodder, 2019. # Added: IPv6 matching, make query based on geoip2 instead of # geoip, which is going away r.s.n. +# Added possibility of processing more than one Nginx log file, +# by adding threading support. 2022 July by Alexey Nizhegolenko import os import re @@ -15,6 +17,7 @@ import geoip2.database import configparser from influxdb import InfluxDBClient from IPy import IP as ipadd +import threading class SyslogBOMFormatter(logging.Formatter): @@ -30,7 +33,7 @@ root = logging.getLogger(__name__) root.setLevel(os.environ.get("LOGLEVEL", "INFO")) root.addHandler(handler) -def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE): # NOQA +def logparse(LOGPATH, WEBSITE, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE): # NOQA # Preparing variables and params IPS = {} COUNT = {} @@ -43,8 +46,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER re_IPV6 = re.compile('(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))') # NOQA GI = geoip2.database.Reader(GEOIPDB) - - # Main loop to parse access.log file in tailf style with sending metrcs + # Main loop that parses log file in tailf style with sending metrics out with open(LOGPATH, "r") as FILE: STR_RESULTS = os.stat(LOGPATH) ST_SIZE = STR_RESULTS[6] @@ -55,7 +57,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER LINE = FILE.readline() INODENEW = os.stat(LOGPATH).st_ino if INODE != INODENEW: - break + return if not LINE: time.sleep(1) FILE.seek(WHERE) @@ -74,6 +76,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER COUNT['count'] = 1 GEOHASH['geohash'] = HASH GEOHASH['host'] = HOSTNAME + GEOHASH['website'] = WEBSITE GEOHASH['country_code'] = INFO.country.iso_code GEOHASH['country_name'] = INFO.country.name GEOHASH['city_name'] = INFO.city.name @@ -81,8 +84,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER IPS['fields'] = COUNT IPS['measurement'] = MEASUREMENT METRICS.append(IPS) - - # Sending json data to InfluxDB + # Sending json data itto InfluxDB try: CLIENT.write_points(METRICS) except Exception: @@ -90,14 +92,14 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER def main(): - # Preparing for reading config file + # Preparing for reading the config file PWD = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) CONFIG = configparser.ConfigParser() - CONFIG.read('%s/settings.ini' % PWD) + CONFIG.read(f'{PWD}/settings.ini') # Getting params from config GEOIPDB = CONFIG.get('GEOIP', 'geoipdb') - LOGPATH = CONFIG.get('NGINX_LOG', 'logpath') + LOGPATH = CONFIG.get('NGINX_LOGS', 'logpath').split() INFLUXHOST = CONFIG.get('INFLUXDB', 'host') INFLUXPORT = CONFIG.get('INFLUXDB', 'port') INFLUXDBDB = CONFIG.get('INFLUXDB', 'database') @@ -107,14 +109,29 @@ def main(): # Parsing log file and sending metrics to Influxdb while True: - # Get inode from log file - INODE = os.stat(LOGPATH).st_ino - # Run main loop and grep a log file - if os.path.exists(LOGPATH): - logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE) # NOQA - else: - logging.info('Nginx log file %s not found', LOGPATH) - print('Nginx log file %s not found' % LOGPATH) + logs = [] + thread_names = [] + for logitem in LOGPATH: + logs.append(logitem.split(":")) + for website, log in logs: + # Get inode from log file + if os.path.exists(log): + INODE = os.stat(log).st_ino + else: + logging.info('Nginx log file %s not found', log) + print('Nginx log file %s not found' % log) + return + # Run the main loop and grep data in separate threads + t = website + if os.path.exists(log): + t = threading.Thread(target=logparse, args=[log, website, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE], daemon=True, name=website) # NOQA + for thread in threading.enumerate(): + thread_names.append(thread.name) + if website not in thread_names: + t.start() + else: + logging.info('Nginx log file %s not found', log) + print('Nginx log file %s not found' % log) if __name__ == '__main__': diff --git a/geostat.png b/geostat.png index e504855..70f2c49 100644 Binary files a/geostat.png and b/geostat.png differ diff --git a/settings.ini.back b/settings.ini.back index dbdc2ee..f827e10 100644 --- a/settings.ini.back +++ b/settings.ini.back @@ -1,6 +1,6 @@ -[NGINX_LOG] +[NGINX_LOGS] #Path for the log file (Nginx) -logpath = /var/log/nginx/access.log +logpath = website1:/var/log/website1/access.log website2:/var/log/website2/access.log [GEOIP] #Path for the GEOIP DB file