Merge pull request #20 from ratibor78/multilog

Added the multi website log parsing availability

geomap.json

@@ -2,58 +2,71 @@
   "__inputs": [
     {
       "name": "DS_INFLUXDB",
-      "label": "Influxdb",
+      "label": "InfluxDB",
       "description": "",
       "type": "datasource",
       "pluginId": "influxdb",
       "pluginName": "InfluxDB"
     }
   ],
+  "__elements": [],
   "__requires": [
     {
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "5.2.3"
+      "version": "9.0.2"
     },
     {
       "type": "panel",
       "id": "grafana-worldmap-panel",
       "name": "Worldmap Panel",
-      "version": "0.1.2"
+      "version": "0.3.3"
     },
     {
       "type": "datasource",
       "id": "influxdb",
       "name": "InfluxDB",
-      "version": "5.0.0"
+      "version": "1.0.0"
     },
     {
       "type": "panel",
-      "id": "table",
+      "id": "table-old",
-      "name": "Table",
+      "name": "Table (old)",
-      "version": "5.0.0"
+      "version": ""
     }
   ],
   "annotations": {
     "list": [
       {
         "builtIn": 1,
-        "datasource": "-- Grafana --",
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
         "enable": true,
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
         "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
         "type": "dashboard"
       }
     ]
   },
+  "description": "Dashboard for showing GEOIP information from Nginx logs, for the GeoStat script. ",
   "editable": true,
-  "gnetId": null,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 8342,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1539373533704,
+  "iteration": 1658128917676,
   "links": [],
+  "liveNow": false,
   "panels": [
     {
       "circleMaxSize": "8",
@@ -63,7 +76,10 @@
         "rgba(237, 129, 40, 0.89)",
         "rgba(50, 172, 45, 0.97)"
       ],
-      "datasource": "${DS_INFLUXDB}",
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
       "decimals": 0,
       "esMetric": "Count",
       "gridPos": {
@@ -95,6 +111,10 @@
       },
       "targets": [
         {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
           "groupBy": [
             {
               "params": [
@@ -133,6 +153,12 @@
               "key": "host",
               "operator": "=~",
               "value": "/^$host$/"
+            },
+            {
+              "condition": "AND",
+              "key": "website",
+              "operator": "=~",
+              "value": "/^$website$/"
             }
           ]
         }
@@ -147,7 +173,10 @@
     },
     {
       "columns": [],
-      "datasource": "${DS_INFLUXDB}",
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
       "fontSize": "100%",
       "gridPos": {
         "h": 13,
@@ -167,7 +196,7 @@
       "styles": [
         {
           "alias": "Country",
-          "colorMode": null,
+          "align": "auto",
           "colors": [
             "rgba(245, 54, 54, 0.9)",
             "rgba(237, 129, 40, 0.89)",
@@ -183,7 +212,7 @@
         },
         {
           "alias": "",
-          "colorMode": null,
+          "align": "auto",
           "colors": [
             "rgba(245, 54, 54, 0.9)",
             "rgba(237, 129, 40, 0.89)",
@@ -200,7 +229,7 @@
         },
         {
           "alias": "Count",
-          "colorMode": null,
+          "align": "auto",
           "colors": [
             "rgba(245, 54, 54, 0.9)",
             "rgba(237, 129, 40, 0.89)",
@@ -217,6 +246,10 @@
       ],
       "targets": [
         {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
           "groupBy": [
             {
               "params": [
@@ -255,59 +288,90 @@
               "key": "host",
               "operator": "=~",
               "value": "/^$host$/"
+            },
+            {
+              "condition": "AND",
+              "key": "website",
+              "operator": "=~",
+              "value": "/^$website$/"
             }
           ]
         }
       ],
       "title": "Country SUM",
       "transform": "table",
-      "type": "table"
+      "type": "table-old"
     }
   ],
-  "refresh": "30s",
+  "refresh": "5s",
-  "schemaVersion": 16,
+  "schemaVersion": 36,
   "style": "dark",
   "tags": [],
   "templating": {
     "list": [
       {
         "current": {
+          "selected": false,
           "text": "default",
           "value": "default"
         },
         "hide": 0,
-        "label": null,
+        "includeAll": false,
+        "multi": false,
         "name": "datasource",
         "options": [],
         "query": "influxdb",
+        "queryValue": "",
         "refresh": 1,
         "regex": "",
+        "skipUrlSync": false,
         "type": "datasource"
       },
       {
-        "allValue": null,
         "current": {},
-        "datasource": "${DS_INFLUXDB}",
+        "datasource": {
+          "type": "influxdb",
+          "uid": "${DS_INFLUXDB}"
+        },
+        "definition": "",
         "hide": 0,
         "includeAll": false,
-        "label": null,
         "multi": false,
         "name": "host",
         "options": [],
         "query": "SHOW TAG VALUES FROM \"geodata\" WITH KEY = \"host\"",
         "refresh": 1,
         "regex": "",
+        "skipUrlSync": false,
         "sort": 0,
         "tagValuesQuery": "",
-        "tags": [],
         "tagsQuery": "",
         "type": "query",
         "useTags": false
+      },
+      {
+        "current": {},
+        "datasource": {
+          "type": "influxdb",
+          "uid": "${DS_INFLUXDB}"
+        },
+        "definition": "SHOW TAG VALUES FROM \"geodata\" WITH KEY = \"website\"",
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "website",
+        "options": [],
+        "query": "SHOW TAG VALUES FROM \"geodata\" WITH KEY = \"website\"",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
       }
     ]
   },
   "time": {
-    "from": "now-6h",
+    "from": "now-5m",
     "to": "now"
   },
   "timepicker": {
@@ -336,7 +400,8 @@
     ]
   },
   "timezone": "",
-  "title": "GeoMap",
+  "title": "Nginx GEOIP Statistic",
   "uid": "IYo4xyJmz",
-  "version": 6
+  "version": 5,
-}
+  "weekStart": ""
+}

geoparser.py

@@ -3,6 +3,8 @@
 # Parts added by Remko Lodder, 2019.
 # Added: IPv6 matching, make query based on geoip2 instead of
 # geoip, which is going away r.s.n.
+# Added possibility of processing more than one Nginx log file,
+# by adding threading support. 2022 July by Alexey Nizhegolenko
 
 import os
 import re
@@ -15,6 +17,7 @@ import geoip2.database
 import configparser
 from influxdb import InfluxDBClient
 from IPy import IP as ipadd
+import threading
 
 
 class SyslogBOMFormatter(logging.Formatter):
@@ -30,7 +33,7 @@ root = logging.getLogger(__name__)
 root.setLevel(os.environ.get("LOGLEVEL", "INFO"))
 root.addHandler(handler)
 
-def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE): # NOQA
+def logparse(LOGPATH, WEBSITE, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE): # NOQA
     # Preparing variables and params
     IPS = {}
     COUNT = {}
@@ -43,8 +46,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
     re_IPV6 = re.compile('(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))') # NOQA
 
     GI = geoip2.database.Reader(GEOIPDB)
-
+    # Main loop that parses log file in tailf style with sending metrics out
-    # Main loop to parse access.log file in tailf style with sending metrcs
     with open(LOGPATH, "r") as FILE:
         STR_RESULTS = os.stat(LOGPATH)
         ST_SIZE = STR_RESULTS[6]
@@ -55,7 +57,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
             LINE = FILE.readline()
             INODENEW = os.stat(LOGPATH).st_ino
             if INODE != INODENEW:
-                break
+                return
             if not LINE:
                 time.sleep(1)
                 FILE.seek(WHERE)
@@ -74,6 +76,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
                         COUNT['count'] = 1
                         GEOHASH['geohash'] = HASH
                         GEOHASH['host'] = HOSTNAME
+                        GEOHASH['website'] = WEBSITE
                         GEOHASH['country_code'] = INFO.country.iso_code
                         GEOHASH['country_name'] = INFO.country.name
                         GEOHASH['city_name'] = INFO.city.name
@@ -81,8 +84,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
                         IPS['fields'] = COUNT
                         IPS['measurement'] = MEASUREMENT
                         METRICS.append(IPS)
-
+                        # Sending json data itto InfluxDB
-                        # Sending json data to InfluxDB
                         try:
                             CLIENT.write_points(METRICS)
                         except Exception:
@@ -90,14 +92,14 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
 
 
 def main():
-    # Preparing for reading config file
+    # Preparing for reading the config file
     PWD = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
     CONFIG = configparser.ConfigParser()
-    CONFIG.read('%s/settings.ini' % PWD)
+    CONFIG.read(f'{PWD}/settings.ini')
 
     # Getting params from config
     GEOIPDB = CONFIG.get('GEOIP', 'geoipdb')
-    LOGPATH = CONFIG.get('NGINX_LOG', 'logpath')
+    LOGPATH = CONFIG.get('NGINX_LOGS', 'logpath').split()
     INFLUXHOST = CONFIG.get('INFLUXDB', 'host')
     INFLUXPORT = CONFIG.get('INFLUXDB', 'port')
     INFLUXDBDB = CONFIG.get('INFLUXDB', 'database')
@@ -107,14 +109,29 @@ def main():
 
     # Parsing log file and sending metrics to Influxdb
     while True:
-        # Get inode from log file
+        logs = []
-        INODE = os.stat(LOGPATH).st_ino
+        thread_names = []
-        # Run main loop and grep a log file
+        for logitem in LOGPATH:
-        if os.path.exists(LOGPATH):
+            logs.append(logitem.split(":"))
-            logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE) # NOQA
+        for website, log in logs:
-        else:
+            # Get inode from log file
-            logging.info('Nginx log file %s not found', LOGPATH)
+            if os.path.exists(log):
-            print('Nginx log file %s not found' % LOGPATH)
+                INODE = os.stat(log).st_ino
+            else:
+                logging.info('Nginx log file %s not found', log)
+                print('Nginx log file %s not found' % log)
+                return
+            # Run the main loop and grep data in separate threads
+            t = website
+            if os.path.exists(log):
+                t = threading.Thread(target=logparse, args=[log, website, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE], daemon=True, name=website) # NOQA
+                for thread in threading.enumerate():
+                    thread_names.append(thread.name)
+                if website not in thread_names:
+                    t.start()
+            else:
+                logging.info('Nginx log file %s not found', log)
+                print('Nginx log file %s not found' % log)
 
 
 if __name__ == '__main__':

settings.ini.back

@@ -1,6 +1,6 @@
-[NGINX_LOG]
+[NGINX_LOGS]
 #Path for the log file (Nginx)
-logpath = /var/log/nginx/access.log
+logpath = website1:/var/log/website1/access.log website2:/var/log/website2/access.log
 
 [GEOIP]
 #Path for the GEOIP DB file