simono41/geostat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Added the standard logging syslog support, also were added a additional tags with city.name and country.name

Browse source
This commit is contained in:
Alexey Nizhegolenko 2021-01-24 14:44:33 +02:00
parent 9f264cdcef
commit 2ba86f3f1b
2 changed files with 29 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
geoparser.py
View file

@ -1,4 +1,4 @@
#! /usr/bin/env python #! /usr/bin/env python3
# Getting GEO information from Nginx access.log by IP's. # Getting GEO information from Nginx access.log by IP's.
# Alexey Nizhegolenko 2018 # Alexey Nizhegolenko 2018
@ -10,12 +10,28 @@ import os
import re import re
import sys import sys
import time import time
import geoip2.database
import geohash import geohash
import logging
import logging.handlers
import geoip2.database
import configparser import configparser
from influxdb import InfluxDBClient from influxdb import InfluxDBClient
from IPy import IP as ipadd from IPy import IP as ipadd
class SyslogBOMFormatter(logging.Formatter):
def format(self, record):
result = super().format(record)
return "ufeff" + result
handler = logging.handlers.SysLogHandler('/dev/log')
formatter = SyslogBOMFormatter(logging.BASIC_FORMAT)
handler.setFormatter(formatter)
root = logging.getLogger(__name__)
root.setLevel(os.environ.get("LOGLEVEL", "INFO"))
root.addHandler(handler)
def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE): # NOQA def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE): # NOQA
# Preparing variables and params # Preparing variables and params
IPS = {} IPS = {}
@ -53,7 +69,7 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
m = re_IPV6.match(LINE) m = re_IPV6.match(LINE)
IP = m.group(1) IP = m.group(1)
if ipadd(IP).iptype() == 'PUBLIC' and IP: if ipadd(IP).iptype() == 'PUBLIC' and IP:
INFO = GI.city(IP) INFO = GI.city(IP)
if INFO is not None: if INFO is not None:
HASH = geohash.encode(INFO.location.latitude, INFO.location.longitude) # NOQA HASH = geohash.encode(INFO.location.latitude, INFO.location.longitude) # NOQA
@ -61,13 +77,18 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
GEOHASH['geohash'] = HASH GEOHASH['geohash'] = HASH
GEOHASH['host'] = HOSTNAME GEOHASH['host'] = HOSTNAME
GEOHASH['country_code'] = INFO.country.iso_code GEOHASH['country_code'] = INFO.country.iso_code
GEOHASH['country_name'] = INFO.country.country.name
GEOHASH['city_name'] = INFO.city.name
IPS['tags'] = GEOHASH IPS['tags'] = GEOHASH
IPS['fields'] = COUNT IPS['fields'] = COUNT
IPS['measurement'] = MEASUREMENT IPS['measurement'] = MEASUREMENT
METRICS.append(IPS) METRICS.append(IPS)
# Sending json data to InfluxDB # Sending json data to InfluxDB
CLIENT.write_points(METRICS) try:
CLIENT.write_points(METRICS)
except Exception:
logging.exception("Cannot establish connection to InfluxDB: ")
def main(): def main():
@ -100,5 +121,8 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
try: try:
main() main()
except Exception:
logging.exception("Exception in main(): ")
except KeyboardInterrupt: except KeyboardInterrupt:
logging.exception("Exception KeyboardInterrupt: ")
sys.exit(0) sys.exit(0)

1
requirements.txt
View file

@ -2,3 +2,4 @@ configparser==3.5.0
influxdb==5.2.0 influxdb==5.2.0
geoip2==2.9.0 geoip2==2.9.0
IPy==1.00 IPy==1.00
logging