simono41/geostat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fixed logparse function arguments pass

Browse source
This commit is contained in:
Alexey Nizhegolenko 2022-07-21 13:59:18 +03:00
parent 96ddf180a9
commit 156453cf76
1 changed files with 12 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
geoparser.py
View file

@ -48,7 +48,7 @@ def logparse(LOGPATH, WEBSITE, MEASUREMENT, GEOIPDB, INODE, INFLUXDB_VERSION,
CLIENT = InfluxDBClient(host=INFLUXHOST, port=INFLUXPORT, CLIENT = InfluxDBClient(host=INFLUXHOST, port=INFLUXPORT,
username=INFLUXUSER, password=INFLUXUSERPASS, database=INFLUXDBDB) # NOQA username=INFLUXUSER, password=INFLUXUSERPASS, database=INFLUXDBDB) # NOQA
elif INFLUXDB_VERSION == "2": elif INFLUXDB_VERSION == "2":
CLIENT = InfluxDBClient(url=URL, token=INFLUXDBTOKEN, org=INFLUXDBORG) # NOQA CLIENT = InfluxDBClient2(url=URL, token=INFLUXDBTOKEN, org=INFLUXDBORG) # NOQA
re_IPV4 = re.compile('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') re_IPV4 = re.compile('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})')
re_IPV6 = re.compile('(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))') # NOQA re_IPV6 = re.compile('(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))') # NOQA
@ -108,7 +108,8 @@ def main():
PWD = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) PWD = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
CONFIG = configparser.ConfigParser() CONFIG = configparser.ConfigParser()
CONFIG.read(f'{PWD}/settings.ini') CONFIG.read(f'{PWD}/settings.ini')
KWARGS1 = {}
KWARGS2 = {}
# Get the InfluxDB version so we can parse only needed part of config # Get the InfluxDB version so we can parse only needed part of config
INFLUXDB_VERSION = CONFIG.get('INFLUXDB_VERSION', 'version') INFLUXDB_VERSION = CONFIG.get('INFLUXDB_VERSION', 'version')
@ -122,15 +123,21 @@ def main():
INFLUXUSER = CONFIG.get('INFLUXDB1', 'username') INFLUXUSER = CONFIG.get('INFLUXDB1', 'username')
MEASUREMENT = CONFIG.get('INFLUXDB1', 'measurement') MEASUREMENT = CONFIG.get('INFLUXDB1', 'measurement')
INFLUXUSERPASS = CONFIG.get('INFLUXDB1', 'password') INFLUXUSERPASS = CONFIG.get('INFLUXDB1', 'password')
KWARGS1 = {'GEOIPDB': GEOIPDB, 'LOGPATH': LOGPATH, 'INFLUXHOST': INFLUXHOST,
'INFLUXPORT': INFLUXPORT, 'INFLUXDBDB': INFLUXDBDB,
'INFLUXUSER': INFLUXUSER, 'MEASUREMENT': MEASUREMENT,
'INFLUXUSERPASS': INFLUXUSERPASS, 'INFLUXDB_VERSION': INFLUXDB_VERSION} # NOQA
elif INFLUXDB_VERSION == "2": elif INFLUXDB_VERSION == "2":
# Getting params from config for version 2 # Getting params from config for version 2
GEOIPDB = CONFIG.get('GEOIP', 'geoipdb')
LOGPATH = CONFIG.get('NGINX_LOGS', 'logpath').split() LOGPATH = CONFIG.get('NGINX_LOGS', 'logpath').split()
URL = CONFIG.get('INFLUXDB2', 'url') URL = CONFIG.get('INFLUXDB2', 'url')
INFLUXDBTOKEN = CONFIG.get('INFLUXDB2', 'token') INFLUXDBTOKEN = CONFIG.get('INFLUXDB2', 'token')
INFLUXDBBUCKET = CONFIG.get('INFLUXDB2', 'bucket') INFLUXDBBUCKET = CONFIG.get('INFLUXDB2', 'bucket')
MEASUREMENT = CONFIG.get('INFLUXDB2', 'measurement') MEASUREMENT = CONFIG.get('INFLUXDB2', 'measurement')
INFLUXDBORG = CONFIG.get('INFLUXDB2', 'organization') INFLUXDBORG = CONFIG.get('INFLUXDB2', 'organization')
KWARGS2 = {'LOGPATH': LOGPATH, 'URL': URL, 'INFLUXDBTOKEN': INFLUXDBTOKEN,
'INFLUXDBBUCKET': INFLUXDBBUCKET, 'MEASUREMENT': MEASUREMENT,
'INFLUXDBORG': INFLUXDBORG} # NOQA
# Parsing log file and sending metrics to Influxdb # Parsing log file and sending metrics to Influxdb
while True: while True:
@ -151,7 +158,7 @@ def main():
# Run the main loop and grep data in separate threads # Run the main loop and grep data in separate threads
t = website t = website
if os.path.exists(log): if os.path.exists(log):
t = threading.Thread(target=logparse, args=[log, website, INFLUXDB_VERSION, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT, GEOIPDB, INODE], daemon=True, name=website) # NOQA t = threading.Thread(target=logparse, kwargs=KWARGS1, daemon=True, name=website) # NOQA
for thread in threading.enumerate(): for thread in threading.enumerate():
thread_names.append(thread.name) thread_names.append(thread.name)
if website not in thread_names: if website not in thread_names:
@ -163,7 +170,7 @@ def main():
# Run the main loop and grep data in separate threads # Run the main loop and grep data in separate threads
t = website t = website
if os.path.exists(log): if os.path.exists(log):
t = threading.Thread(target=logparse, args=[log, website, INFLUXDB_VERSION, URL, INFLUXDBTOKEN, INFLUXDBBUCKET, INFLUXDBORG, MEASUREMENT, GEOIPDB, INODE], daemon=True, name=website) # NOQA t = threading.Thread(target=logparse, kwargs=KWARGS2, daemon=True, name=website) # NOQA
for thread in threading.enumerate(): for thread in threading.enumerate():
thread_names.append(thread.name) thread_names.append(thread.name)
if website not in thread_names: if website not in thread_names: