simono41/geostat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Cleared a code a bit

Browse source
This commit is contained in:
Alexey 2018-10-08 23:25:40 +03:00
parent ddd2d58649
commit 50700ce12e
1 changed files with 19 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
geoparser.py
View file

@ -1,30 +1,29 @@
#! /usr/bin/env python #! /usr/bin/env python
# Getting GEO information for Nginx access.log IP's. # Getting GEO information from Nginx access.log by IP's.
# Alexey Nizhegolenko 2018 # Alexey Nizhegolenko 2018
import os import os
import re import re
# import sys import sys
import time import time
import json
import pygeoip import pygeoip
# import subprocess
import Geohash import Geohash
import configparser import configparser
from influxdb import InfluxDBClient from influxdb import InfluxDBClient
# from collections import Counter
# from datetime import datetime
def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT): # NOQA def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT): # NOQA
# Preparing variables and params
IPS = {}
COUNT = {}
GEOHASH = {}
CLIENT = InfluxDBClient(host=INFLUXHOST, port=INFLUXPORT, CLIENT = InfluxDBClient(host=INFLUXHOST, port=INFLUXPORT,
username=INFLUXUSER, password=INFLUXUSERPASS, database=INFLUXDBDB) # NOQA username=INFLUXUSER, password=INFLUXUSERPASS, database=INFLUXDBDB) # NOQA
GETIP = r"^(?P<remote_host>[0-9]{,3}\.[0-9]{,3}\.[0-9]{,3}\.[0-9]{,3})" GETIP = r"^(?P<remote_host>[0-9]{,3}\.[0-9]{,3}\.[0-9]{,3}\.[0-9]{,3})"
GI = pygeoip.GeoIP('GeoLiteCity.dat', pygeoip.const.MEMORY_CACHE) GI = pygeoip.GeoIP('GeoLiteCity.dat', pygeoip.const.MEMORY_CACHE)
GEOHASH = {}
COUNT = {} # Main loop to parse access.log file in tailf style with sending metrcs
IPS = {}
with open(LOGPATH, "r") as FILE: with open(LOGPATH, "r") as FILE:
STR_RESULTS = os.stat(LOGPATH) STR_RESULTS = os.stat(LOGPATH)
ST_SIZE = STR_RESULTS[6] ST_SIZE = STR_RESULTS[6]
@ -42,19 +41,19 @@ def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSER
INFO = GI.record_by_addr(IP) INFO = GI.record_by_addr(IP)
if INFO is not None: if INFO is not None:
HASH = Geohash.encode(INFO['latitude'], INFO['longitude']) # NOQA HASH = Geohash.encode(INFO['latitude'], INFO['longitude']) # NOQA
GEOHASH['geohash'] = HASH
COUNT['count'] = 1 COUNT['count'] = 1
IPS['measurement'] = MEASUREMENT GEOHASH['geohash'] = HASH
IPS['tags'] = GEOHASH IPS['tags'] = GEOHASH
IPS['fields'] = COUNT IPS['fields'] = COUNT
IPS['measurement'] = MEASUREMENT
METRICS.append(IPS) METRICS.append(IPS)
RESULT = json.dumps(METRICS)
CLIENT.write_points(RESULT) # Sending json data to InfluxDB
CLIENT.write_points(METRICS)
def main(): def main():
# Preparing for reading config file
# Preparing of config reading
PWD = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) PWD = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
CONFIG = configparser.ConfigParser() CONFIG = configparser.ConfigParser()
CONFIG.read('%s/settings.ini' % PWD) CONFIG.read('%s/settings.ini' % PWD)
@ -73,4 +72,8 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
main() try:
main()
except KeyboardInterrupt:
os.system('clear')
sys.exit()