simono41/geostat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Adding influxdb support

Browse source
This commit is contained in:
Alexey 2018-10-08 19:00:33 +03:00
parent 82844733ff
commit 58d4cf394a
2 changed files with 31 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
geoparser.py
View file

@ -7,21 +7,24 @@ import os
import re import re
# import sys # import sys
import time import time
# import json import json
import pygeoip import pygeoip
# import subprocess # import subprocess
import Geohash import Geohash
import configparser import configparser
from influxdb import InfluxDBClient
# from collections import Counter # from collections import Counter
# from datetime import datetime # from datetime import datetime
def logparse(logpath): def logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT): # NOQA
GI = pygeoip.GeoIP('GeoLiteCity.dat', pygeoip.const.MEMORY_CACHE) GI = pygeoip.GeoIP('GeoLiteCity.dat', pygeoip.const.MEMORY_CACHE)
GETIP = r"^(?P<remote_host>[0-9]{,3}\.[0-9]{,3}\.[0-9]{,3}\.[0-9]{,3})" GETIP = r"^(?P<remote_host>[0-9]{,3}\.[0-9]{,3}\.[0-9]{,3}\.[0-9]{,3})"
IPS = {} IPS = {}
with open(logpath, "r") as FILE: GEOHASH = {}
STR_RESULTS = os.stat(logpath) COUNT = {}
with open(LOGPATH, "r") as FILE:
STR_RESULTS = os.stat(LOGPATH)
ST_SIZE = STR_RESULTS[6] ST_SIZE = STR_RESULTS[6]
FILE.seek(ST_SIZE) FILE.seek(ST_SIZE)
while 1: while 1:
@ -36,22 +39,33 @@ def logparse(logpath):
INFO = GI.record_by_addr(IP) INFO = GI.record_by_addr(IP)
if INFO is not None: if INFO is not None:
HASH = Geohash.encode(INFO['latitude'], INFO['longitude']) # NOQA HASH = Geohash.encode(INFO['latitude'], INFO['longitude']) # NOQA
IPS['count'] = 1 GEOHASH['geohash'] = HASH
IPS['geohash'] = HASH COUNT['count'] = 1
print(IPS) IPS['measurement'] = 'MEASUREMENT'
IPS['tags'] = GEOHASH
IPS['fields'] = COUNT
RESULT = json.dumps(IPS)
print(RESULT)
def main(): def main():
# Preparing of config reading
PWD = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
CONFIG = configparser.ConfigParser()
CONFIG.read('%s/settings.ini' % PWD)
# Getting params from config # Getting params from config
pwd = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) LOGPATH = CONFIG.get('NGINX_LOG', 'logpath')
config = configparser.ConfigParser() INFLUXHOST = CONFIG.get('INFLUXDB', 'host')
config.read('%s/settings.ini' % pwd) INFLUXPORT = CONFIG.get('INFLUXDB', 'port')
logpath = config.get('NGINX_LOG', 'logpath') INFLUXDBDB = CONFIG.get('INFLUXDB', 'database')
# Parse given log file and send metrics INFLUXUSER = CONFIG.get('INFLUXDB', 'username')
logparse(logpath) MEASUREMENT = CONFIG.get('INFLUXDB', 'measurement')
# Send result to fle INFLUXUSERPASS = CONFIG.get('INFLUXDB', 'password')
# with open('/tmp/metrics.json', 'w') as outputfile:
# json.dump(IPS_IN, outputfile, indent=4, sort_keys=True) # Parsing log file and sending metrics to Influxdb
logparse(LOGPATH, INFLUXHOST, INFLUXPORT, INFLUXDBDB, INFLUXUSER, INFLUXUSERPASS, MEASUREMENT) # NOQA
if __name__ == '__main__': if __name__ == '__main__':

1
requirements.txt
View file

@ -1,4 +1,5 @@
configparser==3.5.0 configparser==3.5.0
parsedatetime==2.4 parsedatetime==2.4
influxdb==5.2.0
pygeoip==0.3.2 pygeoip==0.3.2
Geohash==1.0 Geohash==1.0