Compare commits

..

No commits in common. "f51d800aca7f0c4ae1fa4614134b3b097c4f908e" and "03c25129c9e955710dad4316246d6506304e5a7a" have entirely different histories.

4 changed files with 28 additions and 578 deletions

View file

@ -31,12 +31,11 @@ services:
- /run/utmp:/run/utmp - /run/utmp:/run/utmp
restart: unless-stopped restart: unless-stopped
``` ```
Note: It is vital that /run/utmp is mapped in the docker container, otherwise the program can't get your session info!
### As a python script ### As a python script
Alternatively, you can simply run the prometheus-ssh-exporter.py file with python3. Alternatively, you can simply run the prometheus-ssh-exporter.py file with python3.
The command line arguments are explained if you use `python3 ./prometheus-ssh-exporter.py --help` The command line arguments are explained if you use `python3 ./prometheus-ssh-exporter.py -h`
Make sure you set the right external port using the -p or --port argument. Make sure you set the right external port using the -p or --port argument.
@ -58,8 +57,6 @@ If everything is set up correctly you should get the metrics.
### Grafana ### Grafana
I have published a grafana dashboard that can be found at https://snapshots.raintank.io/dashboard/snapshot/fnZ63M865o3J0N0ne9lSKfezGe7ERYVN I have published a grafana dashboard that can be found at https://snapshots.raintank.io/dashboard/snapshot/bZSxJ2Ig9EkPbl0Ozt5lS9SYADtSlvIY
or imported with the grafana-dashboard.json file.
![prometheus-grafana](https://user-images.githubusercontent.com/48520760/232910344-89fb6557-0160-4f83-a794-ebcca4df28df.png)

View file

@ -1,499 +0,0 @@
{
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "9.4.3"
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "stat",
"name": "Stat",
"version": ""
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "Overview of SSH session activity intended for personal servers",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "builder",
"exemplar": false,
"expr": "sum(ssh_num_sessions{remote_ip!=\"-\"})",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Current SSH Sessions",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"id": 11,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "builder",
"exemplar": false,
"expr": "sum(ssh_num_sessions{remote_ip=\"localhost\"})",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Current Local Sessions",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "builder",
"expr": "sum(ssh_num_sessions)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Total Sessions",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "builder",
"exemplar": false,
"expr": "ssh_num_sessions{remote_ip!=\"-\", job=\"ssh\"}",
"format": "time_series",
"instant": false,
"legendFormat": "{{ remote_ip }}",
"range": true,
"refId": "A"
}
],
"title": "SSH Sessions per IP",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Time"
},
"properties": [
{
"id": "custom.width",
"value": 195
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 20
},
"id": 10,
"options": {
"footer": {
"countRows": false,
"enablePagination": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": []
},
"pluginVersion": "9.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "builder",
"exemplar": false,
"expr": "sum by(remote_ip) (ssh_num_sessions)",
"format": "table",
"instant": true,
"legendFormat": "{{remote_ip}}",
"range": false,
"refId": "A"
}
],
"title": "List of current Sessions",
"type": "table"
}
],
"refresh": "5s",
"revision": 1,
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-5m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "SSH Sessions",
"uid": "-fpS1UL4k",
"version": 7,
"weekStart": ""
}

View file

@ -2,21 +2,11 @@ import prometheus_client
import time import time
import argparse import argparse
import utmp import utmp
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler, FileModifiedEvent
# These defaults can be overwritten by command line arguments # These defaults can be overwritten by command line arguments
SERVER_HOST = '0.0.0.0' SERVER_HOST = '0.0.0.0'
SERVER_PORT = 9999 SERVER_PORT = 9999
FETCH_INTERVAL = 15 FETCH_INTERVAL = 15
WATCHFILE = '/var/run/utmp'
class FileOpenedHandler(FileSystemEventHandler):
def on_modified(self, event):
if not event.is_directory and isinstance(event, FileModifiedEvent) and event.src_path == WATCHFILE:
handle_sessions_changed()
class Session: class Session:
@ -48,51 +38,21 @@ def get_utmp_data() -> list[Session]:
Returns a list of User Objects Returns a list of User Objects
The function uses the utmp library. The utmp file contains information about ALL currently logged in users, The function uses the utmp library. The utmp file contains information about ALL currently logged in users,
including local users (not SSH sessions). We filter out the local users by checking if the remote IP address including local users (not SSH sessions). We filter out the local users by checking if the remote IP address
is empty and set the hostname for the local sessions to "localhost". is empty.
""" """
users : list[Session] = [] users : list[Session] = []
with open('/var/run/utmp', 'rb') as fd: with open('/var/run/utmp', 'rb') as fd:
buffer = fd.read() buffer = fd.read()
for record in utmp.read(buffer): for record in utmp.read(buffer):
if record.type == utmp.UTmpRecordType.user_process: if record.type == utmp.UTmpRecordType.user_process and record.host != '':
if record.host != '': users.append(Session(record.user, record.line, record.host, record.sec))
users.append(Session(record.user, record.line, record.host, record.sec))
else:
users.append(Session(record.user, record.line, 'localhost', record.sec))
return users return users
def handle_sessions_changed() -> None:
"""
This function fetches the current list of SSH sessions and compares it to the previous list of SSH sessions.
If the number of sessions has changed, it increments or decrements the gauge_num_sessions metric
and updates the session_data and num_sessions variables.
"""
global session_data, num_sessions, gauge_num_sessions, old_session_data, old_num_sessions
old_session_data = session_data
old_num_sessions = len(old_session_data)
session_data = get_utmp_data()
num_sessions = len(session_data)
for maybe_new_session in session_data:
# Looking for newly found SSH sessions
if not maybe_new_session in old_session_data:
print("Session connected: %s" % maybe_new_session.from_)
gauge_num_sessions.labels(remote_ip=maybe_new_session.from_).inc()
for maybe_old_session in old_session_data:
# Looking for SSH sessions that no longer exist
if not maybe_old_session in session_data:
print("Session disconnected: %s" % maybe_old_session.from_)
gauge_num_sessions.labels(remote_ip=maybe_old_session.from_).dec()
def parse_arguments() -> None: def parse_arguments() -> None:
global FETCH_INTERVAL, SERVER_PORT, SERVER_HOST, WATCHFILE
global FETCH_INTERVAL, SERVER_PORT, SERVER_HOST
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog='python prometheus-ssh-exporter.py', prog='python prometheus-ssh-exporter.py',
@ -103,15 +63,11 @@ def parse_arguments() -> None:
help='Port for the server to listen to') help='Port for the server to listen to')
parser.add_argument('-i', '--interval', type=int, default=15, parser.add_argument('-i', '--interval', type=int, default=15,
help='Interval in seconds to fetch SSH sessions data') help='Interval in seconds to fetch SSH sessions data')
parser.add_argument('-f', '--file', type=str, default='/var/run/utmp',
help='File that changes every time a new SSH session is opened or closed')
args = parser.parse_args() args = parser.parse_args()
FETCH_INTERVAL = args.interval FETCH_INTERVAL = args.interval
SERVER_PORT = args.port SERVER_PORT = args.port
SERVER_HOST = args.host SERVER_HOST = args.host
WATCHFILE = args.file
if __name__ == '__main__': if __name__ == '__main__':
@ -122,7 +78,6 @@ if __name__ == '__main__':
or sum them up to get the total number of sessions. or sum them up to get the total number of sessions.
""" """
parse_arguments() parse_arguments()
# Start up the server to expose the metrics. # Start up the server to expose the metrics.
@ -131,37 +86,35 @@ if __name__ == '__main__':
gauge_num_sessions = prometheus_client.Gauge( gauge_num_sessions = prometheus_client.Gauge(
'ssh_num_sessions', 'Number of SSH sessions', ['remote_ip']) 'ssh_num_sessions', 'Number of SSH sessions', ['remote_ip'])
# session_data contains the current list of sessions
session_data = get_utmp_data() session_data = get_utmp_data()
num_sessions = len(session_data) num_sessions = len(session_data)
# Initial metrics # Initial metrics
print("Active sessions at startup:")
for session in session_data: for session in session_data:
gauge_num_sessions.labels(remote_ip=session.from_).inc() gauge_num_sessions.labels(remote_ip=session.from_).inc()
print("Initial connection: {}".format(session.from_)) print("Initial connection: {}".format(session.from_))
"""
Start the watchdog to monitor the WATCHDOG file for changes.
This is used to immediately look for changes in the SSH sessions when a new session is opened or closed
to prevent missing any sessions that lasted less than the FETCH_INTERVAL.
"""
print("Watching file {} for changes...".format(WATCHFILE))
event_handler = FileOpenedHandler()
observer = Observer()
observer.schedule(event_handler, path=WATCHFILE, recursive=False)
observer.start()
# Generate some requests. # Generate some requests.
print("Looking for SSH connection changes at interval {}".format(FETCH_INTERVAL)) print("Looking for SSH connection changes at interval {}".format(FETCH_INTERVAL))
try: while True:
while True: old_session_data = session_data
# Keep looking for changes in the SSH sessions in case the watchdog missed something old_num_sessions = len(old_session_data)
handle_sessions_changed()
time.sleep(FETCH_INTERVAL)
except: session_data = get_utmp_data()
print("Terminating...") num_sessions = len(session_data)
observer.stop()
observer.join() for maybe_new_session in session_data:
# Looking for newly found SSH sessions
if not maybe_new_session in old_session_data:
print("Session connected: %s" % maybe_new_session.from_)
gauge_num_sessions.labels(remote_ip=maybe_new_session.from_).inc()
for maybe_old_session in old_session_data:
# Looking for SSH sessions that no longer exist
if not maybe_old_session in session_data:
print("Session disconnected: %s" % maybe_old_session.from_)
gauge_num_sessions.labels(remote_ip=maybe_old_session.from_).dec()
time.sleep(FETCH_INTERVAL)

View file

@ -1,3 +1,2 @@
prometheus_client==0.16.0 prometheus_client==0.16.0
utmp==21.10.0 utmp==21.10.0
watchdog==3.0.0