Compare commits

...

10 commits

Author SHA1 Message Date
Florian Rupp
f51d800aca
Merge pull request #1 from flor0/testing
minor change to README
2023-05-10 18:31:58 +02:00
Florian Rupp
16e2308409 minor change to README 2023-05-10 18:29:23 +02:00
Florian Rupp
859085137f better info in stdout 2023-04-21 18:35:06 +02:00
Florian Rupp
3c40861c70 Fixed except 2023-04-21 18:32:48 +02:00
Florian Rupp
75dda28308 Merge branch 'main' of github.com:flor0/prometheus-ssh-exporter 2023-04-21 18:27:30 +02:00
Florian Rupp
a72844c3b2 Handles short sessions
Can now handle sessions of duration < FETCH_INTERVAL
2023-04-21 18:27:28 +02:00
Florian Rupp
ba7fc6d2ff
Update README.md grafana screenshot 2023-04-18 23:35:19 +02:00
Florian Rupp
e9a0742b6e added grafana dashboard 2023-04-18 22:59:43 +02:00
Florian Rupp
c4172cc19e update readme 2023-04-18 22:54:17 +02:00
Florian Rupp
edbc83cb42 Added support for local sessions
Local sessions (not over SSH) are identified by 'localhost'
2023-04-18 22:39:21 +02:00
4 changed files with 578 additions and 28 deletions

View file

@ -31,11 +31,12 @@ services:
- /run/utmp:/run/utmp - /run/utmp:/run/utmp
restart: unless-stopped restart: unless-stopped
``` ```
Note: It is vital that /run/utmp is mapped in the docker container, otherwise the program can't get your session info!
### As a python script ### As a python script
Alternatively, you can simply run the prometheus-ssh-exporter.py file with python3. Alternatively, you can simply run the prometheus-ssh-exporter.py file with python3.
The command line arguments are explained if you use `python3 ./prometheus-ssh-exporter.py -h` The command line arguments are explained if you use `python3 ./prometheus-ssh-exporter.py --help`
Make sure you set the right external port using the -p or --port argument. Make sure you set the right external port using the -p or --port argument.
@ -57,6 +58,8 @@ If everything is set up correctly you should get the metrics.
### Grafana ### Grafana
I have published a grafana dashboard that can be found at https://snapshots.raintank.io/dashboard/snapshot/bZSxJ2Ig9EkPbl0Ozt5lS9SYADtSlvIY I have published a grafana dashboard that can be found at https://snapshots.raintank.io/dashboard/snapshot/fnZ63M865o3J0N0ne9lSKfezGe7ERYVN
or imported with the grafana-dashboard.json file.
![prometheus-grafana](https://user-images.githubusercontent.com/48520760/232910344-89fb6557-0160-4f83-a794-ebcca4df28df.png)

499
grafana-dashboard.json Normal file
View file

@ -0,0 +1,499 @@
{
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "9.4.3"
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "stat",
"name": "Stat",
"version": ""
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "Overview of SSH session activity intended for personal servers",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "builder",
"exemplar": false,
"expr": "sum(ssh_num_sessions{remote_ip!=\"-\"})",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Current SSH Sessions",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 10
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"id": 11,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "builder",
"exemplar": false,
"expr": "sum(ssh_num_sessions{remote_ip=\"localhost\"})",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Current Local Sessions",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "builder",
"expr": "sum(ssh_num_sessions)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Total Sessions",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "builder",
"exemplar": false,
"expr": "ssh_num_sessions{remote_ip!=\"-\", job=\"ssh\"}",
"format": "time_series",
"instant": false,
"legendFormat": "{{ remote_ip }}",
"range": true,
"refId": "A"
}
],
"title": "SSH Sessions per IP",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Time"
},
"properties": [
{
"id": "custom.width",
"value": 195
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 20
},
"id": 10,
"options": {
"footer": {
"countRows": false,
"enablePagination": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": []
},
"pluginVersion": "9.4.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "builder",
"exemplar": false,
"expr": "sum by(remote_ip) (ssh_num_sessions)",
"format": "table",
"instant": true,
"legendFormat": "{{remote_ip}}",
"range": false,
"refId": "A"
}
],
"title": "List of current Sessions",
"type": "table"
}
],
"refresh": "5s",
"revision": 1,
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-5m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "SSH Sessions",
"uid": "-fpS1UL4k",
"version": 7,
"weekStart": ""
}

View file

@ -2,11 +2,21 @@ import prometheus_client
import time import time
import argparse import argparse
import utmp import utmp
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler, FileModifiedEvent
# These defaults can be overwritten by command line arguments # These defaults can be overwritten by command line arguments
SERVER_HOST = '0.0.0.0' SERVER_HOST = '0.0.0.0'
SERVER_PORT = 9999 SERVER_PORT = 9999
FETCH_INTERVAL = 15 FETCH_INTERVAL = 15
WATCHFILE = '/var/run/utmp'
class FileOpenedHandler(FileSystemEventHandler):
def on_modified(self, event):
if not event.is_directory and isinstance(event, FileModifiedEvent) and event.src_path == WATCHFILE:
handle_sessions_changed()
class Session: class Session:
@ -38,66 +48,28 @@ def get_utmp_data() -> list[Session]:
Returns a list of User Objects Returns a list of User Objects
The function uses the utmp library. The utmp file contains information about ALL currently logged in users, The function uses the utmp library. The utmp file contains information about ALL currently logged in users,
including local users (not SSH sessions). We filter out the local users by checking if the remote IP address including local users (not SSH sessions). We filter out the local users by checking if the remote IP address
is empty. is empty and set the hostname for the local sessions to "localhost".
""" """
users : list[Session] = [] users : list[Session] = []
with open('/var/run/utmp', 'rb') as fd: with open('/var/run/utmp', 'rb') as fd:
buffer = fd.read() buffer = fd.read()
for record in utmp.read(buffer): for record in utmp.read(buffer):
if record.type == utmp.UTmpRecordType.user_process and record.host != '': if record.type == utmp.UTmpRecordType.user_process:
if record.host != '':
users.append(Session(record.user, record.line, record.host, record.sec)) users.append(Session(record.user, record.line, record.host, record.sec))
else:
users.append(Session(record.user, record.line, 'localhost', record.sec))
return users return users
def parse_arguments() -> None: def handle_sessions_changed() -> None:
global FETCH_INTERVAL, SERVER_PORT, SERVER_HOST
parser = argparse.ArgumentParser(
prog='python prometheus-ssh-exporter.py',
description='Prometheus exporter for info about SSH sessions')
parser.add_argument('-H', '--host', type=str,
default='0.0.0.0', help='Hostname to bind to')
parser.add_argument('-p', '--port', type=int, default=9999,
help='Port for the server to listen to')
parser.add_argument('-i', '--interval', type=int, default=15,
help='Interval in seconds to fetch SSH sessions data')
args = parser.parse_args()
FETCH_INTERVAL = args.interval
SERVER_PORT = args.port
SERVER_HOST = args.host
if __name__ == '__main__':
""" """
This program exports the number of SSH sessions as a metric "ssh_num_sessions" for prometheus. This function fetches the current list of SSH sessions and compares it to the previous list of SSH sessions.
It applies a label to each increment or decrement of that number, containing the remote IP address. If the number of sessions has changed, it increments or decrements the gauge_num_sessions metric
That way we can filter by the remote IP in Grafana, getting the number of SSH sessions by IP address, and updates the session_data and num_sessions variables.
or sum them up to get the total number of sessions.
""" """
global session_data, num_sessions, gauge_num_sessions, old_session_data, old_num_sessions
parse_arguments()
# Start up the server to expose the metrics.
prometheus_client.start_http_server(SERVER_PORT)
print("Started metrics server bound to {}:{}".format(SERVER_HOST, SERVER_PORT))
gauge_num_sessions = prometheus_client.Gauge(
'ssh_num_sessions', 'Number of SSH sessions', ['remote_ip'])
session_data = get_utmp_data()
num_sessions = len(session_data)
# Initial metrics
print("Active sessions at startup:")
for session in session_data:
gauge_num_sessions.labels(remote_ip=session.from_).inc()
print("Initial connection: {}".format(session.from_))
# Generate some requests.
print("Looking for SSH connection changes at interval {}".format(FETCH_INTERVAL))
while True:
old_session_data = session_data old_session_data = session_data
old_num_sessions = len(old_session_data) old_num_sessions = len(old_session_data)
@ -117,4 +89,79 @@ if __name__ == '__main__':
print("Session disconnected: %s" % maybe_old_session.from_) print("Session disconnected: %s" % maybe_old_session.from_)
gauge_num_sessions.labels(remote_ip=maybe_old_session.from_).dec() gauge_num_sessions.labels(remote_ip=maybe_old_session.from_).dec()
def parse_arguments() -> None:
global FETCH_INTERVAL, SERVER_PORT, SERVER_HOST, WATCHFILE
parser = argparse.ArgumentParser(
prog='python prometheus-ssh-exporter.py',
description='Prometheus exporter for info about SSH sessions')
parser.add_argument('-H', '--host', type=str,
default='0.0.0.0', help='Hostname to bind to')
parser.add_argument('-p', '--port', type=int, default=9999,
help='Port for the server to listen to')
parser.add_argument('-i', '--interval', type=int, default=15,
help='Interval in seconds to fetch SSH sessions data')
parser.add_argument('-f', '--file', type=str, default='/var/run/utmp',
help='File that changes every time a new SSH session is opened or closed')
args = parser.parse_args()
FETCH_INTERVAL = args.interval
SERVER_PORT = args.port
SERVER_HOST = args.host
WATCHFILE = args.file
if __name__ == '__main__':
"""
This program exports the number of SSH sessions as a metric "ssh_num_sessions" for prometheus.
It applies a label to each increment or decrement of that number, containing the remote IP address.
That way we can filter by the remote IP in Grafana, getting the number of SSH sessions by IP address,
or sum them up to get the total number of sessions.
"""
parse_arguments()
# Start up the server to expose the metrics.
prometheus_client.start_http_server(SERVER_PORT)
print("Started metrics server bound to {}:{}".format(SERVER_HOST, SERVER_PORT))
gauge_num_sessions = prometheus_client.Gauge(
'ssh_num_sessions', 'Number of SSH sessions', ['remote_ip'])
# session_data contains the current list of sessions
session_data = get_utmp_data()
num_sessions = len(session_data)
# Initial metrics
for session in session_data:
gauge_num_sessions.labels(remote_ip=session.from_).inc()
print("Initial connection: {}".format(session.from_))
"""
Start the watchdog to monitor the WATCHDOG file for changes.
This is used to immediately look for changes in the SSH sessions when a new session is opened or closed
to prevent missing any sessions that lasted less than the FETCH_INTERVAL.
"""
print("Watching file {} for changes...".format(WATCHFILE))
event_handler = FileOpenedHandler()
observer = Observer()
observer.schedule(event_handler, path=WATCHFILE, recursive=False)
observer.start()
# Generate some requests.
print("Looking for SSH connection changes at interval {}".format(FETCH_INTERVAL))
try:
while True:
# Keep looking for changes in the SSH sessions in case the watchdog missed something
handle_sessions_changed()
time.sleep(FETCH_INTERVAL) time.sleep(FETCH_INTERVAL)
except:
print("Terminating...")
observer.stop()
observer.join()

View file

@ -1,2 +1,3 @@
prometheus_client==0.16.0 prometheus_client==0.16.0
utmp==21.10.0 utmp==21.10.0
watchdog==3.0.0