Compare commits
10 commits
03c25129c9
...
f51d800aca
| Author | SHA1 | Date | |
|---|---|---|---|
|
f51d800aca | ||
|
16e2308409 | ||
|
859085137f | ||
|
|
3c40861c70 | ||
|
|
75dda28308 | ||
|
|
a72844c3b2 | ||
|
|
ba7fc6d2ff | ||
|
|
e9a0742b6e | ||
|
|
c4172cc19e | ||
|
|
edbc83cb42 |
4 changed files with 578 additions and 28 deletions
|
|
@ -31,11 +31,12 @@ services:
|
|||
- /run/utmp:/run/utmp
|
||||
restart: unless-stopped
|
||||
```
|
||||
Note: It is vital that /run/utmp is mapped in the docker container, otherwise the program can't get your session info!
|
||||
|
||||
### As a python script
|
||||
Alternatively, you can simply run the prometheus-ssh-exporter.py file with python3.
|
||||
|
||||
The command line arguments are explained if you use `python3 ./prometheus-ssh-exporter.py -h`
|
||||
The command line arguments are explained if you use `python3 ./prometheus-ssh-exporter.py --help`
|
||||
|
||||
Make sure you set the right external port using the -p or --port argument.
|
||||
|
||||
|
|
@ -57,6 +58,8 @@ If everything is set up correctly you should get the metrics.
|
|||
|
||||
### Grafana
|
||||
|
||||
I have published a grafana dashboard that can be found at https://snapshots.raintank.io/dashboard/snapshot/bZSxJ2Ig9EkPbl0Ozt5lS9SYADtSlvIY
|
||||
I have published a grafana dashboard that can be found at https://snapshots.raintank.io/dashboard/snapshot/fnZ63M865o3J0N0ne9lSKfezGe7ERYVN
|
||||
or imported with the grafana-dashboard.json file.
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
499
grafana-dashboard.json
Normal file
499
grafana-dashboard.json
Normal file
|
|
@ -0,0 +1,499 @@
|
|||
{
|
||||
"__inputs": [
|
||||
{
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "Prometheus",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "prometheus",
|
||||
"pluginName": "Prometheus"
|
||||
}
|
||||
],
|
||||
"__elements": {},
|
||||
"__requires": [
|
||||
{
|
||||
"type": "grafana",
|
||||
"id": "grafana",
|
||||
"name": "Grafana",
|
||||
"version": "9.4.3"
|
||||
},
|
||||
{
|
||||
"type": "datasource",
|
||||
"id": "prometheus",
|
||||
"name": "Prometheus",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "stat",
|
||||
"name": "Stat",
|
||||
"version": ""
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "table",
|
||||
"name": "Table",
|
||||
"version": ""
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "timeseries",
|
||||
"name": "Time series",
|
||||
"version": ""
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Overview of SSH session activity intended for personal servers",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 10
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "9.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "builder",
|
||||
"exemplar": false,
|
||||
"expr": "sum(ssh_num_sessions{remote_ip!=\"-\"})",
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Current SSH Sessions",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 10
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 0
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "9.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "builder",
|
||||
"exemplar": false,
|
||||
"expr": "sum(ssh_num_sessions{remote_ip=\"localhost\"})",
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Current Local Sessions",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "builder",
|
||||
"expr": "sum(ssh_num_sessions)",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Total Sessions",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "builder",
|
||||
"exemplar": false,
|
||||
"expr": "ssh_num_sessions{remote_ip!=\"-\", job=\"ssh\"}",
|
||||
"format": "time_series",
|
||||
"instant": false,
|
||||
"legendFormat": "{{ remote_ip }}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "SSH Sessions per IP",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"cellOptions": {
|
||||
"type": "auto"
|
||||
},
|
||||
"inspect": false
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Time"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 195
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 20
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"footer": {
|
||||
"countRows": false,
|
||||
"enablePagination": false,
|
||||
"fields": "",
|
||||
"reducer": [
|
||||
"sum"
|
||||
],
|
||||
"show": false
|
||||
},
|
||||
"showHeader": true,
|
||||
"sortBy": []
|
||||
},
|
||||
"pluginVersion": "9.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "builder",
|
||||
"exemplar": false,
|
||||
"expr": "sum by(remote_ip) (ssh_num_sessions)",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"legendFormat": "{{remote_ip}}",
|
||||
"range": false,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "List of current Sessions",
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
"revision": 1,
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-5m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "SSH Sessions",
|
||||
"uid": "-fpS1UL4k",
|
||||
"version": 7,
|
||||
"weekStart": ""
|
||||
}
|
||||
|
|
@ -2,11 +2,21 @@ import prometheus_client
|
|||
import time
|
||||
import argparse
|
||||
import utmp
|
||||
from watchdog.observers import Observer
|
||||
from watchdog.events import FileSystemEventHandler, FileModifiedEvent
|
||||
|
||||
|
||||
# These defaults can be overwritten by command line arguments
|
||||
SERVER_HOST = '0.0.0.0'
|
||||
SERVER_PORT = 9999
|
||||
FETCH_INTERVAL = 15
|
||||
WATCHFILE = '/var/run/utmp'
|
||||
|
||||
|
||||
class FileOpenedHandler(FileSystemEventHandler):
|
||||
def on_modified(self, event):
|
||||
if not event.is_directory and isinstance(event, FileModifiedEvent) and event.src_path == WATCHFILE:
|
||||
handle_sessions_changed()
|
||||
|
||||
|
||||
class Session:
|
||||
|
|
@ -38,21 +48,51 @@ def get_utmp_data() -> list[Session]:
|
|||
Returns a list of User Objects
|
||||
The function uses the utmp library. The utmp file contains information about ALL currently logged in users,
|
||||
including local users (not SSH sessions). We filter out the local users by checking if the remote IP address
|
||||
is empty.
|
||||
is empty and set the hostname for the local sessions to "localhost".
|
||||
"""
|
||||
users : list[Session] = []
|
||||
with open('/var/run/utmp', 'rb') as fd:
|
||||
buffer = fd.read()
|
||||
for record in utmp.read(buffer):
|
||||
if record.type == utmp.UTmpRecordType.user_process and record.host != '':
|
||||
users.append(Session(record.user, record.line, record.host, record.sec))
|
||||
if record.type == utmp.UTmpRecordType.user_process:
|
||||
if record.host != '':
|
||||
users.append(Session(record.user, record.line, record.host, record.sec))
|
||||
else:
|
||||
users.append(Session(record.user, record.line, 'localhost', record.sec))
|
||||
return users
|
||||
|
||||
|
||||
|
||||
def parse_arguments() -> None:
|
||||
def handle_sessions_changed() -> None:
|
||||
"""
|
||||
This function fetches the current list of SSH sessions and compares it to the previous list of SSH sessions.
|
||||
If the number of sessions has changed, it increments or decrements the gauge_num_sessions metric
|
||||
and updates the session_data and num_sessions variables.
|
||||
"""
|
||||
global session_data, num_sessions, gauge_num_sessions, old_session_data, old_num_sessions
|
||||
|
||||
global FETCH_INTERVAL, SERVER_PORT, SERVER_HOST
|
||||
old_session_data = session_data
|
||||
old_num_sessions = len(old_session_data)
|
||||
|
||||
session_data = get_utmp_data()
|
||||
num_sessions = len(session_data)
|
||||
|
||||
for maybe_new_session in session_data:
|
||||
# Looking for newly found SSH sessions
|
||||
if not maybe_new_session in old_session_data:
|
||||
print("Session connected: %s" % maybe_new_session.from_)
|
||||
gauge_num_sessions.labels(remote_ip=maybe_new_session.from_).inc()
|
||||
|
||||
for maybe_old_session in old_session_data:
|
||||
# Looking for SSH sessions that no longer exist
|
||||
if not maybe_old_session in session_data:
|
||||
print("Session disconnected: %s" % maybe_old_session.from_)
|
||||
gauge_num_sessions.labels(remote_ip=maybe_old_session.from_).dec()
|
||||
|
||||
|
||||
|
||||
def parse_arguments() -> None:
|
||||
global FETCH_INTERVAL, SERVER_PORT, SERVER_HOST, WATCHFILE
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='python prometheus-ssh-exporter.py',
|
||||
|
|
@ -63,11 +103,15 @@ def parse_arguments() -> None:
|
|||
help='Port for the server to listen to')
|
||||
parser.add_argument('-i', '--interval', type=int, default=15,
|
||||
help='Interval in seconds to fetch SSH sessions data')
|
||||
parser.add_argument('-f', '--file', type=str, default='/var/run/utmp',
|
||||
help='File that changes every time a new SSH session is opened or closed')
|
||||
|
||||
args = parser.parse_args()
|
||||
FETCH_INTERVAL = args.interval
|
||||
SERVER_PORT = args.port
|
||||
SERVER_HOST = args.host
|
||||
WATCHFILE = args.file
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
@ -78,6 +122,7 @@ if __name__ == '__main__':
|
|||
or sum them up to get the total number of sessions.
|
||||
"""
|
||||
|
||||
|
||||
parse_arguments()
|
||||
|
||||
# Start up the server to expose the metrics.
|
||||
|
|
@ -86,35 +131,37 @@ if __name__ == '__main__':
|
|||
gauge_num_sessions = prometheus_client.Gauge(
|
||||
'ssh_num_sessions', 'Number of SSH sessions', ['remote_ip'])
|
||||
|
||||
# session_data contains the current list of sessions
|
||||
session_data = get_utmp_data()
|
||||
num_sessions = len(session_data)
|
||||
|
||||
# Initial metrics
|
||||
print("Active sessions at startup:")
|
||||
for session in session_data:
|
||||
gauge_num_sessions.labels(remote_ip=session.from_).inc()
|
||||
print("Initial connection: {}".format(session.from_))
|
||||
|
||||
|
||||
"""
|
||||
Start the watchdog to monitor the WATCHDOG file for changes.
|
||||
This is used to immediately look for changes in the SSH sessions when a new session is opened or closed
|
||||
to prevent missing any sessions that lasted less than the FETCH_INTERVAL.
|
||||
"""
|
||||
print("Watching file {} for changes...".format(WATCHFILE))
|
||||
event_handler = FileOpenedHandler()
|
||||
observer = Observer()
|
||||
observer.schedule(event_handler, path=WATCHFILE, recursive=False)
|
||||
observer.start()
|
||||
|
||||
# Generate some requests.
|
||||
print("Looking for SSH connection changes at interval {}".format(FETCH_INTERVAL))
|
||||
while True:
|
||||
try:
|
||||
|
||||
old_session_data = session_data
|
||||
old_num_sessions = len(old_session_data)
|
||||
while True:
|
||||
# Keep looking for changes in the SSH sessions in case the watchdog missed something
|
||||
handle_sessions_changed()
|
||||
time.sleep(FETCH_INTERVAL)
|
||||
|
||||
session_data = get_utmp_data()
|
||||
num_sessions = len(session_data)
|
||||
|
||||
for maybe_new_session in session_data:
|
||||
# Looking for newly found SSH sessions
|
||||
if not maybe_new_session in old_session_data:
|
||||
print("Session connected: %s" % maybe_new_session.from_)
|
||||
gauge_num_sessions.labels(remote_ip=maybe_new_session.from_).inc()
|
||||
|
||||
for maybe_old_session in old_session_data:
|
||||
# Looking for SSH sessions that no longer exist
|
||||
if not maybe_old_session in session_data:
|
||||
print("Session disconnected: %s" % maybe_old_session.from_)
|
||||
gauge_num_sessions.labels(remote_ip=maybe_old_session.from_).dec()
|
||||
|
||||
time.sleep(FETCH_INTERVAL)
|
||||
except:
|
||||
print("Terminating...")
|
||||
observer.stop()
|
||||
observer.join()
|
||||
|
|
@ -1,2 +1,3 @@
|
|||
prometheus_client==0.16.0
|
||||
utmp==21.10.0
|
||||
watchdog==3.0.0
|
||||
Loading…
Reference in a new issue