From 091ee429f117c934496bf849672ecd8a43399d58 Mon Sep 17 00:00:00 2001 From: Pierre Verkest Date: Sun, 3 Sep 2023 21:52:19 +0200 Subject: [PATCH] Add role label in metrics I expecte to refactor and merge metrics together so later those labels will be required --- README.md | 28 +++++++++---- opnsense_exporter/opnsense_api.py | 23 ++++++++-- opnsense_exporter/server.py | 22 ++++++---- tests/test_opnsense_api.py | 43 +++++++++++++++---- tests/test_server.py | 70 +++++++++++++++++++++++-------- 5 files changed, 142 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 987968f..a2320e0 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ This exporter gives following metrics, all metrics received following labels: - `instance`: by default this is set with the hostname where is running this exporter service - `host`: the host of the OPNSense +- `role`: `main` or `backup` ### Enums @@ -81,25 +82,36 @@ You can setup env through `.env` file or environment variables with defined as d - **OPNSENSE_BACKUP_HOST**: default value for `--backup-host` param - **OPNSENSE_USERNAME**: default value for `--opnsense-user` param - **OPNSENSE_PASSWORD**: default value for `--opnsense-password` param +- **OPNSENSE_INTERFACES**: default value for `--opnsense-interfaces` param ## Roadmap +- merge `opnsense_active_server_bytes_received` and `opnsense_active_server_bytes_transmitted` + metrics adding labels to distinguish rates transmitted and rate received +- allow to configure interfaces to get traffic rates for lan,wan and/or other names +- refactor server in a class to avoid transmitted params over methods - allow to change the listening port (today it force using `8000`) +- allow to configure timeouts using environemnt variables +- improves logging to get a debug mode to understand errors based on unexpected payloads ## Changelog -### Version 0.4.0 +### Version 0.0.5 (UNRELEASED) -Higher timeout while getting WAN traffic info +* add role label in metrics -### Version 0.3.0 +### Version 0.4.0 (2023-09-02) -Use proper method to compute WAN traffic +* Higher timeout while getting WAN traffic info -### Version 0.2.0 +### Version 0.3.0 (2023-09-02) -Setup automatic release from gitlab while pushing new tag +* Use proper method to compute WAN traffic -### Version 0.1.0 +### Version 0.2.0 (2023-09-01) -Initial version +* Setup automatic release from gitlab while pushing new tag + +### Version 0.1.0 (2023-09-01) + +* Initial version diff --git a/opnsense_exporter/opnsense_api.py b/opnsense_exporter/opnsense_api.py index fd61843..9af7d74 100644 --- a/opnsense_exporter/opnsense_api.py +++ b/opnsense_exporter/opnsense_api.py @@ -1,4 +1,5 @@ import logging +from enum import Enum import requests from requests import RequestException @@ -6,16 +7,30 @@ from requests import RequestException logger = logging.getLogger(__name__) +class OPNSenseRole(Enum): + MAIN = "main" + BACKUP = "backup" + + class OPNSenseAPI: host: str = None login: str = None password: str = None + role: OPNSenseRole = None - def __init__(self, host, login, password): + def __init__(self, role, host, login, password): + self.role = role self.host = host self.login = login self.password = password + @property + def labels(self): + return { + "host": self.host, + "role": self.role.value, + } + def prepare_url(self, path): return f"https://{self.host}{path}" @@ -65,7 +80,7 @@ class OPNSenseAPI: received = 0 transmitted = 0 - for record in data["wan"]["records"]: - received += record["rate_bits_in"] - transmitted += record["rate_bits_out"] + for record in data.get("wan", []).get("records", []): + received += record.get("rate_bits_in", 0) + transmitted += record.get("rate_bits_out", 0) return received, transmitted diff --git a/opnsense_exporter/server.py b/opnsense_exporter/server.py index 2949b43..4339690 100644 --- a/opnsense_exporter/server.py +++ b/opnsense_exporter/server.py @@ -6,7 +6,7 @@ import time from dotenv import load_dotenv from prometheus_client import Enum, Gauge, start_http_server -from opnsense_exporter.opnsense_api import OPNSenseAPI +from opnsense_exporter.opnsense_api import OPNSenseAPI, OPNSenseRole load_dotenv() @@ -17,6 +17,7 @@ main_ha_state = Enum( [ "instance", "host", + "role", ], states=HA_STATES, ) @@ -26,6 +27,7 @@ backup_ha_state = Enum( [ "instance", "host", + "role", ], states=HA_STATES, ) @@ -35,6 +37,7 @@ active_server_bytes_received = Gauge( [ "instance", "host", + "role", ], ) active_server_bytes_transmitted = Gauge( @@ -43,6 +46,7 @@ active_server_bytes_transmitted = Gauge( [ "instance", "host", + "role", ], ) @@ -51,8 +55,8 @@ def process_requests(main, backup, exporter_instance: str = ""): """A dummy function that takes some time.""" main_state = main.get_interface_vip_status() backup_sate = backup.get_interface_vip_status() - main_ha_state.labels(instance=exporter_instance, host=main.host).state(main_state) - backup_ha_state.labels(instance=exporter_instance, host=backup.host).state( + main_ha_state.labels(instance=exporter_instance, **main.labels).state(main_state) + backup_ha_state.labels(instance=exporter_instance, **backup.labels).state( backup_sate ) active_opnsense = None @@ -64,11 +68,11 @@ def process_requests(main, backup, exporter_instance: str = ""): bytes_received, bytes_transmitted = active_opnsense.get_wan_trafic() if bytes_received or bytes_received == 0: active_server_bytes_received.labels( - instance=exporter_instance, host=active_opnsense.host + instance=exporter_instance, **active_opnsense.labels ).set(bytes_received) if bytes_transmitted or bytes_transmitted == 0: active_server_bytes_transmitted.labels( - instance=exporter_instance, host=active_opnsense.host + instance=exporter_instance, **active_opnsense.labels ).set(bytes_transmitted) @@ -144,8 +148,12 @@ def run(): arguments = parser.parse_args() start_server( - OPNSenseAPI(arguments.main, arguments.user, arguments.password), - OPNSenseAPI(arguments.backup, arguments.user, arguments.password), + OPNSenseAPI( + OPNSenseRole.MAIN, arguments.main, arguments.user, arguments.password + ), + OPNSenseAPI( + OPNSenseRole.BACKUP, arguments.backup, arguments.user, arguments.password + ), check_frequency=arguments.frequency, exporter_instance=arguments.prom_instance, ) diff --git a/tests/test_opnsense_api.py b/tests/test_opnsense_api.py index 798a3bb..6ec0693 100644 --- a/tests/test_opnsense_api.py +++ b/tests/test_opnsense_api.py @@ -1,8 +1,9 @@ import responses -from opnsense_exporter.opnsense_api import OPNSenseAPI +from opnsense_exporter.opnsense_api import OPNSenseAPI, OPNSenseRole from .common import ( + BACKUP_HOST, LOGIN, MAIN_HOST, PASSWORD, @@ -20,7 +21,10 @@ def test_get_interface_vip_status_active(): ) assert ( - OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_interface_vip_status() == "active" + OPNSenseAPI( + OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD + ).get_interface_vip_status() + == "active" ) @@ -33,7 +37,9 @@ def test_get_interface_vip_status_backup(): ) assert ( - OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_interface_vip_status() + OPNSenseAPI( + OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD + ).get_interface_vip_status() == "hot_standby" ) @@ -47,7 +53,9 @@ def test_get_interface_vip_status_mainteance_mode(): ) assert ( - OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_interface_vip_status() + OPNSenseAPI( + OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD + ).get_interface_vip_status() == "maintenancemode" ) @@ -60,7 +68,9 @@ def test_get_interface_vip_status_unavailable_weird_case(): body=generate_get_vip_status_paylaod("MASTER", "BACKUP", False), ) assert ( - OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_interface_vip_status() + OPNSenseAPI( + OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD + ).get_interface_vip_status() == "unavailable" ) @@ -74,7 +84,9 @@ def test_get_interface_vip_status_unavailable_rest_api_error(): status=404, ) assert ( - OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_interface_vip_status() + OPNSenseAPI( + OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD + ).get_interface_vip_status() == "unavailable" ) @@ -86,7 +98,9 @@ def test_get_wan_traffic(): f"https://{MAIN_HOST}/api/diagnostics/traffic/top/wan", body=generate_diagnostics_traffic_interface_paylaod(), ) - assert OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_wan_trafic() == ( + assert OPNSenseAPI( + OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD + ).get_wan_trafic() == ( 20538, 10034, ) @@ -100,7 +114,20 @@ def test_get_wan_traffic_none(): json={"error": "not found"}, status=404, ) - assert OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_wan_trafic() == ( + assert OPNSenseAPI( + OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD + ).get_wan_trafic() == ( None, None, ) + + +def test_labels(): + assert OPNSenseAPI(OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD).labels == { + "role": "main", + "host": MAIN_HOST, + } + assert OPNSenseAPI(OPNSenseRole.BACKUP, BACKUP_HOST, LOGIN, PASSWORD).labels == { + "role": "backup", + "host": BACKUP_HOST, + } diff --git a/tests/test_server.py b/tests/test_server.py index 3b206ac..2774776 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -2,7 +2,7 @@ from unittest import mock import responses -from opnsense_exporter.opnsense_api import OPNSenseAPI +from opnsense_exporter.opnsense_api import OPNSenseAPI, OPNSenseRole from opnsense_exporter.server import process_requests, run from .common import ( @@ -109,23 +109,32 @@ def test_process_requests(): new=active_server_bytes_transmitted_mock, ): process_requests( - OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD), - OPNSenseAPI(BACKUP_HOST, LOGIN, PASSWORD), + OPNSenseAPI(OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD), + OPNSenseAPI(OPNSenseRole.BACKUP, BACKUP_HOST, LOGIN, PASSWORD), ) assert main_ha_state_mock._state == "active" assert main_ha_state_mock.count_state_calls == 1 - assert main_ha_state_mock._labels == {"instance": "", "host": MAIN_HOST} + assert main_ha_state_mock._labels == { + "instance": "", + "host": MAIN_HOST, + "role": "main", + } assert backup_ha_state_mock._state == "hot_standby" assert backup_ha_state_mock.count_state_calls == 1 - assert backup_ha_state_mock._labels == {"instance": "", "host": BACKUP_HOST} + assert backup_ha_state_mock._labels == { + "instance": "", + "host": BACKUP_HOST, + "role": "backup", + } assert active_server_bytes_received_mock.value == 20538 assert active_server_bytes_received_mock.count_set_calls == 1 assert active_server_bytes_received_mock._labels == { "instance": "", "host": MAIN_HOST, + "role": "main", } assert active_server_bytes_transmitted_mock.value == 10034 @@ -133,6 +142,7 @@ def test_process_requests(): assert active_server_bytes_transmitted_mock._labels == { "instance": "", "host": MAIN_HOST, + "role": "main", } @@ -172,22 +182,31 @@ def test_process_requests_backup_active(): new=active_server_bytes_transmitted_mock, ): process_requests( - OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD), - OPNSenseAPI(BACKUP_HOST, LOGIN, PASSWORD), + OPNSenseAPI(OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD), + OPNSenseAPI(OPNSenseRole.BACKUP, BACKUP_HOST, LOGIN, PASSWORD), ) assert main_ha_state_mock._state == "maintenancemode" assert main_ha_state_mock.count_state_calls == 1 - assert main_ha_state_mock._labels == {"instance": "", "host": MAIN_HOST} + assert main_ha_state_mock._labels == { + "instance": "", + "host": MAIN_HOST, + "role": "main", + } assert backup_ha_state_mock._state == "active" assert backup_ha_state_mock.count_state_calls == 1 - assert backup_ha_state_mock._labels == {"instance": "", "host": BACKUP_HOST} + assert backup_ha_state_mock._labels == { + "instance": "", + "host": BACKUP_HOST, + "role": "backup", + } assert active_server_bytes_received_mock.value == 20538 assert active_server_bytes_received_mock.count_set_calls == 1 assert active_server_bytes_received_mock._labels == { "instance": "", "host": BACKUP_HOST, + "role": "backup", } assert active_server_bytes_transmitted_mock.value == 10034 @@ -195,6 +214,7 @@ def test_process_requests_backup_active(): assert active_server_bytes_transmitted_mock._labels == { "instance": "", "host": BACKUP_HOST, + "role": "backup", } @@ -235,17 +255,25 @@ def test_process_no_active(): new=active_server_bytes_transmitted_mock, ): process_requests( - OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD), - OPNSenseAPI(BACKUP_HOST, LOGIN, PASSWORD), + OPNSenseAPI(OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD), + OPNSenseAPI(OPNSenseRole.BACKUP, BACKUP_HOST, LOGIN, PASSWORD), ) assert main_ha_state_mock._state == "maintenancemode" assert main_ha_state_mock.count_state_calls == 1 - assert main_ha_state_mock._labels == {"instance": "", "host": MAIN_HOST} + assert main_ha_state_mock._labels == { + "instance": "", + "host": MAIN_HOST, + "role": "main", + } assert backup_ha_state_mock._state == "unavailable" assert backup_ha_state_mock.count_state_calls == 1 - assert backup_ha_state_mock._labels == {"instance": "", "host": BACKUP_HOST} + assert backup_ha_state_mock._labels == { + "instance": "", + "host": BACKUP_HOST, + "role": "backup", + } assert active_server_bytes_received_mock.count_set_calls == 0 assert active_server_bytes_transmitted_mock.count_set_calls == 0 @@ -288,16 +316,24 @@ def test_process_with_falsy_value(): new=active_server_bytes_transmitted_mock, ): process_requests( - OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD), - OPNSenseAPI(BACKUP_HOST, LOGIN, PASSWORD), + OPNSenseAPI(OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD), + OPNSenseAPI(OPNSenseRole.BACKUP, BACKUP_HOST, LOGIN, PASSWORD), ) assert main_ha_state_mock._state == "active" assert main_ha_state_mock.count_state_calls == 1 - assert main_ha_state_mock._labels == {"instance": "", "host": MAIN_HOST} + assert main_ha_state_mock._labels == { + "instance": "", + "host": MAIN_HOST, + "role": "main", + } assert backup_ha_state_mock.count_state_calls == 1 assert backup_ha_state_mock._state == "hot_standby" - assert backup_ha_state_mock._labels == {"instance": "", "host": BACKUP_HOST} + assert backup_ha_state_mock._labels == { + "instance": "", + "host": BACKUP_HOST, + "role": "backup", + } assert active_server_bytes_received_mock.count_set_calls == 0 assert active_server_bytes_transmitted_mock.count_set_calls == 0