Add role label in metrics

I expecte to refactor and merge metrics together so
later those labels will be required

README.md

@@ -26,6 +26,7 @@ This exporter gives following metrics, all metrics received following labels:
 
 - `instance`: by default this is set with the hostname where is running this exporter service
 - `host`: the host of the OPNSense
+- `role`: `main` or `backup`
 
 ### Enums
 
@@ -81,25 +82,36 @@ You can setup env through `.env` file or environment variables with defined as d
 - **OPNSENSE_BACKUP_HOST**: default value for `--backup-host` param
 - **OPNSENSE_USERNAME**: default value for `--opnsense-user` param
 - **OPNSENSE_PASSWORD**: default value for `--opnsense-password` param
+- **OPNSENSE_INTERFACES**: default value for `--opnsense-interfaces` param
 
 ## Roadmap
 
+- merge `opnsense_active_server_bytes_received` and `opnsense_active_server_bytes_transmitted`
+  metrics adding labels to distinguish rates transmitted and rate received
+- allow to configure interfaces to get traffic rates for lan,wan and/or other names
+- refactor server in a class to avoid transmitted params over methods
 - allow to change the listening port (today it force using `8000`)
+- allow to configure timeouts using environemnt variables
+- improves logging to get a debug mode to understand errors based on unexpected payloads
 
 ## Changelog
 
-### Version 0.4.0
+### Version 0.0.5 (UNRELEASED)
 
-Higher timeout while getting WAN traffic info
+* add role label in metrics
 
-### Version 0.3.0
+### Version 0.4.0 (2023-09-02)
 
-Use proper method to compute WAN traffic
+* Higher timeout while getting WAN traffic info
 
-### Version 0.2.0
+### Version 0.3.0 (2023-09-02)
 
-Setup automatic release from gitlab while pushing new tag
+* Use proper method to compute WAN traffic
 
-### Version 0.1.0
+### Version 0.2.0 (2023-09-01)
 
-Initial version
+* Setup automatic release from gitlab while pushing new tag
+
+### Version 0.1.0 (2023-09-01)
+
+* Initial version

opnsense_exporter/opnsense_api.py

@@ -1,4 +1,5 @@
 import logging
+from enum import Enum
 
 import requests
 from requests import RequestException
@@ -6,16 +7,30 @@ from requests import RequestException
 logger = logging.getLogger(__name__)
 
 
+class OPNSenseRole(Enum):
+    MAIN = "main"
+    BACKUP = "backup"
+
+
 class OPNSenseAPI:
     host: str = None
     login: str = None
     password: str = None
+    role: OPNSenseRole = None
 
-    def __init__(self, host, login, password):
+    def __init__(self, role, host, login, password):
+        self.role = role
         self.host = host
         self.login = login
         self.password = password
 
+    @property
+    def labels(self):
+        return {
+            "host": self.host,
+            "role": self.role.value,
+        }
+
     def prepare_url(self, path):
         return f"https://{self.host}{path}"
 
@@ -65,7 +80,7 @@ class OPNSenseAPI:
 
         received = 0
         transmitted = 0
-        for record in data["wan"]["records"]:
+        for record in data.get("wan", []).get("records", []):
-            received += record["rate_bits_in"]
+            received += record.get("rate_bits_in", 0)
-            transmitted += record["rate_bits_out"]
+            transmitted += record.get("rate_bits_out", 0)
         return received, transmitted

opnsense_exporter/server.py

@@ -6,7 +6,7 @@ import time
 from dotenv import load_dotenv
 from prometheus_client import Enum, Gauge, start_http_server
 
-from opnsense_exporter.opnsense_api import OPNSenseAPI
+from opnsense_exporter.opnsense_api import OPNSenseAPI, OPNSenseRole
 
 load_dotenv()
 
@@ -17,6 +17,7 @@ main_ha_state = Enum(
     [
         "instance",
         "host",
+        "role",
     ],
     states=HA_STATES,
 )
@@ -26,6 +27,7 @@ backup_ha_state = Enum(
     [
         "instance",
         "host",
+        "role",
     ],
     states=HA_STATES,
 )
@@ -35,6 +37,7 @@ active_server_bytes_received = Gauge(
     [
         "instance",
         "host",
+        "role",
     ],
 )
 active_server_bytes_transmitted = Gauge(
@@ -43,6 +46,7 @@ active_server_bytes_transmitted = Gauge(
     [
         "instance",
         "host",
+        "role",
     ],
 )
 
@@ -51,8 +55,8 @@ def process_requests(main, backup, exporter_instance: str = ""):
     """A dummy function that takes some time."""
     main_state = main.get_interface_vip_status()
     backup_sate = backup.get_interface_vip_status()
-    main_ha_state.labels(instance=exporter_instance, host=main.host).state(main_state)
+    main_ha_state.labels(instance=exporter_instance, **main.labels).state(main_state)
-    backup_ha_state.labels(instance=exporter_instance, host=backup.host).state(
+    backup_ha_state.labels(instance=exporter_instance, **backup.labels).state(
         backup_sate
     )
     active_opnsense = None
@@ -64,11 +68,11 @@ def process_requests(main, backup, exporter_instance: str = ""):
         bytes_received, bytes_transmitted = active_opnsense.get_wan_trafic()
         if bytes_received or bytes_received == 0:
             active_server_bytes_received.labels(
-                instance=exporter_instance, host=active_opnsense.host
+                instance=exporter_instance, **active_opnsense.labels
             ).set(bytes_received)
         if bytes_transmitted or bytes_transmitted == 0:
             active_server_bytes_transmitted.labels(
-                instance=exporter_instance, host=active_opnsense.host
+                instance=exporter_instance, **active_opnsense.labels
             ).set(bytes_transmitted)
 
 
@@ -144,8 +148,12 @@ def run():
 
     arguments = parser.parse_args()
     start_server(
-        OPNSenseAPI(arguments.main, arguments.user, arguments.password),
+        OPNSenseAPI(
-        OPNSenseAPI(arguments.backup, arguments.user, arguments.password),
+            OPNSenseRole.MAIN, arguments.main, arguments.user, arguments.password
+        ),
+        OPNSenseAPI(
+            OPNSenseRole.BACKUP, arguments.backup, arguments.user, arguments.password
+        ),
         check_frequency=arguments.frequency,
         exporter_instance=arguments.prom_instance,
     )

tests/test_opnsense_api.py

@@ -1,8 +1,9 @@
 import responses
 
-from opnsense_exporter.opnsense_api import OPNSenseAPI
+from opnsense_exporter.opnsense_api import OPNSenseAPI, OPNSenseRole
 
 from .common import (
+    BACKUP_HOST,
     LOGIN,
     MAIN_HOST,
     PASSWORD,
@@ -20,7 +21,10 @@ def test_get_interface_vip_status_active():
     )
 
     assert (
-        OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_interface_vip_status() == "active"
+        OPNSenseAPI(
+            OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD
+        ).get_interface_vip_status()
+        == "active"
     )
 
 
@@ -33,7 +37,9 @@ def test_get_interface_vip_status_backup():
     )
 
     assert (
-        OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_interface_vip_status()
+        OPNSenseAPI(
+            OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD
+        ).get_interface_vip_status()
         == "hot_standby"
     )
 
@@ -47,7 +53,9 @@ def test_get_interface_vip_status_mainteance_mode():
     )
 
     assert (
-        OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_interface_vip_status()
+        OPNSenseAPI(
+            OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD
+        ).get_interface_vip_status()
         == "maintenancemode"
     )
 
@@ -60,7 +68,9 @@ def test_get_interface_vip_status_unavailable_weird_case():
         body=generate_get_vip_status_paylaod("MASTER", "BACKUP", False),
     )
     assert (
-        OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_interface_vip_status()
+        OPNSenseAPI(
+            OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD
+        ).get_interface_vip_status()
         == "unavailable"
     )
 
@@ -74,7 +84,9 @@ def test_get_interface_vip_status_unavailable_rest_api_error():
         status=404,
     )
     assert (
-        OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_interface_vip_status()
+        OPNSenseAPI(
+            OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD
+        ).get_interface_vip_status()
         == "unavailable"
     )
 
@@ -86,7 +98,9 @@ def test_get_wan_traffic():
         f"https://{MAIN_HOST}/api/diagnostics/traffic/top/wan",
         body=generate_diagnostics_traffic_interface_paylaod(),
     )
-    assert OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_wan_trafic() == (
+    assert OPNSenseAPI(
+        OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD
+    ).get_wan_trafic() == (
         20538,
         10034,
     )
@@ -100,7 +114,20 @@ def test_get_wan_traffic_none():
         json={"error": "not found"},
         status=404,
     )
-    assert OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD).get_wan_trafic() == (
+    assert OPNSenseAPI(
+        OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD
+    ).get_wan_trafic() == (
         None,
         None,
     )
+
+
+def test_labels():
+    assert OPNSenseAPI(OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD).labels == {
+        "role": "main",
+        "host": MAIN_HOST,
+    }
+    assert OPNSenseAPI(OPNSenseRole.BACKUP, BACKUP_HOST, LOGIN, PASSWORD).labels == {
+        "role": "backup",
+        "host": BACKUP_HOST,
+    }

tests/test_server.py

@@ -2,7 +2,7 @@ from unittest import mock
 
 import responses
 
-from opnsense_exporter.opnsense_api import OPNSenseAPI
+from opnsense_exporter.opnsense_api import OPNSenseAPI, OPNSenseRole
 from opnsense_exporter.server import process_requests, run
 
 from .common import (
@@ -109,23 +109,32 @@ def test_process_requests():
                     new=active_server_bytes_transmitted_mock,
                 ):
                     process_requests(
-                        OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD),
+                        OPNSenseAPI(OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD),
-                        OPNSenseAPI(BACKUP_HOST, LOGIN, PASSWORD),
+                        OPNSenseAPI(OPNSenseRole.BACKUP, BACKUP_HOST, LOGIN, PASSWORD),
                     )
 
     assert main_ha_state_mock._state == "active"
     assert main_ha_state_mock.count_state_calls == 1
-    assert main_ha_state_mock._labels == {"instance": "", "host": MAIN_HOST}
+    assert main_ha_state_mock._labels == {
+        "instance": "",
+        "host": MAIN_HOST,
+        "role": "main",
+    }
 
     assert backup_ha_state_mock._state == "hot_standby"
     assert backup_ha_state_mock.count_state_calls == 1
-    assert backup_ha_state_mock._labels == {"instance": "", "host": BACKUP_HOST}
+    assert backup_ha_state_mock._labels == {
+        "instance": "",
+        "host": BACKUP_HOST,
+        "role": "backup",
+    }
 
     assert active_server_bytes_received_mock.value == 20538
     assert active_server_bytes_received_mock.count_set_calls == 1
     assert active_server_bytes_received_mock._labels == {
         "instance": "",
         "host": MAIN_HOST,
+        "role": "main",
     }
 
     assert active_server_bytes_transmitted_mock.value == 10034
@@ -133,6 +142,7 @@ def test_process_requests():
     assert active_server_bytes_transmitted_mock._labels == {
         "instance": "",
         "host": MAIN_HOST,
+        "role": "main",
     }
 
 
@@ -172,22 +182,31 @@ def test_process_requests_backup_active():
                     new=active_server_bytes_transmitted_mock,
                 ):
                     process_requests(
-                        OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD),
+                        OPNSenseAPI(OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD),
-                        OPNSenseAPI(BACKUP_HOST, LOGIN, PASSWORD),
+                        OPNSenseAPI(OPNSenseRole.BACKUP, BACKUP_HOST, LOGIN, PASSWORD),
                     )
     assert main_ha_state_mock._state == "maintenancemode"
     assert main_ha_state_mock.count_state_calls == 1
-    assert main_ha_state_mock._labels == {"instance": "", "host": MAIN_HOST}
+    assert main_ha_state_mock._labels == {
+        "instance": "",
+        "host": MAIN_HOST,
+        "role": "main",
+    }
 
     assert backup_ha_state_mock._state == "active"
     assert backup_ha_state_mock.count_state_calls == 1
-    assert backup_ha_state_mock._labels == {"instance": "", "host": BACKUP_HOST}
+    assert backup_ha_state_mock._labels == {
+        "instance": "",
+        "host": BACKUP_HOST,
+        "role": "backup",
+    }
 
     assert active_server_bytes_received_mock.value == 20538
     assert active_server_bytes_received_mock.count_set_calls == 1
     assert active_server_bytes_received_mock._labels == {
         "instance": "",
         "host": BACKUP_HOST,
+        "role": "backup",
     }
 
     assert active_server_bytes_transmitted_mock.value == 10034
@@ -195,6 +214,7 @@ def test_process_requests_backup_active():
     assert active_server_bytes_transmitted_mock._labels == {
         "instance": "",
         "host": BACKUP_HOST,
+        "role": "backup",
     }
 
 
@@ -235,17 +255,25 @@ def test_process_no_active():
                     new=active_server_bytes_transmitted_mock,
                 ):
                     process_requests(
-                        OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD),
+                        OPNSenseAPI(OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD),
-                        OPNSenseAPI(BACKUP_HOST, LOGIN, PASSWORD),
+                        OPNSenseAPI(OPNSenseRole.BACKUP, BACKUP_HOST, LOGIN, PASSWORD),
                     )
 
     assert main_ha_state_mock._state == "maintenancemode"
     assert main_ha_state_mock.count_state_calls == 1
-    assert main_ha_state_mock._labels == {"instance": "", "host": MAIN_HOST}
+    assert main_ha_state_mock._labels == {
+        "instance": "",
+        "host": MAIN_HOST,
+        "role": "main",
+    }
 
     assert backup_ha_state_mock._state == "unavailable"
     assert backup_ha_state_mock.count_state_calls == 1
-    assert backup_ha_state_mock._labels == {"instance": "", "host": BACKUP_HOST}
+    assert backup_ha_state_mock._labels == {
+        "instance": "",
+        "host": BACKUP_HOST,
+        "role": "backup",
+    }
 
     assert active_server_bytes_received_mock.count_set_calls == 0
     assert active_server_bytes_transmitted_mock.count_set_calls == 0
@@ -288,16 +316,24 @@ def test_process_with_falsy_value():
                     new=active_server_bytes_transmitted_mock,
                 ):
                     process_requests(
-                        OPNSenseAPI(MAIN_HOST, LOGIN, PASSWORD),
+                        OPNSenseAPI(OPNSenseRole.MAIN, MAIN_HOST, LOGIN, PASSWORD),
-                        OPNSenseAPI(BACKUP_HOST, LOGIN, PASSWORD),
+                        OPNSenseAPI(OPNSenseRole.BACKUP, BACKUP_HOST, LOGIN, PASSWORD),
                     )
     assert main_ha_state_mock._state == "active"
     assert main_ha_state_mock.count_state_calls == 1
-    assert main_ha_state_mock._labels == {"instance": "", "host": MAIN_HOST}
+    assert main_ha_state_mock._labels == {
+        "instance": "",
+        "host": MAIN_HOST,
+        "role": "main",
+    }
 
     assert backup_ha_state_mock.count_state_calls == 1
     assert backup_ha_state_mock._state == "hot_standby"
-    assert backup_ha_state_mock._labels == {"instance": "", "host": BACKUP_HOST}
+    assert backup_ha_state_mock._labels == {
+        "instance": "",
+        "host": BACKUP_HOST,
+        "role": "backup",
+    }
 
     assert active_server_bytes_received_mock.count_set_calls == 0
     assert active_server_bytes_transmitted_mock.count_set_calls == 0