Bump version 1.5.0

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## 1.5.0 (2024/01/20)
+
+* Replaced RESTIC_REPO_URL, RESTIC_REPO_PASSWORD and RESTIC_REPO_PASSWORD_FILE environment variables with the Restic equivalents
+* Add new label "snapshot_tags" in the list of tags separated by comma. The label "snapshot_tag" only contains the first tag
+* Update Restic 0.16.3
+* Update Python dependencies
+* Update base Docker image to Alpine 3.19
+
+## 1.4.0 (2023/10/14)
+
+* Include metric label client_version. Resolves #5
+* Update Grafana dashboard to include repository locks and client version
+* Update Restic 0.16.0
+* Update Python 3.12
+
 ## 1.3.0 (2023/07/30)
 
 * Add new metric "restic_locks_total" with the number of repository locks

Dockerfile

@@ -1,6 +1,6 @@
-FROM golang:1.20-alpine3.18 AS builder
+FROM golang:1.20-alpine3.19 AS builder
 
-ENV RESTIC_VERSION 0.16.0
+ENV RESTIC_VERSION 0.16.3
 ENV CGO_ENABLED 0
 
 RUN cd /tmp \
@@ -12,7 +12,7 @@ RUN cd /tmp \
     # flag -ldflags "-s -w" produces a smaller executable
     && go build -ldflags "-s -w" -v -o /tmp/restic ./cmd/restic
 
-FROM python:3.11-alpine3.18
+FROM python:3.12-alpine3.19
 
 RUN apk add --no-cache --update openssh tzdata
 

README.md

@@ -19,8 +19,8 @@ Requirements:
 ```bash
 pip install -r /requirements.txt
 
-export RESTIC_REPO_URL=/data
-export RESTIC_REPO_PASSWORD_FILE=/restic_password_file
+export RESTIC_REPOSITORY=/data
+export RESTIC_PASSWORD_FILE=/restic_password_file
 python restic-exporter.py
 ```
 
@@ -59,9 +59,9 @@ services:
     container_name: restic-exporter
     environment:
       - TZ=Europe/Madrid
-      - RESTIC_REPO_URL=/data
-      - RESTIC_REPO_PASSWORD=<password_here>
-      # - RESTIC_REPO_PASSWORD_FILE=</file_with_password_here>
+      - RESTIC_REPOSITORY=/data
+      - RESTIC_PASSWORD=<password_here>
+      # - RESTIC_PASSWORD_FILE=</file_with_password_here>
       - REFRESH_INTERVAL=1800 # 30 min
     volumes:
       - /host_path/restic/data:/data
@@ -76,8 +76,8 @@ services:
 docker run -d \
   --name=restic-exporter \
   -e TZ=Europe/Madrid \
-  -e RESTIC_REPO_URL=/data \
-  -e RESTIC_REPO_PASSWORD=<password_here> \
+  -e RESTIC_REPOSITORY=/data \
+  -e RESTIC_PASSWORD=<password_here> \
   -e REFRESH_INTERVAL=1800 \
   -p 8001:8001 \
   --restart unless-stopped \
@@ -91,17 +91,17 @@ Some of them need additional environment variables for the secrets.
 
 All configuration is done with environment variables:
 
-- `RESTIC_REPO_URL`: Restic repository URL. All backends are supported. Examples:
+- `RESTIC_REPOSITORY`: Restic repository URL. All backends are supported. Examples:
   * Local repository: `/data`
   * REST Server: `rest:http://user:password@127.0.0.1:8000/`
   * Amazon S3: `s3:s3.amazonaws.com/bucket_name`
   * Backblaze B2: `b2:bucketname:path/to/repo`
   * Rclone (see notes below): `rclone:gd-backup:/restic`
 
-- `RESTIC_REPO_PASSWORD`: Restic repository password in plain text. This is only
-required if `RESTIC_REPO_PASSWORD_FILE` is not defined.
-- `RESTIC_REPO_PASSWORD_FILE`: File with the Restic repository password in plain
-text. This is only required if `RESTIC_REPO_PASSWORD` is not defined. Remember
+- `RESTIC_PASSWORD`: Restic repository password in plain text. This is only
+required if `RESTIC_PASSWORD_FILE` is not defined.
+- `RESTIC_PASSWORD_FILE`: File with the Restic repository password in plain
+text. This is only required if `RESTIC_PASSWORD` is not defined. Remember
 to mount the Docker volume with the file.
 - `AWS_ACCESS_KEY_ID`: (Optional) Required for Amazon S3, Minio and Wasabi
 backends.
@@ -138,8 +138,8 @@ services:
     container_name: restic-exporter
     environment:
       - TZ=Europe/Madrid
-      - RESTIC_REPO_URL=rclone:gd-backup:/restic
-      - RESTIC_REPO_PASSWORD= 
+      - RESTIC_REPOSITORY=rclone:gd-backup:/restic
+      - RESTIC_PASSWORD= 
       - REFRESH_INTERVAL=1800 # 30 min
     volumes:
       - /host_path/restic/data:/data
@@ -164,16 +164,16 @@ restic_locks_total 1.0
 restic_snapshots_total 100.0
 # HELP restic_backup_timestamp Timestamp of the last backup
 # TYPE restic_backup_timestamp gauge
-restic_backup_timestamp{client_hostname="product.example.com",client_username="root",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_paths="/mysql/data,/mysql/config"} 1.666273638e+09
+restic_backup_timestamp{client_hostname="product.example.com",client_username="root",client_version="restic 0.16.0",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_tags="mysql,tag2",snapshot_paths="/mysql/data,/mysql/config"} 1.666273638e+09
 # HELP restic_backup_files_total Number of files in the backup
 # TYPE restic_backup_files_total counter
-restic_backup_files_total{client_hostname="product.example.com",client_username="root",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_paths="/mysql/data,/mysql/config"} 8.0
+restic_backup_files_total{client_hostname="product.example.com",client_username="root",client_version="restic 0.16.0",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_tags="mysql,tag2",snapshot_paths="/mysql/data,/mysql/config"} 8.0
 # HELP restic_backup_size_total Total size of backup in bytes
 # TYPE restic_backup_size_total counter
-restic_backup_size_total{client_hostname="product.example.com",client_username="root",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_paths="/mysql/data,/mysql/config"} 4.3309562e+07
+restic_backup_size_total{client_hostname="product.example.com",client_username="root",client_version="restic 0.16.0",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_tags="mysql,tag2",snapshot_paths="/mysql/data,/mysql/config"} 4.3309562e+07
 # HELP restic_backup_snapshots_total Total number of snapshots
 # TYPE restic_backup_snapshots_total counter
-restic_backup_snapshots_total{client_hostname="product.example.com",client_username="root",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_paths="/mysql/data,/mysql/config"} 1.0
+restic_backup_snapshots_total{client_hostname="product.example.com",client_username="root",client_version="restic 0.16.0",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_tags="mysql,tag2",snapshot_paths="/mysql/data,/mysql/config"} 1.0
 # HELP restic_scrape_duration_seconds Amount of time each scrape takes
 # TYPE restic_scrape_duration_seconds gauge
 restic_scrape_duration_seconds 166.9411084651947

docker-compose.yml

@@ -6,9 +6,9 @@ services:
     container_name: restic-exporter
     environment:
       - TZ=Europe/Madrid
-      - RESTIC_REPO_URL=/data
-      - RESTIC_REPO_PASSWORD=password_here
-      # - RESTIC_REPO_PASSWORD_FILE=/file_with_password_here
+      - RESTIC_REPOSITORY=/data
+      - RESTIC_PASSWORD=password_here
+      # - RESTIC_PASSWORD_FILE=/file_with_password_here
       - REFRESH_INTERVAL=1800 # 30 min
     volumes:
       - /host_path/restic/data:/data

entrypoint.sh

@@ -3,14 +3,23 @@
 # Exit on error. For debug use set -x
 set -e
 
-if [ -z "${RESTIC_REPO_PASSWORD}" ]; then
-  if [ -z "${RESTIC_REPO_PASSWORD_FILE}" ]; then
-    echo "You have to define one of these environment variables: RESTIC_REPO_PASSWORD or RESTIC_REPO_PASSWORD_FILE"
+if [ -n "${RESTIC_REPO_PASSWORD}" ]; then
+  echo "The environment variable RESTIC_REPO_PASSWORD is deprecated, please use RESTIC_PASSWORD instead."
+  export RESTIC_PASSWORD="${RESTIC_REPO_PASSWORD}"
+fi
+if [ -n "${RESTIC_REPO_PASSWORD_FILE}" ]; then
+  echo "The environment variable RESTIC_REPO_PASSWORD_FILE is deprecated, please use RESTIC_PASSWORD_FILE instead."
+  export RESTIC_PASSWORD_FILE="${RESTIC_REPO_PASSWORD_FILE}"
+fi
+
+if [ -z "${RESTIC_PASSWORD}" ]; then
+  if [ -z "${RESTIC_PASSWORD_FILE}" ]; then
+    echo "You have to define one of these environment variables: RESTIC_PASSWORD or RESTIC_PASSWORD_FILE"
     exit 1
   fi
 else
-  export RESTIC_REPO_PASSWORD_FILE="/tmp/restic_passwd"
-  echo "${RESTIC_REPO_PASSWORD}" > "${RESTIC_REPO_PASSWORD_FILE}"
+  export RESTIC_PASSWORD_FILE="/tmp/restic_passwd"
+  echo "${RESTIC_PASSWORD}" > "${RESTIC_PASSWORD_FILE}"
 fi
 
 /usr/local/bin/python -u /restic-exporter.py

grafana/grafana_dashboard.json

@@ -132,7 +132,7 @@
       },
       "gridPos": {
         "h": 3,
-        "w": 24,
+        "w": 18,
         "x": 0,
         "y": 0
       },
@@ -169,6 +169,74 @@
       "title": "Repository Check",
       "type": "stat"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 2
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 42,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.3.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "restic_locks_total",
+          "interval": "",
+          "legendFormat": "",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Locks",
+      "type": "stat"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -254,7 +322,7 @@
       },
       "gridPos": {
         "h": 9,
-        "w": 15,
+        "w": 18,
         "x": 0,
         "y": 3
       },
@@ -328,7 +396,7 @@
               "client_id": true,
               "client_os_version": true,
               "client_username": true,
-              "client_version": true,
+              "client_version": false,
               "instance": true,
               "job": true,
               "snapshot_hash": true,
@@ -378,8 +446,8 @@
       },
       "gridPos": {
         "h": 9,
-        "w": 9,
-        "x": 15,
+        "w": 6,
+        "x": 18,
         "y": 3
       },
       "id": 32,
@@ -405,6 +473,11 @@
       "pluginVersion": "9.3.0",
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
           "exemplar": true,
           "expr": "restic_backup_snapshots_total",
           "format": "table",
@@ -412,11 +485,7 @@
           "interval": "",
           "intervalFactor": 1,
           "legendFormat": "{{client_hostname}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          }
+          "refId": "A"
         }
       ],
       "title": "Total snapshot count",
@@ -494,7 +563,7 @@
       },
       "gridPos": {
         "h": 9,
-        "w": 9,
+        "w": 18,
         "x": 0,
         "y": 12
       },
@@ -521,6 +590,10 @@
       "pluginVersion": "9.3.0",
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
           "exemplar": true,
           "expr": "restic_backup_size_total",
           "format": "table",
@@ -528,36 +601,71 @@
           "interval": "",
           "intervalFactor": 1,
           "legendFormat": "{{client_hostname}}",
-          "refId": "A",
+          "refId": "A"
+        },
+        {
           "datasource": {
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
-          }
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "restic_backup_files_total",
+          "format": "table",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{client_hostname}}",
+          "refId": "B"
         }
       ],
-      "title": "Total backup size",
+      "title": "Total backup size & files",
       "transformations": [
+        {
+          "id": "joinByField",
+          "options": {
+            "byField": "client_hostname",
+            "mode": "outer"
+          }
+        },
         {
           "id": "organize",
           "options": {
             "excludeByName": {
               "Time": true,
+              "Time 1": true,
+              "Time 2": true,
               "__name__": true,
+              "__name__ 1": true,
               "backup_id": true,
               "backup_type": true,
               "client_hostname": false,
               "client_id": true,
               "client_os_version": true,
               "client_username": true,
+              "client_username 1": true,
+              "client_username 2": true,
               "client_version": true,
+              "client_version 1": true,
+              "client_version 2": true,
               "instance": true,
+              "instance 1": true,
+              "instance 2": true,
               "job": true,
+              "job 1": true,
+              "job 2": true,
               "snapshot_hash": true,
+              "snapshot_hash 1": true,
+              "snapshot_hash 2": true,
               "snapshot_id": true
             },
             "indexByName": {},
             "renameByName": {
-              "Value": "total_backup_size"
+              "Time 2": "",
+              "Value": "total_backup_size",
+              "Value #A": "total_backup_size",
+              "Value #B": "total_backup_files"
             }
           }
         }
@@ -590,7 +698,7 @@
       "gridPos": {
         "h": 9,
         "w": 6,
-        "x": 9,
+        "x": 18,
         "y": 12
       },
       "id": 35,
@@ -661,109 +769,6 @@
       ],
       "type": "piechart"
     },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_PROMETHEUS}"
-      },
-      "description": "",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "custom": {
-            "align": "auto",
-            "displayMode": "auto",
-            "filterable": false,
-            "inspect": false
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "none"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 9,
-        "w": 9,
-        "x": 15,
-        "y": 12
-      },
-      "id": 33,
-      "links": [],
-      "maxDataPoints": 100,
-      "options": {
-        "footer": {
-          "fields": "",
-          "reducer": [
-            "sum"
-          ],
-          "show": false
-        },
-        "frameIndex": 1,
-        "showHeader": true,
-        "sortBy": [
-          {
-            "desc": false,
-            "displayName": "client_hostname"
-          }
-        ]
-      },
-      "pluginVersion": "9.3.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          },
-          "exemplar": true,
-          "expr": "restic_backup_files_total",
-          "format": "table",
-          "instant": true,
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "{{client_hostname}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Total backup files",
-      "transformations": [
-        {
-          "id": "organize",
-          "options": {
-            "excludeByName": {
-              "Time": true,
-              "__name__": true,
-              "backup_id": true,
-              "backup_type": true,
-              "client_hostname": false,
-              "client_id": true,
-              "client_os_version": true,
-              "client_username": true,
-              "client_version": true,
-              "instance": true,
-              "job": true,
-              "snapshot_hash": true,
-              "snapshot_id": true
-            },
-            "indexByName": {},
-            "renameByName": {
-              "Value": "total_backup_files"
-            }
-          }
-        }
-      ],
-      "type": "table"
-    },
     {
       "datasource": {
         "type": "prometheus",
@@ -1103,6 +1108,6 @@
   "timezone": "",
   "title": "Restic Exporter",
   "uid": "2JzZl3B7k",
-  "version": 23,
+  "version": 25,
   "weekStart": ""
 }

package.json

@@ -1,5 +1,5 @@
 {
   "name": "ngosang-restic-exporter",
-  "version": "1.3.0",
+  "version": "1.5.0",
   "author": "ngosang@hotmail.es"
 }

requirements.txt

@@ -1 +1 @@
-prometheus-client==0.17.1
+prometheus-client==0.19.0

restic-exporter.py

@@ -40,8 +40,10 @@ class ResticCollector(object):
         common_label_names = [
             "client_hostname",
             "client_username",
+            "client_version",
             "snapshot_hash",
             "snapshot_tag",
+            "snapshot_tags",
             "snapshot_paths",
         ]
 
@@ -94,8 +96,10 @@ class ResticCollector(object):
             common_label_values = [
                 client["hostname"],
                 client["username"],
+                client["version"],
                 client["snapshot_hash"],
                 client["snapshot_tag"],
+                client["snapshot_tags"],
                 client["snapshot_paths"],
             ]
 
@@ -179,8 +183,10 @@ class ResticCollector(object):
                 {
                     "hostname": snap["hostname"],
                     "username": snap["username"],
+                    "version": snap["program_version"] if "program_version" in snap else "",
                     "snapshot_hash": snap["hash"],
                     "snapshot_tag": snap["tags"][0] if "tags" in snap else "",
+                    "snapshot_tags": ",".join(snap["tags"]) if "tags" in snap else "",
                     "snapshot_paths": ",".join(snap["paths"]) if self.include_paths else "",
                     "timestamp": snap["timestamp"],
                     "size_total": stats["total_size"],
@@ -341,16 +347,28 @@ if __name__ == "__main__":
     logging.info("Starting Restic Prometheus Exporter")
     logging.info("It could take a while if the repository is remote")
 
-    try:
-        restic_repo_url = os.environ["RESTIC_REPO_URL"]
-    except Exception:
-        logging.error("The environment variable RESTIC_REPO_URL is mandatory")
+    restic_repo_url = os.environ.get("RESTIC_REPOSITORY")
+    if restic_repo_url is None:
+        restic_repo_url = os.environ.get("RESTIC_REPO_URL")
+        if restic_repo_url is not None:
+            logging.warning(
+                "The environment variable RESTIC_REPO_URL is deprecated, "
+                "please use RESTIC_REPOSITORY instead."
+            )
+    if restic_repo_url is None:
+        logging.error("The environment variable RESTIC_REPOSITORY is mandatory")
         sys.exit(1)
 
-    try:
-        restic_repo_password_file = os.environ["RESTIC_REPO_PASSWORD_FILE"]
-    except Exception:
-        logging.error("The environment variable RESTIC_REPO_PASSWORD_FILE is mandatory")
+    restic_repo_password_file = os.environ.get("RESTIC_PASSWORD_FILE")
+    if restic_repo_password_file is None:
+        restic_repo_password_file = os.environ.get("RESTIC_REPO_PASSWORD_FILE")
+        if restic_repo_password_file is not None:
+            logging.warning(
+                "The environment variable RESTIC_REPO_PASSWORD_FILE is deprecated, "
+                "please use RESTIC_PASSWORD_FILE instead."
+            )
+    if restic_repo_password_file is None:
+        logging.error("The environment variable RESTIC_PASSWORD_FILE is mandatory")
         sys.exit(1)
 
     exporter_address = os.environ.get("LISTEN_ADDRESS", "0.0.0.0")