Compare commits

...

22 commits
1.2.1 ... main

Author SHA1 Message Date
ngosang
cb89566b67 Bump version 1.5.0 2024-01-20 16:04:50 +01:00
ngosang
4efcaba7c5 Replaced RESTIC_REPO_URL, RESTIC_REPO_PASSWORD and RESTIC_REPO_PASSWORD_FILE environment variables 2024-01-20 16:00:11 +01:00
ngosang
10dae8da3a Add new label "snapshot_tags" in the list of tags separated by comma 2024-01-20 15:28:45 +01:00
ngosang
71fb2a6695 Update Python dependencies 2024-01-20 15:17:44 +01:00
ngosang
e94816eb54 Update Restic 0.16.3 2024-01-20 15:15:42 +01:00
ngosang
c882d9a9a9 Update base Docker image to Alpine 3.19 2024-01-20 15:14:47 +01:00
ngosang
9c5e0d0071 Bump version 1.4.0 2023-10-14 23:38:09 +02:00
ngosang
9070dc9aef Update Grafana dashboard to include repository locks and client version 2023-10-14 23:36:10 +02:00
ngosang
a4243a8554 Include metric label client_version. Resolves #5 2023-10-14 23:35:18 +02:00
ngosang
d90f46a4d1 Update Python 3.12 2023-10-14 22:48:58 +02:00
ngosang
d26e181f21 Update Restic 0.16.0 2023-10-14 22:47:05 +02:00
ngosang
e1973cb48f Update changelog 2023-07-30 01:30:03 +02:00
ngosang
539582566d Bump version 1.3.0 2023-07-30 01:15:42 +02:00
ngosang
8697539748 Include backup paths in the exported metrics. Resolves #17 2023-07-30 01:02:04 +02:00
ngosang
2f4bab16ef Fix typos and static methods 2023-07-30 00:46:09 +02:00
ngosang
0bc9a62563 New metric restic_locks_total. Resolves #10 2023-07-30 00:37:32 +02:00
ngosang
3e183cbf83 Add Rclone instructions in the readme 2023-07-29 23:51:17 +02:00
ngosang
8463e245c2 Update Python dependencies 2023-07-29 23:38:15 +02:00
ngosang
d63604f1b8 Update Restic 0.15.2 2023-07-29 23:36:17 +02:00
ngosang
9621db9e79 Update base Docker image to Alpine 3.18 2023-07-29 23:35:21 +02:00
ngosang
a4e37f8770 Bump version 1.2.2 2023-03-31 15:31:51 +02:00
ngosang
c80b74c1ec Include OpenSSH in the Docker image to support SFTP protocol 2023-03-31 15:30:55 +02:00
10 changed files with 303 additions and 171 deletions

View file

@ -1,5 +1,35 @@
# Changelog # Changelog
## 1.5.0 (2024/01/20)
* Replaced RESTIC_REPO_URL, RESTIC_REPO_PASSWORD and RESTIC_REPO_PASSWORD_FILE environment variables with the Restic equivalents
* Add new label "snapshot_tags" in the list of tags separated by comma. The label "snapshot_tag" only contains the first tag
* Update Restic 0.16.3
* Update Python dependencies
* Update base Docker image to Alpine 3.19
## 1.4.0 (2023/10/14)
* Include metric label client_version. Resolves #5
* Update Grafana dashboard to include repository locks and client version
* Update Restic 0.16.0
* Update Python 3.12
## 1.3.0 (2023/07/30)
* Add new metric "restic_locks_total" with the number of repository locks
* Add new label "snapshot_paths" in the metrics with the backup paths
* Add NO_LOCKS env var to skip restic locks collection
* Add INCLUDE_PATHS env var to include the backup paths in the metrics
* Add Rclone instructions in the readme
* Update Restic 0.15.2
* Update Python dependencies
* Update base Docker image to Alpine 3.18
## 1.2.2 (2023/03/31)
* Include OpenSSH in the Docker image to support SFTP protocol
## 1.2.1 (2023/03/26) ## 1.2.1 (2023/03/26)
* Improve hash calculation to avoid duplicate clients (snapshot_hash label changes) * Improve hash calculation to avoid duplicate clients (snapshot_hash label changes)

View file

@ -1,6 +1,6 @@
FROM golang:1.19-alpine3.17 AS builder FROM golang:1.20-alpine3.19 AS builder
ENV RESTIC_VERSION 0.15.1 ENV RESTIC_VERSION 0.16.3
ENV CGO_ENABLED 0 ENV CGO_ENABLED 0
RUN cd /tmp \ RUN cd /tmp \
@ -12,9 +12,9 @@ RUN cd /tmp \
# flag -ldflags "-s -w" produces a smaller executable # flag -ldflags "-s -w" produces a smaller executable
&& go build -ldflags "-s -w" -v -o /tmp/restic ./cmd/restic && go build -ldflags "-s -w" -v -o /tmp/restic ./cmd/restic
FROM python:3.11-alpine3.17 FROM python:3.12-alpine3.19
RUN apk add --no-cache --update tzdata RUN apk add --no-cache --update openssh tzdata
COPY --from=builder /tmp/restic /usr/bin COPY --from=builder /tmp/restic /usr/bin
COPY entrypoint.sh requirements.txt / COPY entrypoint.sh requirements.txt /

View file

@ -19,8 +19,8 @@ Requirements:
```bash ```bash
pip install -r /requirements.txt pip install -r /requirements.txt
export RESTIC_REPO_URL=/data export RESTIC_REPOSITORY=/data
export RESTIC_REPO_PASSWORD_FILE=/restic_password_file export RESTIC_PASSWORD_FILE=/restic_password_file
python restic-exporter.py python restic-exporter.py
``` ```
@ -59,9 +59,9 @@ services:
container_name: restic-exporter container_name: restic-exporter
environment: environment:
- TZ=Europe/Madrid - TZ=Europe/Madrid
- RESTIC_REPO_URL=/data - RESTIC_REPOSITORY=/data
- RESTIC_REPO_PASSWORD=<password_here> - RESTIC_PASSWORD=<password_here>
# - RESTIC_REPO_PASSWORD_FILE=</file_with_password_here> # - RESTIC_PASSWORD_FILE=</file_with_password_here>
- REFRESH_INTERVAL=1800 # 30 min - REFRESH_INTERVAL=1800 # 30 min
volumes: volumes:
- /host_path/restic/data:/data - /host_path/restic/data:/data
@ -76,8 +76,8 @@ services:
docker run -d \ docker run -d \
--name=restic-exporter \ --name=restic-exporter \
-e TZ=Europe/Madrid \ -e TZ=Europe/Madrid \
-e RESTIC_REPO_URL=/data \ -e RESTIC_REPOSITORY=/data \
-e RESTIC_REPO_PASSWORD=<password_here> \ -e RESTIC_PASSWORD=<password_here> \
-e REFRESH_INTERVAL=1800 \ -e REFRESH_INTERVAL=1800 \
-p 8001:8001 \ -p 8001:8001 \
--restart unless-stopped \ --restart unless-stopped \
@ -91,16 +91,17 @@ Some of them need additional environment variables for the secrets.
All configuration is done with environment variables: All configuration is done with environment variables:
- `RESTIC_REPO_URL`: Restic repository URL. All backends are supported. Examples: - `RESTIC_REPOSITORY`: Restic repository URL. All backends are supported. Examples:
* Local repository: `/data` * Local repository: `/data`
* REST Server: `rest:http://user:password@127.0.0.1:8000/` * REST Server: `rest:http://user:password@127.0.0.1:8000/`
* Amazon S3: `s3:s3.amazonaws.com/bucket_name` * Amazon S3: `s3:s3.amazonaws.com/bucket_name`
* Backblaze B2: `b2:bucketname:path/to/repo` * Backblaze B2: `b2:bucketname:path/to/repo`
* Rclone (see notes below): `rclone:gd-backup:/restic`
- `RESTIC_REPO_PASSWORD`: Restic repository password in plain text. This is only - `RESTIC_PASSWORD`: Restic repository password in plain text. This is only
required if `RESTIC_REPO_PASSWORD_FILE` is not defined. required if `RESTIC_PASSWORD_FILE` is not defined.
- `RESTIC_REPO_PASSWORD_FILE`: File with the Restic repository password in plain - `RESTIC_PASSWORD_FILE`: File with the Restic repository password in plain
text. This is only required if `RESTIC_REPO_PASSWORD` is not defined. Remember text. This is only required if `RESTIC_PASSWORD` is not defined. Remember
to mount the Docker volume with the file. to mount the Docker volume with the file.
- `AWS_ACCESS_KEY_ID`: (Optional) Required for Amazon S3, Minio and Wasabi - `AWS_ACCESS_KEY_ID`: (Optional) Required for Amazon S3, Minio and Wasabi
backends. backends.
@ -109,7 +110,7 @@ backends.
- `B2_ACCOUNT_ID`: (Optional) Required for Backblaze B2 backend. - `B2_ACCOUNT_ID`: (Optional) Required for Backblaze B2 backend.
- `B2_ACCOUNT_KEY`: (Optional) Required for Backblaze B2 backend. - `B2_ACCOUNT_KEY`: (Optional) Required for Backblaze B2 backend.
- `REFRESH_INTERVAL`: (Optional) Refresh interval for the metrics in seconds. - `REFRESH_INTERVAL`: (Optional) Refresh interval for the metrics in seconds.
Computing the metrics is a expensive task, keep this value as high as possible. Computing the metrics is an expensive task, keep this value as high as possible.
Default is `60` seconds. Default is `60` seconds.
- `LISTEN_PORT`: (Optional) The address the exporter should listen on. The - `LISTEN_PORT`: (Optional) The address the exporter should listen on. The
default is `8001`. default is `8001`.
@ -122,29 +123,58 @@ is `Flase` (only log error, such as network error with Cloud backends).
reasons. Default is `False` (perform `restic check`). reasons. Default is `False` (perform `restic check`).
- `NO_STATS`: (Optional) Do not collect per backup statistics for performance - `NO_STATS`: (Optional) Do not collect per backup statistics for performance
reasons. Default is `False` (collect per backup statistics). reasons. Default is `False` (collect per backup statistics).
- `NO_LOCKS`: (Optional) Do not collect the number of locks. Default is `False` (collect the number of locks).
- `INCLUDE_PATHS`: (Optional) Include snapshot paths for each backup. The paths are separated by commas. Default is `False` (not collect the paths).
### Configuration for Rclone
Rclone is not included in the Docker image. You have to mount the Rclone executable and the Rclone configuration from the host machine. Here is an example with docker-compose:
```yaml
version: '2.1'
services:
restic-exporter:
image: ngosang/restic-exporter
container_name: restic-exporter
environment:
- TZ=Europe/Madrid
- RESTIC_REPOSITORY=rclone:gd-backup:/restic
- RESTIC_PASSWORD=
- REFRESH_INTERVAL=1800 # 30 min
volumes:
- /host_path/restic/data:/data
- /usr/bin/rclone:/usr/bin/rclone:ro
- /host_path/restic/rclone.conf:/root/.config/rclone/rclone.conf:ro
ports:
- "8001:8001"
restart: unless-stopped
```
## Exported metrics ## Exported metrics
```python ```bash
# HELP restic_check_success Result of restic check operation in the repository # HELP restic_check_success Result of restic check operation in the repository
# TYPE restic_check_success gauge # TYPE restic_check_success gauge
restic_check_success 1.0 restic_check_success 1.0
# HELP restic_locks_total Total number of locks in the repository
# TYPE restic_locks_total counter
restic_locks_total 1.0
# HELP restic_snapshots_total Total number of snapshots in the repository # HELP restic_snapshots_total Total number of snapshots in the repository
# TYPE restic_snapshots_total counter # TYPE restic_snapshots_total counter
restic_snapshots_total 100.0 restic_snapshots_total 100.0
# HELP restic_backup_timestamp Timestamp of the last backup # HELP restic_backup_timestamp Timestamp of the last backup
# TYPE restic_backup_timestamp gauge # TYPE restic_backup_timestamp gauge
restic_backup_timestamp{client_hostname="product.example.com",client_username="root",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql"} 1.666273638e+09 restic_backup_timestamp{client_hostname="product.example.com",client_username="root",client_version="restic 0.16.0",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_tags="mysql,tag2",snapshot_paths="/mysql/data,/mysql/config"} 1.666273638e+09
# HELP restic_backup_files_total Number of files in the backup # HELP restic_backup_files_total Number of files in the backup
# TYPE restic_backup_files_total counter # TYPE restic_backup_files_total counter
restic_backup_files_total{client_hostname="product.example.com",client_username="root",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql"} 8.0 restic_backup_files_total{client_hostname="product.example.com",client_username="root",client_version="restic 0.16.0",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_tags="mysql,tag2",snapshot_paths="/mysql/data,/mysql/config"} 8.0
# HELP restic_backup_size_total Total size of backup in bytes # HELP restic_backup_size_total Total size of backup in bytes
# TYPE restic_backup_size_total counter # TYPE restic_backup_size_total counter
restic_backup_size_total{client_hostname="product.example.com",client_username="root",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql"} 4.3309562e+07 restic_backup_size_total{client_hostname="product.example.com",client_username="root",client_version="restic 0.16.0",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_tags="mysql,tag2",snapshot_paths="/mysql/data,/mysql/config"} 4.3309562e+07
# HELP restic_backup_snapshots_total Total number of snapshots # HELP restic_backup_snapshots_total Total number of snapshots
# TYPE restic_backup_snapshots_total counter # TYPE restic_backup_snapshots_total counter
restic_backup_snapshots_total{client_hostname="product.example.com",client_username="root",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql"} 1.0 restic_backup_snapshots_total{client_hostname="product.example.com",client_username="root",client_version="restic 0.16.0",snapshot_hash="20795072cba0953bcdbe52e9cf9d75e5726042f5bbf2584bb2999372398ee835",snapshot_tag="mysql",snapshot_tags="mysql,tag2",snapshot_paths="/mysql/data,/mysql/config"} 1.0
# HELP restic_scrape_duration_seconds Ammount of time each scrape takes # HELP restic_scrape_duration_seconds Amount of time each scrape takes
# TYPE restic_scrape_duration_seconds gauge # TYPE restic_scrape_duration_seconds gauge
restic_scrape_duration_seconds 166.9411084651947 restic_scrape_duration_seconds 166.9411084651947
``` ```

View file

@ -6,9 +6,9 @@ services:
container_name: restic-exporter container_name: restic-exporter
environment: environment:
- TZ=Europe/Madrid - TZ=Europe/Madrid
- RESTIC_REPO_URL=/data - RESTIC_REPOSITORY=/data
- RESTIC_REPO_PASSWORD=password_here - RESTIC_PASSWORD=password_here
# - RESTIC_REPO_PASSWORD_FILE=/file_with_password_here # - RESTIC_PASSWORD_FILE=/file_with_password_here
- REFRESH_INTERVAL=1800 # 30 min - REFRESH_INTERVAL=1800 # 30 min
volumes: volumes:
- /host_path/restic/data:/data - /host_path/restic/data:/data

View file

@ -3,14 +3,23 @@
# Exit on error. For debug use set -x # Exit on error. For debug use set -x
set -e set -e
if [ -z "${RESTIC_REPO_PASSWORD}" ]; then if [ -n "${RESTIC_REPO_PASSWORD}" ]; then
if [ -z "${RESTIC_REPO_PASSWORD_FILE}" ]; then echo "The environment variable RESTIC_REPO_PASSWORD is deprecated, please use RESTIC_PASSWORD instead."
echo "You have to define one of these environment variables: RESTIC_REPO_PASSWORD or RESTIC_REPO_PASSWORD_FILE" export RESTIC_PASSWORD="${RESTIC_REPO_PASSWORD}"
fi
if [ -n "${RESTIC_REPO_PASSWORD_FILE}" ]; then
echo "The environment variable RESTIC_REPO_PASSWORD_FILE is deprecated, please use RESTIC_PASSWORD_FILE instead."
export RESTIC_PASSWORD_FILE="${RESTIC_REPO_PASSWORD_FILE}"
fi
if [ -z "${RESTIC_PASSWORD}" ]; then
if [ -z "${RESTIC_PASSWORD_FILE}" ]; then
echo "You have to define one of these environment variables: RESTIC_PASSWORD or RESTIC_PASSWORD_FILE"
exit 1 exit 1
fi fi
else else
export RESTIC_REPO_PASSWORD_FILE="/tmp/restic_passwd" export RESTIC_PASSWORD_FILE="/tmp/restic_passwd"
echo "${RESTIC_REPO_PASSWORD}" > "${RESTIC_REPO_PASSWORD_FILE}" echo "${RESTIC_PASSWORD}" > "${RESTIC_PASSWORD_FILE}"
fi fi
/usr/local/bin/python -u /restic-exporter.py /usr/local/bin/python -u /restic-exporter.py

View file

@ -132,7 +132,7 @@
}, },
"gridPos": { "gridPos": {
"h": 3, "h": 3,
"w": 24, "w": 18,
"x": 0, "x": 0,
"y": 0 "y": 0
}, },
@ -169,6 +169,74 @@
"title": "Repository Check", "title": "Repository Check",
"type": "stat" "type": "stat"
}, },
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 2
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 3,
"w": 6,
"x": 18,
"y": 0
},
"id": 42,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"exemplar": true,
"expr": "restic_locks_total",
"interval": "",
"legendFormat": "",
"range": true,
"refId": "A"
}
],
"title": "Repository Locks",
"type": "stat"
},
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
@ -254,7 +322,7 @@
}, },
"gridPos": { "gridPos": {
"h": 9, "h": 9,
"w": 15, "w": 18,
"x": 0, "x": 0,
"y": 3 "y": 3
}, },
@ -328,7 +396,7 @@
"client_id": true, "client_id": true,
"client_os_version": true, "client_os_version": true,
"client_username": true, "client_username": true,
"client_version": true, "client_version": false,
"instance": true, "instance": true,
"job": true, "job": true,
"snapshot_hash": true, "snapshot_hash": true,
@ -378,8 +446,8 @@
}, },
"gridPos": { "gridPos": {
"h": 9, "h": 9,
"w": 9, "w": 6,
"x": 15, "x": 18,
"y": 3 "y": 3
}, },
"id": 32, "id": 32,
@ -405,6 +473,11 @@
"pluginVersion": "9.3.0", "pluginVersion": "9.3.0",
"targets": [ "targets": [
{ {
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"exemplar": true, "exemplar": true,
"expr": "restic_backup_snapshots_total", "expr": "restic_backup_snapshots_total",
"format": "table", "format": "table",
@ -412,11 +485,7 @@
"interval": "", "interval": "",
"intervalFactor": 1, "intervalFactor": 1,
"legendFormat": "{{client_hostname}}", "legendFormat": "{{client_hostname}}",
"refId": "A", "refId": "A"
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
}
} }
], ],
"title": "Total snapshot count", "title": "Total snapshot count",
@ -494,7 +563,7 @@
}, },
"gridPos": { "gridPos": {
"h": 9, "h": 9,
"w": 9, "w": 18,
"x": 0, "x": 0,
"y": 12 "y": 12
}, },
@ -521,6 +590,10 @@
"pluginVersion": "9.3.0", "pluginVersion": "9.3.0",
"targets": [ "targets": [
{ {
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true, "exemplar": true,
"expr": "restic_backup_size_total", "expr": "restic_backup_size_total",
"format": "table", "format": "table",
@ -528,36 +601,71 @@
"interval": "", "interval": "",
"intervalFactor": 1, "intervalFactor": 1,
"legendFormat": "{{client_hostname}}", "legendFormat": "{{client_hostname}}",
"refId": "A", "refId": "A"
},
{
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "${DS_PROMETHEUS}" "uid": "${DS_PROMETHEUS}"
} },
"editorMode": "code",
"exemplar": true,
"expr": "restic_backup_files_total",
"format": "table",
"hide": false,
"instant": true,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{client_hostname}}",
"refId": "B"
} }
], ],
"title": "Total backup size", "title": "Total backup size & files",
"transformations": [ "transformations": [
{
"id": "joinByField",
"options": {
"byField": "client_hostname",
"mode": "outer"
}
},
{ {
"id": "organize", "id": "organize",
"options": { "options": {
"excludeByName": { "excludeByName": {
"Time": true, "Time": true,
"Time 1": true,
"Time 2": true,
"__name__": true, "__name__": true,
"__name__ 1": true,
"backup_id": true, "backup_id": true,
"backup_type": true, "backup_type": true,
"client_hostname": false, "client_hostname": false,
"client_id": true, "client_id": true,
"client_os_version": true, "client_os_version": true,
"client_username": true, "client_username": true,
"client_username 1": true,
"client_username 2": true,
"client_version": true, "client_version": true,
"client_version 1": true,
"client_version 2": true,
"instance": true, "instance": true,
"instance 1": true,
"instance 2": true,
"job": true, "job": true,
"job 1": true,
"job 2": true,
"snapshot_hash": true, "snapshot_hash": true,
"snapshot_hash 1": true,
"snapshot_hash 2": true,
"snapshot_id": true "snapshot_id": true
}, },
"indexByName": {}, "indexByName": {},
"renameByName": { "renameByName": {
"Value": "total_backup_size" "Time 2": "",
"Value": "total_backup_size",
"Value #A": "total_backup_size",
"Value #B": "total_backup_files"
} }
} }
} }
@ -590,7 +698,7 @@
"gridPos": { "gridPos": {
"h": 9, "h": 9,
"w": 6, "w": 6,
"x": 9, "x": 18,
"y": 12 "y": 12
}, },
"id": 35, "id": 35,
@ -661,109 +769,6 @@
], ],
"type": "piechart" "type": "piechart"
}, },
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"filterable": false,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 9,
"x": 15,
"y": 12
},
"id": 33,
"links": [],
"maxDataPoints": 100,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 1,
"showHeader": true,
"sortBy": [
{
"desc": false,
"displayName": "client_hostname"
}
]
},
"pluginVersion": "9.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"exemplar": true,
"expr": "restic_backup_files_total",
"format": "table",
"instant": true,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{client_hostname}}",
"refId": "A"
}
],
"title": "Total backup files",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"__name__": true,
"backup_id": true,
"backup_type": true,
"client_hostname": false,
"client_id": true,
"client_os_version": true,
"client_username": true,
"client_version": true,
"instance": true,
"job": true,
"snapshot_hash": true,
"snapshot_id": true
},
"indexByName": {},
"renameByName": {
"Value": "total_backup_files"
}
}
}
],
"type": "table"
},
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
@ -1103,6 +1108,6 @@
"timezone": "", "timezone": "",
"title": "Restic Exporter", "title": "Restic Exporter",
"uid": "2JzZl3B7k", "uid": "2JzZl3B7k",
"version": 23, "version": 25,
"weekStart": "" "weekStart": ""
} }

Binary file not shown.

Before

Width:  |  Height:  |  Size: 124 KiB

After

Width:  |  Height:  |  Size: 183 KiB

View file

@ -1,5 +1,5 @@
{ {
"name": "ngosang-restic-exporter", "name": "ngosang-restic-exporter",
"version": "1.2.1", "version": "1.5.0",
"author": "ngosang@hotmail.es" "author": "ngosang@hotmail.es"
} }

View file

@ -1 +1 @@
prometheus-client==0.16.0 prometheus-client==0.19.0

View file

@ -16,13 +16,16 @@ from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, REGIS
class ResticCollector(object): class ResticCollector(object):
def __init__( def __init__(
self, repository, password_file, exit_on_error, disable_check, disable_stats self, repository, password_file, exit_on_error, disable_check,
disable_stats, disable_locks, include_paths
): ):
self.repository = repository self.repository = repository
self.password_file = password_file self.password_file = password_file
self.exit_on_error = exit_on_error self.exit_on_error = exit_on_error
self.disable_check = disable_check self.disable_check = disable_check
self.disable_stats = disable_stats self.disable_stats = disable_stats
self.disable_locks = disable_locks
self.include_paths = include_paths
# todo: the stats cache increases over time -> remove old ids # todo: the stats cache increases over time -> remove old ids
# todo: cold start -> the stats cache could be saved in a persistent volume # todo: cold start -> the stats cache could be saved in a persistent volume
# todo: cold start -> the restic cache (/root/.cache/restic) could be # todo: cold start -> the restic cache (/root/.cache/restic) could be
@ -37,8 +40,11 @@ class ResticCollector(object):
common_label_names = [ common_label_names = [
"client_hostname", "client_hostname",
"client_username", "client_username",
"client_version",
"snapshot_hash", "snapshot_hash",
"snapshot_tag", "snapshot_tag",
"snapshot_tags",
"snapshot_paths",
] ]
check_success = GaugeMetricFamily( check_success = GaugeMetricFamily(
@ -46,52 +52,55 @@ class ResticCollector(object):
"Result of restic check operation in the repository", "Result of restic check operation in the repository",
labels=[], labels=[],
) )
locks_total = CounterMetricFamily(
"restic_locks_total",
"Total number of locks in the repository",
labels=[],
)
snapshots_total = CounterMetricFamily( snapshots_total = CounterMetricFamily(
"restic_snapshots_total", "restic_snapshots_total",
"Total number of snapshots in the repository", "Total number of snapshots in the repository",
labels=[], labels=[],
) )
backup_timestamp = GaugeMetricFamily( backup_timestamp = GaugeMetricFamily(
"restic_backup_timestamp", "restic_backup_timestamp",
"Timestamp of the last backup", "Timestamp of the last backup",
labels=common_label_names, labels=common_label_names,
) )
backup_files_total = CounterMetricFamily( backup_files_total = CounterMetricFamily(
"restic_backup_files_total", "restic_backup_files_total",
"Number of files in the backup", "Number of files in the backup",
labels=common_label_names, labels=common_label_names,
) )
backup_size_total = CounterMetricFamily( backup_size_total = CounterMetricFamily(
"restic_backup_size_total", "restic_backup_size_total",
"Total size of backup in bytes", "Total size of backup in bytes",
labels=common_label_names, labels=common_label_names,
) )
backup_snapshots_total = CounterMetricFamily( backup_snapshots_total = CounterMetricFamily(
"restic_backup_snapshots_total", "restic_backup_snapshots_total",
"Total number of snapshots", "Total number of snapshots",
labels=common_label_names, labels=common_label_names,
) )
scrape_duration_seconds = GaugeMetricFamily( scrape_duration_seconds = GaugeMetricFamily(
"restic_scrape_duration_seconds", "restic_scrape_duration_seconds",
"Ammount of time each scrape takes", "Amount of time each scrape takes",
labels=[], labels=[],
) )
check_success.add_metric([], self.metrics["check_success"]) check_success.add_metric([], self.metrics["check_success"])
locks_total.add_metric([], self.metrics["locks_total"])
snapshots_total.add_metric([], self.metrics["snapshots_total"]) snapshots_total.add_metric([], self.metrics["snapshots_total"])
for client in self.metrics["clients"]: for client in self.metrics["clients"]:
common_label_values = [ common_label_values = [
client["hostname"], client["hostname"],
client["username"], client["username"],
client["version"],
client["snapshot_hash"], client["snapshot_hash"],
client["snapshot_tag"], client["snapshot_tag"],
client["snapshot_tags"],
client["snapshot_paths"],
] ]
backup_timestamp.add_metric(common_label_values, client["timestamp"]) backup_timestamp.add_metric(common_label_values, client["timestamp"])
@ -104,6 +113,7 @@ class ResticCollector(object):
scrape_duration_seconds.add_metric([], self.metrics["duration"]) scrape_duration_seconds.add_metric([], self.metrics["duration"])
yield check_success yield check_success
yield locks_total
yield snapshots_total yield snapshots_total
yield backup_timestamp yield backup_timestamp
yield backup_files_total yield backup_files_total
@ -173,8 +183,11 @@ class ResticCollector(object):
{ {
"hostname": snap["hostname"], "hostname": snap["hostname"],
"username": snap["username"], "username": snap["username"],
"version": snap["program_version"] if "program_version" in snap else "",
"snapshot_hash": snap["hash"], "snapshot_hash": snap["hash"],
"snapshot_tag": snap["tags"][0] if "tags" in snap else "", "snapshot_tag": snap["tags"][0] if "tags" in snap else "",
"snapshot_tags": ",".join(snap["tags"]) if "tags" in snap else "",
"snapshot_paths": ",".join(snap["paths"]) if self.include_paths else "",
"timestamp": snap["timestamp"], "timestamp": snap["timestamp"],
"size_total": stats["total_size"], "size_total": stats["total_size"],
"files_total": stats["total_file_count"], "files_total": stats["total_file_count"],
@ -192,8 +205,15 @@ class ResticCollector(object):
else: else:
check_success = self.get_check() check_success = self.get_check()
if self.disable_locks:
# return 0 as "no-locks" value
locks_total = 0
else:
locks_total = self.get_locks()
metrics = { metrics = {
"check_success": check_success, "check_success": check_success,
"locks_total": locks_total,
"clients": clients, "clients": clients,
"snapshots_total": len(all_snapshots), "snapshots_total": len(all_snapshots),
"duration": time.time() - duration "duration": time.time() - duration
@ -283,11 +303,33 @@ class ResticCollector(object):
) )
return 0 # error return 0 # error
def calc_snapshot_hash(self, snapshot: dict) -> str: def get_locks(self):
cmd = [
"restic",
"-r",
self.repository,
"-p",
self.password_file,
"--no-lock",
"list",
"locks",
]
result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if result.returncode != 0:
raise Exception(
"Error executing restic list locks command: " + self.parse_stderr(result)
)
text_result = result.stdout.decode("utf-8")
return len(text_result.split("\n")) - 1
@staticmethod
def calc_snapshot_hash(snapshot: dict) -> str:
text = snapshot["hostname"] + snapshot["username"] + ",".join(snapshot["paths"]) text = snapshot["hostname"] + snapshot["username"] + ",".join(snapshot["paths"])
return hashlib.sha256(text.encode("utf-8")).hexdigest() return hashlib.sha256(text.encode("utf-8")).hexdigest()
def parse_stderr(self, result): @staticmethod
def parse_stderr(result):
return ( return (
result.stderr.decode("utf-8").replace("\n", " ") result.stderr.decode("utf-8").replace("\n", " ")
+ " Exit code: " + " Exit code: "
@ -305,16 +347,28 @@ if __name__ == "__main__":
logging.info("Starting Restic Prometheus Exporter") logging.info("Starting Restic Prometheus Exporter")
logging.info("It could take a while if the repository is remote") logging.info("It could take a while if the repository is remote")
try: restic_repo_url = os.environ.get("RESTIC_REPOSITORY")
restic_repo_url = os.environ["RESTIC_REPO_URL"] if restic_repo_url is None:
except Exception: restic_repo_url = os.environ.get("RESTIC_REPO_URL")
logging.error("The environment variable RESTIC_REPO_URL is mandatory") if restic_repo_url is not None:
logging.warning(
"The environment variable RESTIC_REPO_URL is deprecated, "
"please use RESTIC_REPOSITORY instead."
)
if restic_repo_url is None:
logging.error("The environment variable RESTIC_REPOSITORY is mandatory")
sys.exit(1) sys.exit(1)
try: restic_repo_password_file = os.environ.get("RESTIC_PASSWORD_FILE")
restic_repo_password_file = os.environ["RESTIC_REPO_PASSWORD_FILE"] if restic_repo_password_file is None:
except Exception: restic_repo_password_file = os.environ.get("RESTIC_REPO_PASSWORD_FILE")
logging.error("The environment variable RESTIC_REPO_PASSWORD_FILE is mandatory") if restic_repo_password_file is not None:
logging.warning(
"The environment variable RESTIC_REPO_PASSWORD_FILE is deprecated, "
"please use RESTIC_PASSWORD_FILE instead."
)
if restic_repo_password_file is None:
logging.error("The environment variable RESTIC_PASSWORD_FILE is mandatory")
sys.exit(1) sys.exit(1)
exporter_address = os.environ.get("LISTEN_ADDRESS", "0.0.0.0") exporter_address = os.environ.get("LISTEN_ADDRESS", "0.0.0.0")
@ -323,6 +377,8 @@ if __name__ == "__main__":
exporter_exit_on_error = bool(os.environ.get("EXIT_ON_ERROR", False)) exporter_exit_on_error = bool(os.environ.get("EXIT_ON_ERROR", False))
exporter_disable_check = bool(os.environ.get("NO_CHECK", False)) exporter_disable_check = bool(os.environ.get("NO_CHECK", False))
exporter_disable_stats = bool(os.environ.get("NO_STATS", False)) exporter_disable_stats = bool(os.environ.get("NO_STATS", False))
exporter_disable_locks = bool(os.environ.get("NO_LOCKS", False))
exporter_include_paths = bool(os.environ.get("INCLUDE_PATHS", False))
try: try:
collector = ResticCollector( collector = ResticCollector(
@ -331,6 +387,8 @@ if __name__ == "__main__":
exporter_exit_on_error, exporter_exit_on_error,
exporter_disable_check, exporter_disable_check,
exporter_disable_stats, exporter_disable_stats,
exporter_disable_locks,
exporter_include_paths,
) )
REGISTRY.register(collector) REGISTRY.register(collector)
start_http_server(exporter_port, exporter_address) start_http_server(exporter_port, exporter_address)