Add ability to silence alerts with ntfy action

For now only for the single alert mode.
This is implemented as an action button, that will send an API request
to ntfy-alertmanager itself instead of Alertmanager. Here ntfy-alertmanager
acts as a proxy and will then do the API request for actually creating
the silence in Alertmanager.
This is founded in a limitation of ntfy, that seemingly does not allow a
json body with more than one key.
This commit is contained in:
Thorben Günther 2023-02-12 03:04:17 +01:00
parent 34c0574163
commit c919ec9af2
No known key found for this signature in database
GPG key ID: 415CD778D8C5AFED
2 changed files with 154 additions and 4 deletions

42
main.go
View file

@ -35,6 +35,7 @@ type payload struct {
GroupLabels map[string]interface{} `json:"groupLabels"` GroupLabels map[string]interface{} `json:"groupLabels"`
CommonLabels map[string]interface{} `json:"commonLabels"` CommonLabels map[string]interface{} `json:"commonLabels"`
CommonAnnotations map[string]interface{} `json:"commonAnnotations"` CommonAnnotations map[string]interface{} `json:"commonAnnotations"`
ExternalURL string `json:"externalURL"`
} }
type alert struct { type alert struct {
@ -45,10 +46,11 @@ type alert struct {
} }
type notification struct { type notification struct {
title string title string
body string body string
priority string priority string
tags string tags string
silenceBody string
} }
func (rcv *receiver) singleAlertNotifications(p *payload) []*notification { func (rcv *receiver) singleAlertNotifications(p *payload) []*notification {
@ -108,6 +110,24 @@ func (rcv *receiver) singleAlertNotifications(p *payload) []*notification {
n.tags = strings.Join(tags, ",") n.tags = strings.Join(tags, ",")
if rcv.cfg.am.SilenceDuration != 0 {
if rcv.cfg.BaseURL == "" {
rcv.logger.Error("Failed to create silence action: No base-url set")
}
// I could not convince ntfy to accept an Action with a body which contains
// a json with more than one key. Instead the json will be base64 encoded
// and sent to the ntfy-alertmanager silences endpoint, that operates as
// a proxy and will do the Alertmanager API request.
s := &silenceBody{AlertManagerURL: p.ExternalURL, Labels: alert.Labels}
b, err := json.Marshal(s)
if err != nil {
rcv.logger.Errorf("Failed to create silence action: %v", err)
}
n.silenceBody = base64.StdEncoding.EncodeToString(b)
}
notifications = append(notifications, n) notifications = append(notifications, n)
} }
@ -205,6 +225,18 @@ func (rcv *receiver) publish(n *notification) error {
req.Header.Set("X-Tags", n.tags) req.Header.Set("X-Tags", n.tags)
} }
if n.silenceBody != "" {
url := rcv.cfg.BaseURL + "/silences"
var authString string
if rcv.cfg.User != "" && rcv.cfg.Password != "" {
auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", rcv.cfg.User, rcv.cfg.Password)))
authString = fmt.Sprintf(", headers.Authorization=Basic %s", auth)
}
req.Header.Set("Actions", fmt.Sprintf("http, Silence, %s, method=POST, body=%s%s", url, n.silenceBody, authString))
}
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
return err return err
@ -325,8 +357,10 @@ func main() {
if cfg.User != "" && cfg.Password != "" { if cfg.User != "" && cfg.Password != "" {
logger.Info("Enabling HTTP Basic Authentication") logger.Info("Enabling HTTP Basic Authentication")
http.HandleFunc("/", receiver.basicAuthMiddleware(receiver.handleWebhooks)) http.HandleFunc("/", receiver.basicAuthMiddleware(receiver.handleWebhooks))
http.HandleFunc("/silences", receiver.basicAuthMiddleware(receiver.handleSilences))
} else { } else {
http.HandleFunc("/", receiver.handleWebhooks) http.HandleFunc("/", receiver.handleWebhooks)
http.HandleFunc("/silences", receiver.handleSilences)
} }
go receiver.runCleanup() go receiver.runCleanup()

116
silence.go Normal file
View file

@ -0,0 +1,116 @@
package main
import (
"bytes"
"encoding/base64"
"encoding/json"
"io"
"net/http"
"time"
)
const dateLayout = "2006-01-02 15:04:05"
type silence struct {
Matchers []matcher `json:"matchers"`
StartsAt string `json:"startsAt"`
EndsAt string `json:"endsAt"`
CreatedBy string `json:"createdBy"`
Comment string `json:"comment"`
}
type matcher struct {
Name string `json:"name"`
Value string `json:"value"`
IsRegex bool `json:"isRegex"`
IsEqual bool `json:"isEqual"`
}
type silenceBody struct {
AlertManagerURL string `json:"alertmanagerURL"`
Labels map[string]interface{} `json:"labels"`
}
func (rcv *receiver) handleSilences(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if r.Method != http.MethodPost {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
rcv.logger.Debugf("silences: illegal HTTP method: expected %q, got %q", "POST", r.Method)
return
}
b, err := io.ReadAll(r.Body)
if err != nil {
rcv.logger.Debugf("silences: %v", err)
return
}
b, err = base64.StdEncoding.DecodeString(string(b))
if err != nil {
rcv.logger.Debugf("silences: %v", err)
return
}
var sb silenceBody
err = json.Unmarshal(b, &sb)
if err != nil {
rcv.logger.Debugf("silences: %v", err)
return
}
var matchers []matcher
for key, value := range sb.Labels {
m := matcher{
Name: key,
Value: value.(string),
IsRegex: false,
IsEqual: true,
}
matchers = append(matchers, m)
}
silence := &silence{
StartsAt: time.Now().UTC().Format(dateLayout),
EndsAt: time.Now().Add(rcv.cfg.am.SilenceDuration).UTC().Format(dateLayout),
CreatedBy: "ntfy-alertmanager",
Comment: "",
Matchers: matchers,
}
b, err = json.Marshal(silence)
if err != nil {
rcv.logger.Debugf("silences: %v", err)
return
}
client := &http.Client{Timeout: time.Second * 3}
url := sb.AlertManagerURL + "/api/v2/silences"
req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(b))
if err != nil {
rcv.logger.Debugf("silences: %v", err)
return
}
req.Header.Add("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
rcv.logger.Debugf("silences: %v", err)
return
}
defer resp.Body.Close()
b, err = io.ReadAll(resp.Body)
if err != nil {
rcv.logger.Debugf("silences: %v", err)
return
}
if resp.StatusCode != http.StatusOK {
rcv.logger.Debugf("silences: received status code %d", resp.StatusCode)
return
}
rcv.logger.Debugf("silences: created new silence %s", string(b))
}