From c919ec9af2f24c9b63f28203a96e150de2729643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thorben=20G=C3=BCnther?= Date: Sun, 12 Feb 2023 03:04:17 +0100 Subject: [PATCH] Add ability to silence alerts with ntfy action For now only for the single alert mode. This is implemented as an action button, that will send an API request to ntfy-alertmanager itself instead of Alertmanager. Here ntfy-alertmanager acts as a proxy and will then do the API request for actually creating the silence in Alertmanager. This is founded in a limitation of ntfy, that seemingly does not allow a json body with more than one key. --- main.go | 42 +++++++++++++++++-- silence.go | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 silence.go diff --git a/main.go b/main.go index 7790c62..df75f8b 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ type payload struct { GroupLabels map[string]interface{} `json:"groupLabels"` CommonLabels map[string]interface{} `json:"commonLabels"` CommonAnnotations map[string]interface{} `json:"commonAnnotations"` + ExternalURL string `json:"externalURL"` } type alert struct { @@ -45,10 +46,11 @@ type alert struct { } type notification struct { - title string - body string - priority string - tags string + title string + body string + priority string + tags string + silenceBody string } func (rcv *receiver) singleAlertNotifications(p *payload) []*notification { @@ -108,6 +110,24 @@ func (rcv *receiver) singleAlertNotifications(p *payload) []*notification { n.tags = strings.Join(tags, ",") + if rcv.cfg.am.SilenceDuration != 0 { + if rcv.cfg.BaseURL == "" { + rcv.logger.Error("Failed to create silence action: No base-url set") + } + + // I could not convince ntfy to accept an Action with a body which contains + // a json with more than one key. Instead the json will be base64 encoded + // and sent to the ntfy-alertmanager silences endpoint, that operates as + // a proxy and will do the Alertmanager API request. + s := &silenceBody{AlertManagerURL: p.ExternalURL, Labels: alert.Labels} + b, err := json.Marshal(s) + if err != nil { + rcv.logger.Errorf("Failed to create silence action: %v", err) + } + + n.silenceBody = base64.StdEncoding.EncodeToString(b) + } + notifications = append(notifications, n) } @@ -205,6 +225,18 @@ func (rcv *receiver) publish(n *notification) error { req.Header.Set("X-Tags", n.tags) } + if n.silenceBody != "" { + url := rcv.cfg.BaseURL + "/silences" + + var authString string + if rcv.cfg.User != "" && rcv.cfg.Password != "" { + auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", rcv.cfg.User, rcv.cfg.Password))) + authString = fmt.Sprintf(", headers.Authorization=Basic %s", auth) + } + + req.Header.Set("Actions", fmt.Sprintf("http, Silence, %s, method=POST, body=%s%s", url, n.silenceBody, authString)) + } + resp, err := client.Do(req) if err != nil { return err @@ -325,8 +357,10 @@ func main() { if cfg.User != "" && cfg.Password != "" { logger.Info("Enabling HTTP Basic Authentication") http.HandleFunc("/", receiver.basicAuthMiddleware(receiver.handleWebhooks)) + http.HandleFunc("/silences", receiver.basicAuthMiddleware(receiver.handleSilences)) } else { http.HandleFunc("/", receiver.handleWebhooks) + http.HandleFunc("/silences", receiver.handleSilences) } go receiver.runCleanup() diff --git a/silence.go b/silence.go new file mode 100644 index 0000000..9c6cadb --- /dev/null +++ b/silence.go @@ -0,0 +1,116 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "time" +) + +const dateLayout = "2006-01-02 15:04:05" + +type silence struct { + Matchers []matcher `json:"matchers"` + StartsAt string `json:"startsAt"` + EndsAt string `json:"endsAt"` + CreatedBy string `json:"createdBy"` + Comment string `json:"comment"` +} + +type matcher struct { + Name string `json:"name"` + Value string `json:"value"` + IsRegex bool `json:"isRegex"` + IsEqual bool `json:"isEqual"` +} + +type silenceBody struct { + AlertManagerURL string `json:"alertmanagerURL"` + Labels map[string]interface{} `json:"labels"` +} + +func (rcv *receiver) handleSilences(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + + if r.Method != http.MethodPost { + http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed) + rcv.logger.Debugf("silences: illegal HTTP method: expected %q, got %q", "POST", r.Method) + return + } + + b, err := io.ReadAll(r.Body) + if err != nil { + rcv.logger.Debugf("silences: %v", err) + return + } + + b, err = base64.StdEncoding.DecodeString(string(b)) + if err != nil { + rcv.logger.Debugf("silences: %v", err) + return + } + + var sb silenceBody + err = json.Unmarshal(b, &sb) + if err != nil { + rcv.logger.Debugf("silences: %v", err) + return + } + + var matchers []matcher + for key, value := range sb.Labels { + m := matcher{ + Name: key, + Value: value.(string), + IsRegex: false, + IsEqual: true, + } + + matchers = append(matchers, m) + } + + silence := &silence{ + StartsAt: time.Now().UTC().Format(dateLayout), + EndsAt: time.Now().Add(rcv.cfg.am.SilenceDuration).UTC().Format(dateLayout), + CreatedBy: "ntfy-alertmanager", + Comment: "", + Matchers: matchers, + } + + b, err = json.Marshal(silence) + if err != nil { + rcv.logger.Debugf("silences: %v", err) + return + } + + client := &http.Client{Timeout: time.Second * 3} + url := sb.AlertManagerURL + "/api/v2/silences" + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(b)) + if err != nil { + rcv.logger.Debugf("silences: %v", err) + return + } + + req.Header.Add("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + rcv.logger.Debugf("silences: %v", err) + return + } + defer resp.Body.Close() + + b, err = io.ReadAll(resp.Body) + if err != nil { + rcv.logger.Debugf("silences: %v", err) + return + } + + if resp.StatusCode != http.StatusOK { + rcv.logger.Debugf("silences: received status code %d", resp.StatusCode) + return + } + + rcv.logger.Debugf("silences: created new silence %s", string(b)) +}