Add ability to silence alerts with ntfy action
For now only for the single alert mode. This is implemented as an action button, that will send an API request to ntfy-alertmanager itself instead of Alertmanager. Here ntfy-alertmanager acts as a proxy and will then do the API request for actually creating the silence in Alertmanager. This is founded in a limitation of ntfy, that seemingly does not allow a json body with more than one key.
This commit is contained in:
parent
34c0574163
commit
c919ec9af2
2 changed files with 154 additions and 4 deletions
34
main.go
34
main.go
|
@ -35,6 +35,7 @@ type payload struct {
|
||||||
GroupLabels map[string]interface{} `json:"groupLabels"`
|
GroupLabels map[string]interface{} `json:"groupLabels"`
|
||||||
CommonLabels map[string]interface{} `json:"commonLabels"`
|
CommonLabels map[string]interface{} `json:"commonLabels"`
|
||||||
CommonAnnotations map[string]interface{} `json:"commonAnnotations"`
|
CommonAnnotations map[string]interface{} `json:"commonAnnotations"`
|
||||||
|
ExternalURL string `json:"externalURL"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type alert struct {
|
type alert struct {
|
||||||
|
@ -49,6 +50,7 @@ type notification struct {
|
||||||
body string
|
body string
|
||||||
priority string
|
priority string
|
||||||
tags string
|
tags string
|
||||||
|
silenceBody string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rcv *receiver) singleAlertNotifications(p *payload) []*notification {
|
func (rcv *receiver) singleAlertNotifications(p *payload) []*notification {
|
||||||
|
@ -108,6 +110,24 @@ func (rcv *receiver) singleAlertNotifications(p *payload) []*notification {
|
||||||
|
|
||||||
n.tags = strings.Join(tags, ",")
|
n.tags = strings.Join(tags, ",")
|
||||||
|
|
||||||
|
if rcv.cfg.am.SilenceDuration != 0 {
|
||||||
|
if rcv.cfg.BaseURL == "" {
|
||||||
|
rcv.logger.Error("Failed to create silence action: No base-url set")
|
||||||
|
}
|
||||||
|
|
||||||
|
// I could not convince ntfy to accept an Action with a body which contains
|
||||||
|
// a json with more than one key. Instead the json will be base64 encoded
|
||||||
|
// and sent to the ntfy-alertmanager silences endpoint, that operates as
|
||||||
|
// a proxy and will do the Alertmanager API request.
|
||||||
|
s := &silenceBody{AlertManagerURL: p.ExternalURL, Labels: alert.Labels}
|
||||||
|
b, err := json.Marshal(s)
|
||||||
|
if err != nil {
|
||||||
|
rcv.logger.Errorf("Failed to create silence action: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
n.silenceBody = base64.StdEncoding.EncodeToString(b)
|
||||||
|
}
|
||||||
|
|
||||||
notifications = append(notifications, n)
|
notifications = append(notifications, n)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -205,6 +225,18 @@ func (rcv *receiver) publish(n *notification) error {
|
||||||
req.Header.Set("X-Tags", n.tags)
|
req.Header.Set("X-Tags", n.tags)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if n.silenceBody != "" {
|
||||||
|
url := rcv.cfg.BaseURL + "/silences"
|
||||||
|
|
||||||
|
var authString string
|
||||||
|
if rcv.cfg.User != "" && rcv.cfg.Password != "" {
|
||||||
|
auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", rcv.cfg.User, rcv.cfg.Password)))
|
||||||
|
authString = fmt.Sprintf(", headers.Authorization=Basic %s", auth)
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Actions", fmt.Sprintf("http, Silence, %s, method=POST, body=%s%s", url, n.silenceBody, authString))
|
||||||
|
}
|
||||||
|
|
||||||
resp, err := client.Do(req)
|
resp, err := client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -325,8 +357,10 @@ func main() {
|
||||||
if cfg.User != "" && cfg.Password != "" {
|
if cfg.User != "" && cfg.Password != "" {
|
||||||
logger.Info("Enabling HTTP Basic Authentication")
|
logger.Info("Enabling HTTP Basic Authentication")
|
||||||
http.HandleFunc("/", receiver.basicAuthMiddleware(receiver.handleWebhooks))
|
http.HandleFunc("/", receiver.basicAuthMiddleware(receiver.handleWebhooks))
|
||||||
|
http.HandleFunc("/silences", receiver.basicAuthMiddleware(receiver.handleSilences))
|
||||||
} else {
|
} else {
|
||||||
http.HandleFunc("/", receiver.handleWebhooks)
|
http.HandleFunc("/", receiver.handleWebhooks)
|
||||||
|
http.HandleFunc("/silences", receiver.handleSilences)
|
||||||
}
|
}
|
||||||
|
|
||||||
go receiver.runCleanup()
|
go receiver.runCleanup()
|
||||||
|
|
116
silence.go
Normal file
116
silence.go
Normal file
|
@ -0,0 +1,116 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const dateLayout = "2006-01-02 15:04:05"
|
||||||
|
|
||||||
|
type silence struct {
|
||||||
|
Matchers []matcher `json:"matchers"`
|
||||||
|
StartsAt string `json:"startsAt"`
|
||||||
|
EndsAt string `json:"endsAt"`
|
||||||
|
CreatedBy string `json:"createdBy"`
|
||||||
|
Comment string `json:"comment"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type matcher struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
IsRegex bool `json:"isRegex"`
|
||||||
|
IsEqual bool `json:"isEqual"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type silenceBody struct {
|
||||||
|
AlertManagerURL string `json:"alertmanagerURL"`
|
||||||
|
Labels map[string]interface{} `json:"labels"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (rcv *receiver) handleSilences(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer r.Body.Close()
|
||||||
|
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
|
||||||
|
rcv.logger.Debugf("silences: illegal HTTP method: expected %q, got %q", "POST", r.Method)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := io.ReadAll(r.Body)
|
||||||
|
if err != nil {
|
||||||
|
rcv.logger.Debugf("silences: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err = base64.StdEncoding.DecodeString(string(b))
|
||||||
|
if err != nil {
|
||||||
|
rcv.logger.Debugf("silences: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var sb silenceBody
|
||||||
|
err = json.Unmarshal(b, &sb)
|
||||||
|
if err != nil {
|
||||||
|
rcv.logger.Debugf("silences: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var matchers []matcher
|
||||||
|
for key, value := range sb.Labels {
|
||||||
|
m := matcher{
|
||||||
|
Name: key,
|
||||||
|
Value: value.(string),
|
||||||
|
IsRegex: false,
|
||||||
|
IsEqual: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
matchers = append(matchers, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
silence := &silence{
|
||||||
|
StartsAt: time.Now().UTC().Format(dateLayout),
|
||||||
|
EndsAt: time.Now().Add(rcv.cfg.am.SilenceDuration).UTC().Format(dateLayout),
|
||||||
|
CreatedBy: "ntfy-alertmanager",
|
||||||
|
Comment: "",
|
||||||
|
Matchers: matchers,
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err = json.Marshal(silence)
|
||||||
|
if err != nil {
|
||||||
|
rcv.logger.Debugf("silences: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &http.Client{Timeout: time.Second * 3}
|
||||||
|
url := sb.AlertManagerURL + "/api/v2/silences"
|
||||||
|
req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(b))
|
||||||
|
if err != nil {
|
||||||
|
rcv.logger.Debugf("silences: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Add("Content-Type", "application/json")
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
rcv.logger.Debugf("silences: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
b, err = io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
rcv.logger.Debugf("silences: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
rcv.logger.Debugf("silences: received status code %d", resp.StatusCode)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
rcv.logger.Debugf("silences: created new silence %s", string(b))
|
||||||
|
}
|
Loading…
Reference in a new issue