diff --git a/README.md b/README.md
index a6a5a5c..6deb0e7 100644
--- a/README.md
+++ b/README.md
@@ -69,6 +69,10 @@ alertmanager {
     # to create a silence via the Alertmanager API. Because of limitations in ntfy,
     # the request will be proxied through ntfy-alertmanager. Therefore ntfy-alertmanager
     # needs to be exposed to external network requests and base-url has to be set.
+    #
+    # When alert-mode is set to "single" all alert labels will be used to create the silence.
+    # When it is "multi" common labels between all the alerts will be used. WARNING: This
+    # could silence unwanted alerts.
     silence-duration 24h
     # Basic authentication (https://prometheus.io/docs/alerting/latest/https/)
     user user
diff --git a/main.go b/main.go
index df75f8b..83b9660 100644
--- a/main.go
+++ b/main.go
@@ -199,6 +199,20 @@ func (rcv *receiver) multiAlertNotification(p *payload) *notification {
 	n.priority = priority
 	n.tags = tagString
 
+	if rcv.cfg.am.SilenceDuration != 0 {
+		if rcv.cfg.BaseURL == "" {
+			rcv.logger.Error("Failed to create silence action: No base-url set")
+		}
+
+		s := &silenceBody{AlertManagerURL: p.ExternalURL, Labels: p.CommonLabels}
+		b, err := json.Marshal(s)
+		if err != nil {
+			rcv.logger.Errorf("Failed to create silence action: %v", err)
+		}
+
+		n.silenceBody = base64.StdEncoding.EncodeToString(b)
+	}
+
 	return n
 }