Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

* [CHANGE] notify: The `reason` label on `alertmanager_notifications_failed_total` now distinguishes `authError` (HTTP 401/403) and `rateLimited` (HTTP 429) from the generic `clientError`. Dashboards/alerts matching `reason="clientError"` for these codes must be updated.
* [ENHANCEMENT] notify: The discord and webex integrations now report a failure `reason` on `alertmanager_notifications_failed_total`.
* [ENHANCEMENT] notify: The telegram integration now reports a failure `reason` on `alertmanager_notifications_failed_total`. #3231
* [BUGFIX] webhook: Keep custom `payload` string values verbatim instead of reinterpreting JSON leaves that look like YAML (e.g. values ending with a colon). #5302

## 0.33.1 / 2026-07-04
Expand Down
19 changes: 18 additions & 1 deletion notify/telegram/telegram.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package telegram

import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
Expand Down Expand Up @@ -119,13 +120,29 @@ func (n *Notifier) Notify(ctx context.Context, alert ...*types.Alert) (bool, err
ParseMode: n.conf.ParseMode,
})
if err != nil {
return true, err
return true, wrapWithFailureReason(err)
}
logger.Debug("Telegram message successfully published", "message_id", message.ID, "chat_id", message.Chat.ID)

return false, nil
}

// wrapWithFailureReason classifies errors returned by the Telegram Bot API so
// that the failed notifications metric is labeled with the failure reason.
// Errors that telebot does not surface in a structured form are returned as-is
// and fall back to the default reason.
func wrapWithFailureReason(err error) error {
var floodErr telebot.FloodError
if errors.As(err, &floodErr) {
return notify.NewErrorWithReason(notify.RateLimitedReason, err)
}
var apiErr *telebot.Error
if errors.As(err, &apiErr) {
return notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(apiErr.Code), err)
}
return err
}

func createTelegramClient(apiURL, parseMode string, httpClient *http.Client) (*telebot.Bot, error) {
bot, err := telebot.NewBot(telebot.Settings{
URL: apiURL,
Expand Down
75 changes: 75 additions & 0 deletions notify/telegram/telegram_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,3 +213,78 @@ func TestTelegramNotify(t *testing.T) {
})
}
}

func TestTelegramNotifyFailureReason(t *testing.T) {
token := "secret"

for _, tc := range []struct {
name string
response string
expectedReason notify.Reason
}{
{
name: "Unauthorized is classified as auth error",
response: `{"ok":false,"error_code":401,"description":"Unauthorized"}`,
expectedReason: notify.AuthErrorReason,
},
{
name: "Blocked by user is classified as auth error",
response: `{"ok":false,"error_code":403,"description":"Forbidden: bot was blocked by the user"}`,
expectedReason: notify.AuthErrorReason,
},
{
name: "Chat not found is classified as client error",
response: `{"ok":false,"error_code":400,"description":"Bad Request: chat not found"}`,
expectedReason: notify.ClientErrorReason,
},
{
name: "Internal server error is classified as server error",
response: `{"ok":false,"error_code":500,"description":"Internal Server Error"}`,
expectedReason: notify.ServerErrorReason,
},
{
name: "Flood control is classified as rate limited",
response: `{"ok":false,"error_code":429,"description":"Too Many Requests: retry after 5","parameters":{"retry_after":5}}`,
expectedReason: notify.RateLimitedReason,
},
} {
t.Run(tc.name, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(tc.response))
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)

cfg := config.TelegramConfig{
Message: "test",
HTTPConfig: &commoncfg.HTTPClientConfig{},
BotToken: commoncfg.Secret(token),
APIUrl: &amcommoncfg.URL{URL: u},
}

notifier, err := New(&cfg, test.CreateTmpl(t), promslog.NewNopLogger())
require.NoError(t, err)

ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
ctx = notify.WithGroupKey(ctx, "1")

retry, err := notifier.Notify(ctx, []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"lbl1": "val1"},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
},
},
}...)

require.True(t, retry)
require.Error(t, err)

var reasonError *notify.ErrorWithReason
require.ErrorAs(t, err, &reasonError)
require.Equal(t, tc.expectedReason, reasonError.Reason)
})
}
}