fix(telegram): backtick code spans silently dropped from message content (#518)

* fix(telegram): restore backtick code spans from Telegram entities

Telegram parses backtick-delimited code spans and code blocks from
message text and stores the formatting as entities (type "code"/"pre"),
stripping the backticks from the text field. The broker was ignoring
these entities, silently dropping code formatting from inbound messages.

Extend resolveUserMentions to reconstruct backtick wrapping from "code"
and "pre" entities so agents receive the original formatting. Also fix
stripMentions to preserve newlines (previously collapsed all whitespace
via strings.Fields), which is needed for multi-line code blocks.

Closes #285

* fix(telegram): preserve whitespace in stripMentions for code blocks

Replace strings.Fields-based word processing with regex-based approach
that finds and strips target mentions without collapsing whitespace or
destroying indentation in code blocks and inline code spans.

---------

Co-authored-by: Scion Agent (mi-dev-b) <agent@scion.dev>

Author: ptone

extras/scion-telegram/internal/telegram/api.go

@@ -102,10 +102,11 @@ type TGMessage struct {
 
 // MessageEntity represents a special entity in a Telegram message (e.g. @mentions, commands).
 type MessageEntity struct {
-	Type   string  `json:"type"`
-	Offset int     `json:"offset"`
-	Length int     `json:"length"`
-	User   *TGUser `json:"user,omitempty"`
+	Type     string  `json:"type"`
+	Offset   int     `json:"offset"`
+	Length   int     `json:"length"`
+	User     *TGUser `json:"user,omitempty"`
+	Language string  `json:"language,omitempty"`
 }
 
 // TGUser represents a Telegram user.

extras/scion-telegram/internal/telegram/broker_v2.go

@@ -1958,7 +1958,9 @@ func (b *TelegramBrokerV2) downloadTelegramFile(ctx context.Context, tgMsg *TGMe
 // resolveUserMentions extracts @username and text_mention entities from a
 // Telegram message, looks each up in the user_mappings store, and returns:
 //   - modifiedText: the message text with text_mention display names (offset>0)
-//     replaced by "user:email" for registered users
+//     replaced by "user:email" for registered users, and "code"/"pre" entity
+//     spans re-wrapped in backticks (Telegram strips backticks from the text
+//     field and stores the formatting as entities)
 //   - resolvedJSON: a JSON string mapping display names → "user:email" for all
 //     resolved mentions (empty string if none resolve)
 func (b *TelegramBrokerV2) resolveUserMentions(ctx context.Context, tgMsg *TGMessage) (string, string) {
@@ -2036,6 +2038,30 @@ func (b *TelegramBrokerV2) resolveUserMentions(ctx context.Context, tgMsg *TGMes
 					replacement: scionIdentity,
 				})
 			}
+		case "code":
+			span, ok := utf16Extract(tgMsg.Text, ent.Offset, ent.Length)
+			if !ok {
+				continue
+			}
+			replacements = append(replacements, textReplacement{
+				offset:      ent.Offset,
+				length:      ent.Length,
+				replacement: "`" + span + "`",
+			})
+		case "pre":
+			span, ok := utf16Extract(tgMsg.Text, ent.Offset, ent.Length)
+			if !ok {
+				continue
+			}
+			prefix := "```"
+			if ent.Language != "" {
+				prefix += ent.Language
+			}
+			replacements = append(replacements, textReplacement{
+				offset:      ent.Offset,
+				length:      ent.Length,
+				replacement: prefix + "\n" + span + "\n```",
+			})
 		}
 	}
 

extras/scion-telegram/internal/telegram/broker_v2_test.go

@@ -2314,3 +2314,295 @@ func TestV2_DownloadTelegramFile_DownloadFailure(t *testing.T) {
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "download file")
 }
+
+// --- Code span restoration tests ---
+
+func TestV2_HandleGroupMessage_CodeSpanPreserved(t *testing.T) {
+	tgSrv := newFakeTGServerV2(t)
+	hub := newFakeHubClient()
+	hub.agents["proj-1"] = []AgentInfo{{Slug: "coder"}}
+	b := newTestBrokerV2WithHub(t, tgSrv, hub)
+
+	ctx := context.Background()
+	require.NoError(t, b.store.SaveUserMapping(ctx, &TelegramUserMapping{
+		TelegramUserID: "456",
+		ScionEmail:     "alice@example.com",
+		LinkedAt:       time.Now().UTC(),
+	}))
+	require.NoError(t, b.store.SaveGroupLink(ctx, &GroupLink{
+		ChatID:       -200,
+		ProjectID:    "proj-1",
+		ProjectSlug:  "my-project",
+		DefaultAgent: "coder",
+		LinkedAt:     time.Now().UTC(),
+		Active:       true,
+	}))
+	require.NoError(t, b.store.SaveProjectAgents(ctx, &ProjectAgents{
+		ProjectID:   "proj-1",
+		Agents:      []AgentInfo{{Slug: "coder"}},
+		RefreshedAt: time.Now(),
+	}))
+
+	var deliveredMsg *messages.StructuredMessage
+	done := make(chan struct{}, 1)
+	b.InboundHandler = func(_ string, msg *messages.StructuredMessage) {
+		deliveredMsg = msg
+		select {
+		case done <- struct{}{}:
+		default:
+		}
+	}
+
+	// Simulate: user typed "check the `config` file" in Telegram.
+	// Telegram strips backticks and puts formatting in entities.
+	b.handleGroupMessage(&TGMessage{
+		MessageID: 42,
+		From:      &TGUser{ID: 456, Username: "alice"},
+		Chat:      TGChat{ID: -200, Type: "group"},
+		Date:      time.Now().Unix(),
+		Text:      "@coder check the config file",
+		Entities: []MessageEntity{
+			{Type: "mention", Offset: 0, Length: 6},
+			{Type: "code", Offset: 17, Length: 6},
+		},
+	})
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for delivery")
+	}
+
+	assert.Equal(t, "check the `config` file", deliveredMsg.Msg)
+}
+
+func TestV2_HandleGroupMessage_MultipleCodeSpans(t *testing.T) {
+	tgSrv := newFakeTGServerV2(t)
+	hub := newFakeHubClient()
+	hub.agents["proj-1"] = []AgentInfo{{Slug: "coder"}}
+	b := newTestBrokerV2WithHub(t, tgSrv, hub)
+
+	ctx := context.Background()
+	require.NoError(t, b.store.SaveUserMapping(ctx, &TelegramUserMapping{
+		TelegramUserID: "456",
+		ScionEmail:     "alice@example.com",
+		LinkedAt:       time.Now().UTC(),
+	}))
+	require.NoError(t, b.store.SaveGroupLink(ctx, &GroupLink{
+		ChatID:       -200,
+		ProjectID:    "proj-1",
+		ProjectSlug:  "my-project",
+		DefaultAgent: "coder",
+		LinkedAt:     time.Now().UTC(),
+		Active:       true,
+	}))
+	require.NoError(t, b.store.SaveProjectAgents(ctx, &ProjectAgents{
+		ProjectID:   "proj-1",
+		Agents:      []AgentInfo{{Slug: "coder"}},
+		RefreshedAt: time.Now(),
+	}))
+
+	var deliveredMsg *messages.StructuredMessage
+	done := make(chan struct{}, 1)
+	b.InboundHandler = func(_ string, msg *messages.StructuredMessage) {
+		deliveredMsg = msg
+		select {
+		case done <- struct{}{}:
+		default:
+		}
+	}
+
+	// Simulate: "run foo and bar" where foo and bar were in backticks.
+	b.handleGroupMessage(&TGMessage{
+		MessageID: 43,
+		From:      &TGUser{ID: 456, Username: "alice"},
+		Chat:      TGChat{ID: -200, Type: "group"},
+		Date:      time.Now().Unix(),
+		Text:      "@coder run foo and bar",
+		Entities: []MessageEntity{
+			{Type: "mention", Offset: 0, Length: 6},
+			{Type: "code", Offset: 11, Length: 3},
+			{Type: "code", Offset: 19, Length: 3},
+		},
+	})
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for delivery")
+	}
+
+	assert.Equal(t, "run `foo` and `bar`", deliveredMsg.Msg)
+}
+
+func TestV2_HandleGroupMessage_PreBlockPreserved(t *testing.T) {
+	tgSrv := newFakeTGServerV2(t)
+	hub := newFakeHubClient()
+	hub.agents["proj-1"] = []AgentInfo{{Slug: "coder"}}
+	b := newTestBrokerV2WithHub(t, tgSrv, hub)
+
+	ctx := context.Background()
+	require.NoError(t, b.store.SaveUserMapping(ctx, &TelegramUserMapping{
+		TelegramUserID: "456",
+		ScionEmail:     "alice@example.com",
+		LinkedAt:       time.Now().UTC(),
+	}))
+	require.NoError(t, b.store.SaveGroupLink(ctx, &GroupLink{
+		ChatID:       -200,
+		ProjectID:    "proj-1",
+		ProjectSlug:  "my-project",
+		DefaultAgent: "coder",
+		LinkedAt:     time.Now().UTC(),
+		Active:       true,
+	}))
+	require.NoError(t, b.store.SaveProjectAgents(ctx, &ProjectAgents{
+		ProjectID:   "proj-1",
+		Agents:      []AgentInfo{{Slug: "coder"}},
+		RefreshedAt: time.Now(),
+	}))
+
+	var deliveredMsg *messages.StructuredMessage
+	done := make(chan struct{}, 1)
+	b.InboundHandler = func(_ string, msg *messages.StructuredMessage) {
+		deliveredMsg = msg
+		select {
+		case done <- struct{}{}:
+		default:
+		}
+	}
+
+	// Simulate: user sent a code block (triple backticks) in Telegram.
+	b.handleGroupMessage(&TGMessage{
+		MessageID: 44,
+		From:      &TGUser{ID: 456, Username: "alice"},
+		Chat:      TGChat{ID: -200, Type: "group"},
+		Date:      time.Now().Unix(),
+		Text:      "@coder apply this:\nfmt.Println(\"hello\")",
+		Entities: []MessageEntity{
+			{Type: "mention", Offset: 0, Length: 6},
+			{Type: "pre", Offset: 19, Length: 20},
+		},
+	})
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for delivery")
+	}
+
+	assert.Equal(t, "apply this:\n```\nfmt.Println(\"hello\")\n```", deliveredMsg.Msg)
+}
+
+func TestV2_HandleGroupMessage_PreBlockWithLanguage(t *testing.T) {
+	tgSrv := newFakeTGServerV2(t)
+	hub := newFakeHubClient()
+	hub.agents["proj-1"] = []AgentInfo{{Slug: "coder"}}
+	b := newTestBrokerV2WithHub(t, tgSrv, hub)
+
+	ctx := context.Background()
+	require.NoError(t, b.store.SaveUserMapping(ctx, &TelegramUserMapping{
+		TelegramUserID: "456",
+		ScionEmail:     "alice@example.com",
+		LinkedAt:       time.Now().UTC(),
+	}))
+	require.NoError(t, b.store.SaveGroupLink(ctx, &GroupLink{
+		ChatID:       -200,
+		ProjectID:    "proj-1",
+		ProjectSlug:  "my-project",
+		DefaultAgent: "coder",
+		LinkedAt:     time.Now().UTC(),
+		Active:       true,
+	}))
+	require.NoError(t, b.store.SaveProjectAgents(ctx, &ProjectAgents{
+		ProjectID:   "proj-1",
+		Agents:      []AgentInfo{{Slug: "coder"}},
+		RefreshedAt: time.Now(),
+	}))
+
+	var deliveredMsg *messages.StructuredMessage
+	done := make(chan struct{}, 1)
+	b.InboundHandler = func(_ string, msg *messages.StructuredMessage) {
+		deliveredMsg = msg
+		select {
+		case done <- struct{}{}:
+		default:
+		}
+	}
+
+	// Simulate: user sent ```go\nfmt.Println("hello")\n``` in Telegram.
+	b.handleGroupMessage(&TGMessage{
+		MessageID: 45,
+		From:      &TGUser{ID: 456, Username: "alice"},
+		Chat:      TGChat{ID: -200, Type: "group"},
+		Date:      time.Now().Unix(),
+		Text:      "fmt.Println(\"hello\")",
+		Entities: []MessageEntity{
+			{Type: "pre", Offset: 0, Length: 20, Language: "go"},
+		},
+	})
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for delivery")
+	}
+
+	assert.Equal(t, "```go\nfmt.Println(\"hello\")\n```", deliveredMsg.Msg)
+}
+
+func TestV2_HandleGroupMessage_CodeSpanWithDefaultAgent(t *testing.T) {
+	tgSrv := newFakeTGServerV2(t)
+	hub := newFakeHubClient()
+	hub.agents["proj-1"] = []AgentInfo{{Slug: "coder"}}
+	b := newTestBrokerV2WithHub(t, tgSrv, hub)
+
+	ctx := context.Background()
+	require.NoError(t, b.store.SaveUserMapping(ctx, &TelegramUserMapping{
+		TelegramUserID: "456",
+		ScionEmail:     "alice@example.com",
+		LinkedAt:       time.Now().UTC(),
+	}))
+	require.NoError(t, b.store.SaveGroupLink(ctx, &GroupLink{
+		ChatID:       -200,
+		ProjectID:    "proj-1",
+		ProjectSlug:  "my-project",
+		DefaultAgent: "coder",
+		LinkedAt:     time.Now().UTC(),
+		Active:       true,
+	}))
+	require.NoError(t, b.store.SaveProjectAgents(ctx, &ProjectAgents{
+		ProjectID:   "proj-1",
+		Agents:      []AgentInfo{{Slug: "coder"}},
+		RefreshedAt: time.Now(),
+	}))
+
+	var deliveredMsg *messages.StructuredMessage
+	done := make(chan struct{}, 1)
+	b.InboundHandler = func(_ string, msg *messages.StructuredMessage) {
+		deliveredMsg = msg
+		select {
+		case done <- struct{}{}:
+		default:
+		}
+	}
+
+	// No mention — routes to default agent. Code span should be preserved.
+	b.handleGroupMessage(&TGMessage{
+		MessageID: 46,
+		From:      &TGUser{ID: 456, Username: "alice"},
+		Chat:      TGChat{ID: -200, Type: "group"},
+		Date:      time.Now().Unix(),
+		Text:      "check the config file",
+		Entities: []MessageEntity{
+			{Type: "code", Offset: 10, Length: 6},
+		},
+	})
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for delivery")
+	}
+
+	assert.Equal(t, "check the `config` file", deliveredMsg.Msg)
+}