From 763bd8a265a2946cb3390e807f4f0bc20ee7aa49 Mon Sep 17 00:00:00 2001 From: c0lap5o Date: Tue, 8 Sep 2026 21:29:48 +0100 Subject: [PATCH 1/2] feat(gateway): Add configurable MCP heartbeat Idle MCP connections were torn down by clients with a shorter read timeout, since nothing was sent over the wire to keep them alive. The reference MCP SSE client, for example, defaults sse_read_timeout to 300s. Add an optional periodic heartbeat on both the SSE and Streamable HTTP transports, off by default so existing deployments are unchanged. Enable it with SEARCHBASE_MCP_HEARTBEAT_ENABLED and tune it with SEARCHBASE_MCP_HEARTBEAT_INTERVAL (seconds, default 60). Intervals below 15s fail startup, guarding the gateway and the host running it against an accidentally aggressive configuration. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 3 +- docker-compose.yml | 6 + docs/content/docs/mcp-integration.md | 13 ++ docs/content/docs/self-hosting.md | 2 + search-gateway/cmd/server/main.go | 4 +- search-gateway/internal/mcp/server.go | 24 +++- search-gateway/internal/settings/mcp.go | 36 ++++++ search-gateway/internal/settings/settings.go | 15 ++- .../internal/settings/settings_test.go | 111 ++++++++++++++++++ 9 files changed, 206 insertions(+), 8 deletions(-) create mode 100644 search-gateway/internal/settings/mcp.go diff --git a/AGENTS.md b/AGENTS.md index 7934c68..a8a86c7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,12 +18,13 @@ The front-facing orchestrator. Built in Go for high concurrency, low memory foot * Expose standard REST API (`/api/v1/search` and `/api/v1/fetch`). * Expose Swagger UI documentation (`/docs`). * Expose MCP Server over SSE (`/mcp/sse` and `/mcp/message`) and Streamable HTTP (`/mcp/http`). + * Optionally send periodic MCP heartbeat pings on both transports to keep idle connections from being dropped by clients with a shorter read timeout. Off by default, enabled with `SEARCHBASE_MCP_HEARTBEAT_ENABLED=true` and tuned with `SEARCHBASE_MCP_HEARTBEAT_INTERVAL` (seconds, default `60`, minimum `15` enforced at startup). * Auto-generate OpenAPI specs. * Scrape DuckDuckGo (HTML version: `html.duckduckgo.com`) to extract top URLs when using the native `searchbase_ddg` provider. * Call official provider APIs when configured, including Brave (`SEARCHBASE_SEARCH_PROVIDER=brave` and `SEARCHBASE_BRAVE_API_TOKEN`) and Mojeek (`SEARCHBASE_SEARCH_PROVIDER=mojeek` and `SEARCHBASE_MOJEEK_API_KEY`). * Return compact search results (`title`, `url`, `snippet`) without automatically fetching page content. * Fetch page content only through `/api/v1/fetch` or the MCP `fetch_url` tool when the client/LLM explicitly requests it. - * **Configuration:** Uses Viper library with centralized `settings` package (`internal/settings`) enforcing a strict `SEARCHBASE_` prefix for all environment variables (e.g., `SEARCHBASE_PORT`, `SEARCHBASE_CRAWL_WORKER_ADDRESS`, `SEARCHBASE_LOG_LEVEL`). Supports defaults and automatic env var binding. + * **Configuration:** Uses Viper library with centralized `settings` package (`internal/settings`) enforcing a strict `SEARCHBASE_` prefix for all environment variables (e.g., `SEARCHBASE_PORT`, `SEARCHBASE_CRAWL_WORKER_ADDRESS`, `SEARCHBASE_LOG_LEVEL`). Supports defaults and automatic env var binding. Grouped settings live in their own files (`otel.go` for tracing, `mcp.go` for MCP transport) and expose a `validate()` method so misconfiguration fails fast at startup. * **Design Pattern:** Uses Interfaces (e.g., `SearchProvider`) to allow easy plugging of search APIs and metasearch providers without changing core logic. Current providers include native DuckDuckGo HTML (`searchbase_ddg`), DDGS (`ddgs`), SearXNG (`searxng`), Brave Search API (`brave`), and Mojeek Search API (`mojeek`). ### B. Component 2: `crawl-worker` (Python) diff --git a/docker-compose.yml b/docker-compose.yml index b138b00..92d6e00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,12 @@ services: - SEARCHBASE_CRAWL_WORKER_ADDRESS=http://crawl-worker:8000 - SEARCHBASE_SEARCH_PROVIDER=ddgs - SEARCHBASE_DDGS_PROVIDER_ADDRESS=http://ddgs:8001 + # MCP heartbeat is off unless SEARCHBASE_MCP_HEARTBEAT_ENABLED is explicitly + # set to true here. It keeps idle SSE and streamable HTTP connections alive + # for clients that drop them after a shorter read timeout. + # The interval is in seconds and must be >= 15 when the heartbeat is enabled. + - SEARCHBASE_MCP_HEARTBEAT_ENABLED=true + - SEARCHBASE_MCP_HEARTBEAT_INTERVAL=60 depends_on: - crawl-worker diff --git a/docs/content/docs/mcp-integration.md b/docs/content/docs/mcp-integration.md index 10cc009..893a17c 100644 --- a/docs/content/docs/mcp-integration.md +++ b/docs/content/docs/mcp-integration.md @@ -181,3 +181,16 @@ Clients that support SSE MCP servers can use the same JSON shape: ``` Replace `localhost` with the actual server address if Searchbase is not running on the same machine as your MCP client. + +## Idle Connection Timeouts + +Some MCP clients close a connection when nothing is received for a while. The reference MCP SSE client, for example, defaults `sse_read_timeout` to 300s, so an idle Searchbase session is dropped after roughly five minutes. + +Searchbase can send periodic heartbeat pings on both the SSE and Streamable HTTP transports to keep those connections open. The heartbeat is off by default and is enabled explicitly: + +```env +SEARCHBASE_MCP_HEARTBEAT_ENABLED=true +SEARCHBASE_MCP_HEARTBEAT_INTERVAL=60 +``` + +`SEARCHBASE_MCP_HEARTBEAT_INTERVAL` is expressed in seconds, defaults to `60`, and is only used when the heartbeat is enabled. The minimum accepted value is `15`; anything lower fails startup. Pick an interval comfortably below your client's read timeout. See [Self Hosting](/docs/self-hosting/) for the full configuration reference. diff --git a/docs/content/docs/self-hosting.md b/docs/content/docs/self-hosting.md index 300ae99..aabf276 100644 --- a/docs/content/docs/self-hosting.md +++ b/docs/content/docs/self-hosting.md @@ -68,6 +68,8 @@ The `search-gateway` can be configured using the following environment variables | `SEARCHBASE_ADDRESS` | *(empty)* | Optional. Used to override the base URL sent to MCP clients for SSE connections. **Reverse Proxy Note:** If deploying Searchbase behind a reverse proxy (like Traefik, Nginx, or Cloudflare Tunnels), leave this blank. The server will use relative paths, allowing the proxy to handle domain routing seamlessly. Set this only if you need to force a specific absolute URL. | | `SEARCHBASE_LOG_LEVEL` | `error` | Log level for structured logging (`debug`, `info`, `warn`, `error`). | | `SEARCHBASE_LOG_FORMAT` | `json` | Log format (`json` or `text`). | +| `SEARCHBASE_MCP_HEARTBEAT_ENABLED` | `false` | Send periodic MCP heartbeat pings on the SSE and Streamable HTTP transports. Off unless explicitly set to `true`, so existing deployments keep the current behavior. Enable it when MCP clients drop idle connections, for example the reference MCP SSE client, which closes the stream after its default 300s read timeout. | +| `SEARCHBASE_MCP_HEARTBEAT_INTERVAL` | `60` | Heartbeat interval in seconds. Only used when `SEARCHBASE_MCP_HEARTBEAT_ENABLED=true`. Minimum accepted value is `15`; a lower value fails startup instead of hammering the gateway and its host with excessively frequent pings. | | `SEARCHBASE_TRACING_ENABLED` | `false` | Enable OpenTelemetry distributed tracing. (Experimental) | | `SEARCHBASE_TRACING_ENDPOINT` | *(empty)* | Endpoint for the tracing backend. (Experimental) | diff --git a/search-gateway/cmd/server/main.go b/search-gateway/cmd/server/main.go index bb5e2c5..1b3f760 100644 --- a/search-gateway/cmd/server/main.go +++ b/search-gateway/cmd/server/main.go @@ -63,13 +63,15 @@ func main() { apiServer.RegisterSwaggerRoutes(r) mcpServer := mcp.NewServer(searchProvider, scraperClient, logger) - mcpServer.RegisterRoutes(r, s.Address()) + mcpServer.RegisterRoutes(r, s.Address(), s.Mcp().HeartbeatEnabled(), s.Mcp().HeartbeatInterval()) logger.Info("Starting searchbase Gateway", "port", s.Port(), "search_backend", s.SearchProvider().Name(), "search_provider_address", s.SearchProvider().Address(), "crawl_worker", s.CrawlWorkerAddress(), + "mcp_heartbeat_enabled", s.Mcp().HeartbeatEnabled(), + "mcp_heartbeat_interval", s.Mcp().HeartbeatInterval(), ) if err := r.Run(":" + s.Port()); err != nil && err != http.ErrServerClosed { diff --git a/search-gateway/internal/mcp/server.go b/search-gateway/internal/mcp/server.go index a8b933b..518b8dd 100644 --- a/search-gateway/internal/mcp/server.go +++ b/search-gateway/internal/mcp/server.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "strings" + "time" "github.com/coolapso/searchbase/search-gateway/internal/scraper" "github.com/coolapso/searchbase/search-gateway/internal/search" @@ -64,24 +65,37 @@ func NewServer(searchProvider search.SearchProvider, scraperClient *scraper.Scra // If baseURL is not empty, it configures the SSE options to use it, // otherwise it leaves it empty so the server returns relative paths // for seamless reverse proxy integration. -func (s *MCPServer) RegisterRoutes(r *gin.Engine, baseURL string) { +// When heartbeatEnabled is true, both transports send periodic pings every +// heartbeatInterval to keep otherwise idle connections from being torn down +// by clients with a shorter read timeout. +func (s *MCPServer) RegisterRoutes(r *gin.Engine, baseURL string, heartbeatEnabled bool, heartbeatInterval time.Duration) { opts := []server.SSEOption{ server.WithSSEEndpoint("/mcp/sse"), server.WithMessageEndpoint("/mcp/message"), + server.WithKeepAlive(heartbeatEnabled), } if baseURL != "" { opts = append(opts, server.WithBaseURL(baseURL)) } - sseServer := server.NewSSEServer(s.McpServer, opts...) - r.GET("/mcp/sse", gin.WrapH(sseServer.SSEHandler())) - r.POST("/mcp/message", gin.WrapH(sseServer.MessageHandler())) - httpOpts := []server.StreamableHTTPOption{ server.WithEndpointPath("/mcp/http"), } + // WithKeepAliveInterval implicitly enables keepalive, so it must only be + // added when the heartbeat is actually enabled. WithHeartbeatInterval is + // the only heartbeat knob on the streamable HTTP transport, an unset + // interval means no heartbeat. + if heartbeatEnabled { + opts = append(opts, server.WithKeepAliveInterval(heartbeatInterval)) + httpOpts = append(httpOpts, server.WithHeartbeatInterval(heartbeatInterval)) + } + + sseServer := server.NewSSEServer(s.McpServer, opts...) + r.GET("/mcp/sse", gin.WrapH(sseServer.SSEHandler())) + r.POST("/mcp/message", gin.WrapH(sseServer.MessageHandler())) + httpServer := server.NewStreamableHTTPServer(s.McpServer, httpOpts...) r.Any("/mcp/http", gin.WrapH(httpServer)) } diff --git a/search-gateway/internal/settings/mcp.go b/search-gateway/internal/settings/mcp.go new file mode 100644 index 0000000..fa7418d --- /dev/null +++ b/search-gateway/internal/settings/mcp.go @@ -0,0 +1,36 @@ +package settings + +import ( + "fmt" + "time" +) + +// MinHeartbeatInterval is the lowest accepted MCP heartbeat interval. +// It guards the gateway, and the host running it, against an accidentally +// aggressive configuration, such as setting the interval to 1 assuming +// the value is expressed in milliseconds. +const MinHeartbeatInterval = 15 * time.Second + +// Mcp groups Model Context Protocol transport settings. +type Mcp struct { + heartbeatEnabled bool + heartbeatInterval time.Duration +} + +func (m *Mcp) HeartbeatEnabled() bool { return m.heartbeatEnabled } +func (m *Mcp) HeartbeatInterval() time.Duration { return m.heartbeatInterval } + +// validate enforces the minimum heartbeat interval, but only when the +// heartbeat is enabled. A misconfigured interval is irrelevant while the +// heartbeat is off. +func (m *Mcp) validate() error { + if m.heartbeatEnabled && m.heartbeatInterval < MinHeartbeatInterval { + return fmt.Errorf( + "mcp heartbeat interval too low: SEARCHBASE_MCP_HEARTBEAT_INTERVAL must be >= %ds, got %ds", + int(MinHeartbeatInterval.Seconds()), + int(m.heartbeatInterval.Seconds()), + ) + } + + return nil +} diff --git a/search-gateway/internal/settings/settings.go b/search-gateway/internal/settings/settings.go index da2a48c..97ece1a 100644 --- a/search-gateway/internal/settings/settings.go +++ b/search-gateway/internal/settings/settings.go @@ -3,6 +3,7 @@ package settings import ( "fmt" "strings" + "time" "github.com/gin-gonic/gin" "github.com/spf13/viper" @@ -17,6 +18,7 @@ type Settings struct { address string logLevel string otel *Otel + mcp *Mcp } type providerSpec struct { @@ -39,9 +41,10 @@ func (s *Settings) GinMode() string { return s.ginMode } func (s *Settings) Address() string { return s.address } func (s *Settings) LogLevel() string { return s.logLevel } func (s *Settings) Otel() *Otel { return s.otel } +func (s *Settings) Mcp() *Mcp { return s.mcp } // NewSettings loads SEARCHBASE_* environment variables, applies defaults, and -// validates provider and tracing configuration. +// validates provider, tracing and mcp configuration. func NewSettings() (*Settings, error) { v := viper.New() v.SetEnvPrefix("SEARCHBASE") @@ -52,6 +55,8 @@ func NewSettings() (*Settings, error) { v.SetDefault("SEARCH_PROVIDER", "searchbase_ddg") v.SetDefault("LOG_LEVEL", "error") v.SetDefault("ENVIRONMENT", "production") + v.SetDefault("MCP_HEARTBEAT_ENABLED", false) + v.SetDefault("MCP_HEARTBEAT_INTERVAL", 60) providerName := v.GetString("SEARCH_PROVIDER") if _, ok := providerSpecs[providerName]; !ok { @@ -91,6 +96,10 @@ func NewSettings() (*Settings, error) { endpoint: v.GetString("TRACING_ENDPOINT"), }, }, + mcp: &Mcp{ + heartbeatEnabled: v.GetBool("MCP_HEARTBEAT_ENABLED"), + heartbeatInterval: time.Duration(v.GetInt("MCP_HEARTBEAT_INTERVAL")) * time.Second, + }, } if strings.ToLower(v.GetString("ENVIRONMENT")) == "dev" { @@ -101,5 +110,9 @@ func NewSettings() (*Settings, error) { return nil, err } + if err := s.mcp.validate(); err != nil { + return nil, err + } + return s, nil } diff --git a/search-gateway/internal/settings/settings_test.go b/search-gateway/internal/settings/settings_test.go index ed5a850..e3a7503 100644 --- a/search-gateway/internal/settings/settings_test.go +++ b/search-gateway/internal/settings/settings_test.go @@ -3,6 +3,7 @@ package settings import ( "strings" "testing" + "time" "github.com/gin-gonic/gin" ) @@ -36,6 +37,14 @@ func TestNewSettings_Defaults(t *testing.T) { if s.SearchProvider().Name() != "searchbase_ddg" { t.Errorf("Expected default search provider searchbase_ddg, got %s", s.SearchProvider().Name()) } + + if s.Mcp().HeartbeatEnabled() { + t.Error("Expected mcp heartbeat to be disabled by default") + } + + if s.Mcp().HeartbeatInterval() != 60*time.Second { + t.Errorf("Expected default mcp heartbeat interval 60s, got %s", s.Mcp().HeartbeatInterval()) + } } func TestNewSettings_EnvVars(t *testing.T) { @@ -45,6 +54,8 @@ func TestNewSettings_EnvVars(t *testing.T) { t.Setenv("SEARCHBASE_ENVIRONMENT", "dev") t.Setenv("SEARCHBASE_SEARCH_PROVIDER", "ddgs") t.Setenv("SEARCHBASE_DDGS_PROVIDER_ADDRESS", "http://ddgs-service:8002") + t.Setenv("SEARCHBASE_MCP_HEARTBEAT_ENABLED", "true") + t.Setenv("SEARCHBASE_MCP_HEARTBEAT_INTERVAL", "45") s, err := NewSettings() if err != nil { @@ -74,6 +85,14 @@ func TestNewSettings_EnvVars(t *testing.T) { if s.SearchProvider().Address() != "http://ddgs-service:8002" { t.Errorf("Expected ddgs address http://ddgs-service:8002, got %s", s.SearchProvider().Address()) } + + if !s.Mcp().HeartbeatEnabled() { + t.Error("Expected mcp heartbeat to be enabled") + } + + if s.Mcp().HeartbeatInterval() != 45*time.Second { + t.Errorf("Expected mcp heartbeat interval 45s, got %s", s.Mcp().HeartbeatInterval()) + } } func TestNewSettings_SearchProviderValidationErrors(t *testing.T) { @@ -128,3 +147,95 @@ func TestNewSettings_SearchProviderValidationErrors(t *testing.T) { }) } } + +func TestNewSettings_McpHeartbeatValidation(t *testing.T) { + tests := []struct { + name string + enabled string + interval string + expectedError string + }{ + { + name: "interval below minimum while enabled", + enabled: "true", + interval: "1", + expectedError: "mcp heartbeat interval too low: SEARCHBASE_MCP_HEARTBEAT_INTERVAL must be >= 15s, got 1s", + }, + { + name: "interval just below minimum while enabled", + enabled: "true", + interval: "14", + expectedError: "mcp heartbeat interval too low: SEARCHBASE_MCP_HEARTBEAT_INTERVAL must be >= 15s, got 14s", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("SEARCHBASE_MCP_HEARTBEAT_ENABLED", tt.enabled) + t.Setenv("SEARCHBASE_MCP_HEARTBEAT_INTERVAL", tt.interval) + + _, err := NewSettings() + if err == nil { + t.Fatal("Expected error, got nil") + } + + if err.Error() != tt.expectedError { + t.Fatalf("Expected error %q, got %q", tt.expectedError, err.Error()) + } + }) + } +} + +func TestNewSettings_McpHeartbeatAcceptedIntervals(t *testing.T) { + tests := []struct { + name string + enabled string + interval string + expectedEnabled bool + expectedInterval time.Duration + }{ + { + name: "interval at the minimum while enabled", + enabled: "true", + interval: "15", + expectedEnabled: true, + expectedInterval: 15 * time.Second, + }, + { + name: "enabled without an interval falls back to the default", + enabled: "true", + interval: "", + expectedEnabled: true, + expectedInterval: 60 * time.Second, + }, + { + name: "interval below minimum is ignored while disabled", + enabled: "false", + interval: "1", + expectedEnabled: false, + expectedInterval: 1 * time.Second, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("SEARCHBASE_MCP_HEARTBEAT_ENABLED", tt.enabled) + if tt.interval != "" { + t.Setenv("SEARCHBASE_MCP_HEARTBEAT_INTERVAL", tt.interval) + } + + s, err := NewSettings() + if err != nil { + t.Fatalf("Failed to create settings: %v", err) + } + + if s.Mcp().HeartbeatEnabled() != tt.expectedEnabled { + t.Errorf("Expected mcp heartbeat enabled %t, got %t", tt.expectedEnabled, s.Mcp().HeartbeatEnabled()) + } + + if s.Mcp().HeartbeatInterval() != tt.expectedInterval { + t.Errorf("Expected mcp heartbeat interval %s, got %s", tt.expectedInterval, s.Mcp().HeartbeatInterval()) + } + }) + } +} From afb5902f3803d631d8de2159e5cbc2063aab3144 Mon Sep 17 00:00:00 2001 From: c0lap5o Date: Tue, 8 Sep 2026 21:52:02 +0100 Subject: [PATCH 2/2] Disable SEARCHBASE_MCP heartbeat settings Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 92d6e00..b5d41c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,8 +19,8 @@ services: # set to true here. It keeps idle SSE and streamable HTTP connections alive # for clients that drop them after a shorter read timeout. # The interval is in seconds and must be >= 15 when the heartbeat is enabled. - - SEARCHBASE_MCP_HEARTBEAT_ENABLED=true - - SEARCHBASE_MCP_HEARTBEAT_INTERVAL=60 + # - SEARCHBASE_MCP_HEARTBEAT_ENABLED=true + # - SEARCHBASE_MCP_HEARTBEAT_INTERVAL=60 depends_on: - crawl-worker