fix: tag hedge relays separately so geography doesn't punish slow suppliers

The hedge race fan-out was double-counting in path_relays_total: both the
primary AND the hedge branch's relay completed and called RecordRelay with
request_type="normal", inflating the visible RPS for whichever endpoint
happens to be picked as hedge most often (typically the lowest-latency one).

Symptom observed on canary at 22m post-deploy of #512: rpcgate.xyz with 1
endpoint at 431 RPS while stakenodes.org with 21 endpoints averaged 7.7 RPS
per endpoint. Per-endpoint share was 17x off baseline despite uniform-random
tier-1 selection. The skew traced to (a) hedge metric double-count and (b)
hedge losers getting latency-penalty signals for slowness that only mattered
inside the race they already lost.

This change:
- Adds RelayTypeHedge metric constant
- Adds MarkAsHedge() to ProtocolRequestContext interface
- Hedge branch is tagged before HandleServiceRequest; protocol layer records
  the relay under request_type="hedge" instead of "normal"
- Skips recordLatencyPenaltySignalsIfNeeded for hedge branches — far-from-
  gateway endpoints no longer accumulate slow-response reputation penalties
  for losing hedge races they only entered because of distance
- Reputation success/error signals still fire so endpoints get fair feedback
  about whether they actually work

Dashboards filtering request_type="normal" by default now show fair primary-
traffic distribution. Operators can flip the variable to "hedge" to see the
backup-attempt side independently.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: oten91

gateway/hedge.go

@@ -211,6 +211,12 @@ func (hr *hedgeRacer) race(
 				hedgeDetachedCtx, hedgeDetachedCancel := detachedHedgeCtx(ctx, defaultLoserGraceWindow)
 				hr.hedgeReqCancel = hedgeDetachedCancel
 				hedgeCtx.SetParentContext(hedgeDetachedCtx)
+				// Tag this branch as hedge so the protocol records it under
+				// request_type="hedge" and skips latency-penalty reputation signals.
+				// Without this, fast endpoints picked as hedge accumulate inflated
+				// "normal" relay counts and slow endpoints get latency-penalized for
+				// losing races they only entered because of distance.
+				hedgeCtx.MarkAsHedge()
 				// Start hedge request
 				go hr.executeRequest(ctx, hedgeCtx, hedgeEndpoint, hedgeSupplier, true, 2, time.Now())
 			}

gateway/protocol.go

@@ -179,6 +179,16 @@ type ProtocolRequestContext interface {
 	// than being RST'd the moment the winner is picked and the caller's handler unwinds.
 	SetParentContext(ctx context.Context)
 
+	// MarkAsHedge tags this request as the hedge branch of a hedge race so that
+	// (1) its relay metric is recorded with request_type="hedge" rather than "normal"
+	//     — keeps per-domain RPS dashboards honest by not double-counting fast endpoints
+	//     that win disproportionate hedge races, and
+	// (2) latency-penalty reputation signals are skipped — slow geographies should not
+	//     be penalized for losing hedge races they only entered because of distance.
+	// Reputation success/error signals still fire so endpoints get fair feedback
+	// about whether they actually work.
+	MarkAsHedge()
+
 	// GetObservations builds and returns the set of protocol-specific observations using the current context.
 	//
 	// Hypothetical illustrative example.

metrics/metrics.go

@@ -500,6 +500,12 @@ const (
 	RelayTypeNormal      = "normal"
 	RelayTypeHealthCheck = "health_check"
 	RelayTypeProbation   = "probation"
+	// RelayTypeHedge tags relays issued as the hedge branch of a hedge race —
+	// a backup attempt to a different endpoint when the primary is slow. These
+	// inflate per-domain RPS metrics if grouped with normal traffic, because
+	// fast endpoints are picked as hedge again and again. Dashboards filter to
+	// request_type="normal" by default to show fair primary-traffic distribution.
+	RelayTypeHedge = "hedge"
 )
 
 var RelaysTotal = promauto.NewCounterVec(

protocol/shannon/context.go

@@ -135,6 +135,15 @@ type requestContext struct {
 	// When nil, no reputation-based filtering is applied.
 	reputationService reputation.ReputationService
 
+	// isHedge marks this request as the hedge branch of a hedge race. Set by
+	// MarkAsHedge() before HandleServiceRequest is called. Used at metric/reputation
+	// recording time to:
+	//   - Tag the relay metric as request_type="hedge" (not "normal")
+	//   - Skip latency-penalty reputation signals (don't punish far-from-gateway endpoints)
+	// Success/error reputation signals still fire — endpoints still get fair feedback
+	// about whether they actually work.
+	isHedge bool
+
 	// tieredSelector provides access to tier-based selection and probation status.
 	// Used to check if endpoints are in probation when recording success signals.
 	// If non-nil and endpoint is in probation, RecoverySuccessSignal is used instead of SuccessSignal.
@@ -441,6 +450,13 @@ func (rc *requestContext) SetParentContext(ctx context.Context) {
 	rc.context = ctx
 }
 
+// MarkAsHedge tags this request as the hedge branch of a hedge race.
+// Implements gateway.ProtocolRequestContext. See the field doc on `isHedge`
+// for behavior implications (metric request_type, latency penalty skip).
+func (rc *requestContext) MarkAsHedge() {
+	rc.isHedge = true
+}
+
 // getSelectedEndpoint returns the currently selected endpoint in a thread-safe manner.
 func (rc *requestContext) getSelectedEndpoint() endpoint {
 	rc.selectedEndpointMutex.RLock()
@@ -1212,8 +1228,12 @@ func (rc *requestContext) handleEndpointError(
 		statusCodeStr = metrics.GetStatusCodeCategory(statusCode)
 	}
 
-	// Determine relay type - errors are recorded as normal type (probation detection requires success)
+	// Determine relay type - errors are recorded as normal type (probation detection requires success).
+	// Hedge errors are tagged as hedge so they don't pollute the per-domain RPS view.
 	relayType := metrics.RelayTypeNormal
+	if rc.isHedge {
+		relayType = metrics.RelayTypeHedge
+	}
 	metrics.RecordRelay(domain, rpcTypeStr, string(rc.serviceID), statusCodeStr, reputationSignal, relayType, latency.Seconds())
 
 	// Return the original response (preserving any response bytes) with the error.
@@ -1291,6 +1311,13 @@ func (rc *requestContext) handleEndpointSuccess(
 				Str("endpoint", string(selectedEndpointAddr)).
 				Str("service_id", string(rc.serviceID)).
 				Msg("Backend returned 5xx error - recording critical error signal for reputation")
+		} else if rc.isHedge {
+			// Hedge branch succeeded — record success signal (so the endpoint gets fair
+			// reputation feedback) but tag the relay metric as hedge so it doesn't
+			// inflate per-domain RPS dashboards. Latency penalty is skipped below.
+			signal = reputation.NewSuccessSignal(latency)
+			reputationSignal = metrics.SignalOK
+			relayType = metrics.RelayTypeHedge
 		} else if rc.tieredSelector != nil && rc.tieredSelector.Config().Probation.Enabled && rc.tieredSelector.IsInProbation(endpointKey) {
 			// Probation endpoint succeeded - use recovery signal for stronger positive reinforcement
 			signal = reputation.NewRecoverySuccessSignal(latency)
@@ -1321,8 +1348,11 @@ func (rc *requestContext) handleEndpointSuccess(
 			rc.logger.Warn().Err(err).Msg("Failed to record reputation signal for success")
 		}
 
-		// Record additional latency penalty signals if applicable (only for actual successes)
-		if !isBackend5xx {
+		// Record additional latency penalty signals if applicable (only for actual successes).
+		// Skip for hedge branches: a hedge that's "slow" only failed to win the race —
+		// often because of network distance, not endpoint health. Penalizing it for
+		// latency would systematically drift far-from-gateway endpoints downward.
+		if !isBackend5xx && !rc.isHedge {
 			rc.recordLatencyPenaltySignalsIfNeeded(endpointKey, latency)
 		}
 	} else {
@@ -1332,7 +1362,11 @@ func (rc *requestContext) handleEndpointSuccess(
 		} else {
 			reputationSignal = metrics.SignalOK
 		}
-		relayType = metrics.RelayTypeNormal
+		if rc.isHedge {
+			relayType = metrics.RelayTypeHedge
+		} else {
+			relayType = metrics.RelayTypeNormal
+		}
 	}
 
 	// Record relay metric for successful request

protocol/shannon/hedge_tagging_test.go

@@ -0,0 +1,36 @@
+package shannon
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/pokt-network/path/gateway"
+)
+
+// TestRequestContext_SatisfiesProtocolRequestContext is a compile-time guarantee
+// that *requestContext implements gateway.ProtocolRequestContext, including the
+// new MarkAsHedge method. If this stops compiling, the interface drifted.
+func TestRequestContext_SatisfiesProtocolRequestContext(t *testing.T) {
+	var _ gateway.ProtocolRequestContext = (*requestContext)(nil)
+}
+
+// TestMarkAsHedge_SetsFlag verifies that MarkAsHedge flips the isHedge bit, which
+// is what later gates the relay-type tagging and the latency-penalty skip in
+// handleEndpointSuccess / handleEndpointError.
+func TestMarkAsHedge_SetsFlag(t *testing.T) {
+	rc := &requestContext{}
+	require.False(t, rc.isHedge, "default should be false (non-hedge)")
+	rc.MarkAsHedge()
+	require.True(t, rc.isHedge, "MarkAsHedge must set isHedge=true so downstream relay/reputation paths can branch on it")
+}
+
+// TestMarkAsHedge_Idempotent verifies that calling MarkAsHedge twice is a no-op
+// (no panic, flag stays true). This matters because hedge.go currently calls it
+// once but a future caller might call it defensively.
+func TestMarkAsHedge_Idempotent(t *testing.T) {
+	rc := &requestContext{}
+	rc.MarkAsHedge()
+	rc.MarkAsHedge()
+	require.True(t, rc.isHedge)
+}