From cdff90b52de4593a92502be22a941d4328a35870 Mon Sep 17 00:00:00 2001 From: John Simons Date: Mon, 28 Sep 2026 17:31:45 +1000 Subject: [PATCH 1/2] Fix full text search 1 MB limit, over-length message IDs, and timezone handling in PostgreSQL persistence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `to_tsvector` fails when the vector it builds exceeds 1 MB, so a message body with enough distinct words would fail its entire ingestion batch. The full text index now covers only the first 256 KB of the body, and the search query mirrors that with `Substring`. A new migration (`WidenMessageIdAndCapFullTextBody`) replaces the uncapped index with the capped one and drops the 450-character limit on `message_id` so long IDs are stored in full. Long conversation IDs are truncated to fit the index using a prefix-plus-SHA-256 scheme, keeping messages in different conversations distinguishable even when they share a prefix. The same migration runs on SQL Server as `WidenMessageId` (no full text index change needed there). Non-ASCII characters in headers were being escaped before indexing, so searches for words like "Bestellprüfung" never matched. The JSON serializer now writes letters outside ASCII as-is. Timestamps without a timezone offset are now treated as UTC so Npgsql does not reject them. --- .../FullTextSearchSql.cs | 31 +- .../20260902040422_AddFullTextSearch.cs | 4 +- ...denMessageIdAndCapFullTextBody.Designer.cs | 923 ++++++++++++++++++ ...064706_WidenMessageIdAndCapFullTextBody.cs | 62 ++ ...SqlServiceControlDbContextModelSnapshot.cs | 6 +- .../PostgreSqlFullTextSearchDialect.cs | 2 +- .../20260928064709_WidenMessageId.Designer.cs | 741 ++++++++++++++ .../20260928064709_WidenMessageId.cs | 56 ++ ...verServiceControlDbContextModelSnapshot.cs | 6 +- .../EntityConfigurations/ColumnLengths.cs | 24 + .../FailedErrorImportConfiguration.cs | 2 +- .../FailedMessageConfiguration.cs | 6 +- .../EFRecoverabilityIngestionUnitOfWork.cs | 3 +- .../FailedMessageQueryFilters.cs | 19 +- .../Infrastructure/MessageHeaders.cs | 9 +- .../FullTextSearchIndexTests.cs | 4 +- .../FullTextSearchLimitTests.cs | 30 + .../EFCore/BodyReadTests.cs | 12 + .../EFCore/ErrorIngestionTests.cs | 12 + .../EFCore/FailedErrorImportTests.cs | 12 + .../EFCore/MessagesViewDataStoreTests.cs | 69 ++ .../EFCore/TimeRangeFilterTests.cs | 65 ++ 22 files changed, 2062 insertions(+), 36 deletions(-) create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.Designer.cs create mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.cs create mode 100644 src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchLimitTests.cs create mode 100644 src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs index 1887885ec9..1dc63f1c44 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs @@ -3,10 +3,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql; using Microsoft.EntityFrameworkCore.Migrations.Operations; /// -/// Full text search DDL for the failed messages table. EF Core cannot model a GIN index over an -/// expression, so it is applied by the AddFullTextSearch migration. The statements live here, and -/// not in the migration itself, so that regenerating the migrations with the dotnet-ef CLI only -/// costs a one line migration body. +/// Full text search DDL for the failed messages table. EF Core cannot model a GIN index over an expression, so migrations apply it as SQL: AddFullTextSearch created the index, and WidenMessageIdAndCapFullTextBody replaced it with one that indexes only the start of the body. The statements live here, and not in the migrations themselves, so that regenerating the migrations with the dotnet-ef CLI only costs a one line migration body. /// static class FullTextSearchSql { @@ -17,6 +14,9 @@ static class FullTextSearchSql // stopword removal do more harm than good. public const string Configuration = "simple"; + // to_tsvector fails once the vector it builds passes 1 MB, and the index builds one on every insert, so a body with enough distinct words would fail its whole ingestion batch. Only the start of the body is indexed, which keeps the vector well under that limit. + public const int IndexedBodyLength = 262_144; + // Written the way PostgreSqlFullTextSearchDialect makes EF Core render it, down to the casing // and the redundant looking parentheses: PostgreSQL only uses an expression index when the // query expression parses to the same tree, and a mismatch downgrades search to a sequential @@ -26,12 +26,18 @@ static class FullTextSearchSql // "ServiceControl.MessageFailures.MyMessage" would not otherwise match a search for // "MyMessage". It mirrors the SearchableMessageType that MessageTypeEnricher produces for // RavenDB, and is not the duplicate of the headers it looks like. - public const string IndexedExpression = + public static readonly string IndexedExpression = + $"""to_tsvector('{Configuration}', headers_json || ' ' || substring(COALESCE(body_text, ''), 1, {IndexedBodyLength}) || ' ' || replace(replace(COALESCE(message_type, ''), '.', ' '), '+', ' '))"""; + + // What AddFullTextSearch indexed. WidenMessageIdAndCapFullTextBody replaces it on the way up and restores it on the way down. + const string UncappedIndexedExpression = $"""to_tsvector('{Configuration}', headers_json || ' ' || COALESCE(body_text, '') || ' ' || replace(replace(COALESCE(message_type, ''), '.', ' '), '+', ' '))"""; - public static readonly string Up = CreateIndexSql(null); + public static readonly string CreateUncappedIndex = CreateIndexSql(null, UncappedIndexedExpression); + + public static readonly string CreateIndex = CreateIndexSql(null, IndexedExpression); - public static readonly string Down = DropIndexSql(null); + public static readonly string DropIndex = DropIndexSql(null); /// /// Re-renders the statement the migration carries, this time with the configured schema in it. @@ -42,15 +48,16 @@ static class FullTextSearchSql public static MigrationOperation Rewrite(SqlOperation operation, string schema) => operation.Sql switch { - var sql when sql == Up => WithSql(operation, CreateIndexSql(schema)), - var sql when sql == Down => WithSql(operation, DropIndexSql(schema)), + var sql when sql == CreateUncappedIndex => WithSql(operation, CreateIndexSql(schema, UncappedIndexedExpression)), + var sql when sql == CreateIndex => WithSql(operation, CreateIndexSql(schema, IndexedExpression)), + var sql when sql == DropIndex => WithSql(operation, DropIndexSql(schema)), _ => operation }; - public static bool IsHandled(string sql) => sql == Up || sql == Down; + public static bool IsHandled(string sql) => sql == CreateUncappedIndex || sql == CreateIndex || sql == DropIndex; - static string CreateIndexSql(string? schema) => - $"CREATE INDEX {IndexName} ON {Qualify(schema, TableName)} USING GIN ({IndexedExpression})"; + static string CreateIndexSql(string? schema, string indexedExpression) => + $"CREATE INDEX {IndexName} ON {Qualify(schema, TableName)} USING GIN ({indexedExpression})"; // An index belongs to its table's schema, so it is the index that gets qualified here. static string DropIndexSql(string? schema) => $"DROP INDEX IF EXISTS {Qualify(schema, IndexName)}"; diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260902040422_AddFullTextSearch.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260902040422_AddFullTextSearch.cs index dfcd89e387..e2515b3234 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260902040422_AddFullTextSearch.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260902040422_AddFullTextSearch.cs @@ -8,9 +8,9 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations public partial class AddFullTextSearch : Migration { /// - protected override void Up(MigrationBuilder migrationBuilder) => migrationBuilder.Sql(FullTextSearchSql.Up); + protected override void Up(MigrationBuilder migrationBuilder) => migrationBuilder.Sql(FullTextSearchSql.CreateUncappedIndex); /// - protected override void Down(MigrationBuilder migrationBuilder) => migrationBuilder.Sql(FullTextSearchSql.Down); + protected override void Down(MigrationBuilder migrationBuilder) => migrationBuilder.Sql(FullTextSearchSql.DropIndex); } } diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.Designer.cs new file mode 100644 index 0000000000..ace61e5c47 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.Designer.cs @@ -0,0 +1,923 @@ +// +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlServiceControlDbContext))] + [Migration("20260928064706_WidenMessageIdAndCapFullTextBody")] + partial class WidenMessageIdAndCapFullTextBody + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ArchiveOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("request_id"); + + b.Property("ArchiveType") + .HasColumnType("integer") + .HasColumnName("archive_type"); + + b.Property("OperationType") + .HasColumnType("integer") + .HasColumnName("operation_type"); + + b.Property("CurrentBatch") + .HasColumnType("integer") + .HasColumnName("current_batch"); + + b.Property("GroupName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("group_name"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_id"); + + b.Property("InitiatedByName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_name"); + + b.Property("NumberOfBatches") + .HasColumnType("integer") + .HasColumnName("number_of_batches"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("operation_id"); + + b.Property("Started") + .HasColumnType("timestamp with time zone") + .HasColumnName("started"); + + b.Property("TotalNumberOfMessages") + .HasColumnType("integer") + .HasColumnName("total_number_of_messages"); + + b.HasKey("RequestId", "ArchiveType", "OperationType") + .HasName("pk_archive_operations"); + + b.HasIndex("Started") + .HasDatabaseName("ix_archive_operations_started"); + + b.ToTable("archive_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.CustomCheckEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Category") + .IsRequired() + .HasColumnType("text") + .HasColumnName("category"); + + b.Property("CustomCheckId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("custom_check_id"); + + b.Property("FailureReason") + .HasColumnType("text") + .HasColumnName("failure_reason"); + + b.Property("OriginatingEndpointHost") + .IsRequired() + .HasColumnType("text") + .HasColumnName("originating_endpoint_host"); + + b.Property("OriginatingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("originating_endpoint_host_id"); + + b.Property("OriginatingEndpointName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("originating_endpoint_name"); + + b.Property("ReportedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("reported_at"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_custom_checks"); + + b.HasIndex("ReportedAt") + .HasDatabaseName("ix_custom_checks_reported_at"); + + b.HasIndex("Status", "ReportedAt") + .HasDatabaseName("ix_custom_checks_status_reported_at"); + + b.ToTable("custom_checks", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property("Name") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.Property("TrackInstances") + .HasColumnType("boolean") + .HasColumnName("track_instances"); + + b.HasKey("Name") + .HasName("pk_endpoint_settings"); + + b.ToTable("endpoint_settings", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("event_type"); + + b.Property("RaisedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("raised_at"); + + b.PrimitiveCollection>("RelatedTo") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("related_to"); + + b.Property("Severity") + .HasColumnType("integer") + .HasColumnName("severity"); + + b.HasKey("Id") + .HasName("pk_event_log_items"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending() + .HasDatabaseName("ix_event_log_items_raised_at_id"); + + b.ToTable("event_log_items", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ExternalIntegrationDispatchRequestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DispatchContextJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("dispatch_context_json"); + + b.Property("DispatchContextTypeName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("dispatch_context_type_name"); + + b.HasKey("Id") + .HasName("pk_external_integration_dispatch_requests"); + + b.ToTable("external_integration_dispatch_requests", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("Body") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("body"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("MessageId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_error_imports"); + + b.HasIndex("FailedAt") + .HasDatabaseName("ix_failed_error_imports_failed_at"); + + b.ToTable("failed_error_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEditEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("EditId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("edit_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_edits"); + + b.HasIndex("EditId") + .HasDatabaseName("ix_failed_message_edits_edit_id"); + + b.ToTable("failed_message_edits", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("ExceptionMessage") + .HasColumnType("text") + .HasColumnName("exception_message"); + + b.Property("ExceptionType") + .HasColumnType("text") + .HasColumnName("exception_type"); + + b.Property("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("failing_endpoint_address"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_time_of_failure"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("LastAttemptedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempted_at"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("LastTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_time_of_failure"); + + b.Property("MessageId") + .HasColumnType("text") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_type"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("integer") + .HasColumnName("number_of_processing_attempts"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StatusChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("status_changed_at"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_failed_messages_conversation_id"); + + b.HasIndex("FailingEndpointAddress") + .HasDatabaseName("ix_failed_messages_failing_endpoint_address"); + + b.HasIndex("ReceivingEndpointName") + .HasDatabaseName("ix_failed_messages_receiving_endpoint_name"); + + b.HasIndex("StatusChangedAt") + .HasDatabaseName("ix_failed_messages_status_changed_at") + .HasFilter("status IN (2, 4)"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_failed_messages_time_sent"); + + b.HasIndex("Status", "LastTimeOfFailure") + .HasDatabaseName("ix_failed_messages_status_last_time_of_failure"); + + b.HasIndex("Status", "LastModified", "UniqueMessageId") + .HasDatabaseName("ix_failed_messages_status_last_modified_unique_message_id"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("Status", "LastModified", "UniqueMessageId"), new[] { "FirstTimeOfFailure", "LastTimeOfFailure" }); + + b.HasIndex("Status", "MessageType", "UniqueMessageId") + .HasDatabaseName("ix_failed_messages_status_message_type_unique_message_id"); + + b.ToTable("failed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uuid") + .HasColumnName("failed_message_unique_id"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("type"); + + b.HasKey("FailedMessageUniqueId", "GroupId") + .HasName("pk_failed_message_groups"); + + b.HasIndex("GroupId") + .HasDatabaseName("ix_failed_message_groups_group_id"); + + b.HasIndex("Type", "GroupId") + .HasDatabaseName("ix_failed_message_groups_type_group_id"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("Type", "GroupId"), new[] { "FailedMessageUniqueId", "Title" }); + + b.ToTable("failed_message_groups", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("RetryBatchId") + .HasColumnType("uuid") + .HasColumnName("retry_batch_id"); + + b.Property("StageAttempts") + .HasColumnType("integer") + .HasColumnName("stage_attempts"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_retries"); + + b.HasIndex("RetryBatchId") + .HasDatabaseName("ix_failed_message_retries_retry_batch_id"); + + b.ToTable("failed_message_retries", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.GroupCommentEntity", b => + { + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Comment") + .IsRequired() + .HasColumnType("text") + .HasColumnName("comment"); + + b.HasKey("GroupId") + .HasName("pk_group_comments"); + + b.ToTable("group_comments", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.HistoricRetryOperationEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CompletionTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("completion_time"); + + b.Property("Failed") + .HasColumnType("boolean") + .HasColumnName("failed"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("request_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.HasKey("Id") + .HasName("pk_historic_retry_operations"); + + b.HasIndex("CompletionTime", "Id") + .IsDescending() + .HasDatabaseName("ix_historic_retry_operations_completion_time_id"); + + b.ToTable("historic_retry_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("Monitored") + .HasColumnType("boolean") + .HasColumnName("monitored"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_known_endpoints"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("normalized_name"); + + b.Property("ThroughputSource") + .HasColumnType("integer") + .HasColumnName("throughput_source"); + + b.PrimitiveCollection>("EndpointIndicators") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("endpoint_indicators"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("name"); + + b.Property("NormalizedSanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("normalized_sanitized_name"); + + b.Property("SanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sanitized_name"); + + b.Property("Scope") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("scope"); + + b.Property("UserIndicator") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("user_indicator"); + + b.HasKey("NormalizedName", "ThroughputSource") + .HasName("pk_licensing_endpoints"); + + b.HasIndex("NormalizedSanitizedName") + .HasDatabaseName("ix_licensing_endpoints_normalized_sanitized_name"); + + b.ToTable("licensing_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("normalized_name"); + + b.Property("ThroughputSource") + .HasColumnType("integer") + .HasColumnName("throughput_source"); + + b.Property("DateUtc") + .HasColumnType("date") + .HasColumnName("date_utc"); + + b.Property("MessageCount") + .HasColumnType("bigint") + .HasColumnName("message_count"); + + b.HasKey("NormalizedName", "ThroughputSource", "DateUtc") + .HasName("pk_licensing_endpoint_throughput"); + + b.HasIndex("DateUtc") + .HasDatabaseName("ix_licensing_endpoint_throughput_date_utc"); + + b.ToTable("licensing_endpoint_throughput", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MessageRedirectEntity", b => + { + b.Property("FromPhysicalAddress") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("from_physical_address"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("ToPhysicalAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("to_physical_address"); + + b.HasKey("FromPhysicalAddress") + .HasName("pk_message_redirects"); + + b.ToTable("message_redirects", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Classifier") + .HasColumnType("text") + .HasColumnName("classifier"); + + b.Property("Context") + .HasColumnType("text") + .HasColumnName("context"); + + b.Property("InitialBatchSize") + .HasColumnType("integer") + .HasColumnName("initial_batch_size"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_id"); + + b.Property("InitiatedByName") + .HasColumnType("text") + .HasColumnName("initiated_by_name"); + + b.Property("Last") + .HasColumnType("timestamp with time zone") + .HasColumnName("last"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("operation_id"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("request_id"); + + b.Property("RetrySessionId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("retry_session_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("StagingId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("staging_id"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_retry_batches"); + + b.HasIndex("Status", "RetrySessionId") + .HasDatabaseName("ix_retry_batches_status_retry_session_id"); + + b.ToTable("retry_batches", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchNowForwardingEntity", b => + { + b.Property("Id") + .HasColumnType("integer") + .HasColumnName("id"); + + b.Property("RetryBatchId") + .HasColumnType("uuid") + .HasColumnName("retry_batch_id"); + + b.HasKey("Id") + .HasName("pk_retry_batch_now_forwarding"); + + b.ToTable("retry_batch_now_forwarding", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SettingEntity", b => + { + b.Property("Key") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("key"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text") + .HasColumnName("value"); + + b.HasKey("Key") + .HasName("pk_settings"); + + b.ToTable("settings", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property("MessageType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("message_type"); + + b.Property("TransportAddress") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("transport_address"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("endpoint"); + + b.HasKey("MessageType", "TransportAddress") + .HasName("pk_subscriptions"); + + b.ToTable("subscriptions", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.UnacknowledgedRetryOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("request_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("Classifier") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("classifier"); + + b.Property("CompletionTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("completion_time"); + + b.Property("Failed") + .HasColumnType("boolean") + .HasColumnName("failed"); + + b.Property("Last") + .HasColumnType("timestamp with time zone") + .HasColumnName("last"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.HasKey("RequestId", "RetryType") + .HasName("pk_unacknowledged_retry_operations"); + + b.ToTable("unacknowledged_retry_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_failed_message_groups_failed_messages_failed_message_unique"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", null) + .WithMany() + .HasForeignKey("NormalizedName", "ThroughputSource") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_licensing_endpoint_throughput_licensing_endpoints_normalize"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.cs new file mode 100644 index 0000000000..1f1f47fc20 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.cs @@ -0,0 +1,62 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + /// + public partial class WidenMessageIdAndCapFullTextBody : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "message_id", + table: "failed_messages", + type: "text", + nullable: true, + oldClrType: typeof(string), + oldType: "character varying(450)", + oldMaxLength: 450, + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "message_id", + table: "failed_error_imports", + type: "text", + nullable: false, + oldClrType: typeof(string), + oldType: "character varying(450)", + oldMaxLength: 450); + + migrationBuilder.Sql(FullTextSearchSql.DropIndex); + migrationBuilder.Sql(FullTextSearchSql.CreateIndex); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(FullTextSearchSql.DropIndex); + migrationBuilder.Sql(FullTextSearchSql.CreateUncappedIndex); + + migrationBuilder.AlterColumn( + name: "message_id", + table: "failed_messages", + type: "character varying(450)", + maxLength: 450, + nullable: true, + oldClrType: typeof(string), + oldType: "text", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "message_id", + table: "failed_error_imports", + type: "character varying(450)", + maxLength: 450, + nullable: false, + oldClrType: typeof(string), + oldType: "text"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index 06b1a9b639..3403a3d831 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -264,8 +264,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("MessageId") .IsRequired() - .HasMaxLength(450) - .HasColumnType("character varying(450)") + .HasColumnType("text") .HasColumnName("message_id"); b.HasKey("UniqueMessageId") @@ -366,8 +365,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnName("last_time_of_failure"); b.Property("MessageId") - .HasMaxLength(450) - .HasColumnType("character varying(450)") + .HasColumnType("text") .HasColumnName("message_id"); b.Property("MessageType") diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs index d2c01fb908..8461a8a12c 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs @@ -15,7 +15,7 @@ public IQueryable Search(IQueryable so source.Where(message => EF.Functions.ToTsVector(FullTextSearchSql.Configuration, message.HeadersJson + " " + - (message.BodyText ?? "") + " " + + (message.BodyText ?? "").Substring(0, FullTextSearchSql.IndexedBodyLength) + " " + (message.MessageType ?? "").Replace(".", " ").Replace("+", " ")) .Matches(EF.Functions.WebSearchToTsQuery(FullTextSearchSql.Configuration, ToOrQuery(searchTerms)))); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.Designer.cs new file mode 100644 index 0000000000..704d62a08e --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.Designer.cs @@ -0,0 +1,741 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerServiceControlDbContext))] + [Migration("20260928064709_WidenMessageId")] + partial class WidenMessageId + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ArchiveOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ArchiveType") + .HasColumnType("int"); + + b.Property("OperationType") + .HasColumnType("int"); + + b.Property("CurrentBatch") + .HasColumnType("int"); + + b.Property("GroupName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("InitiatedByName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("NumberOfBatches") + .HasColumnType("int"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Started") + .HasColumnType("datetime2"); + + b.Property("TotalNumberOfMessages") + .HasColumnType("int"); + + b.HasKey("RequestId", "ArchiveType", "OperationType"); + + b.HasIndex("Started"); + + b.ToTable("ArchiveOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.CustomCheckEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Category") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("CustomCheckId") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FailureReason") + .HasColumnType("nvarchar(max)"); + + b.Property("OriginatingEndpointHost") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OriginatingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("OriginatingEndpointName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReportedAt") + .HasColumnType("datetime2"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ReportedAt"); + + b.HasIndex("Status", "ReportedAt"); + + b.ToTable("CustomChecks"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property("Name") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("TrackInstances") + .HasColumnType("bit"); + + b.HasKey("Name"); + + b.ToTable("EndpointSettings"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Description") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RaisedAt") + .HasColumnType("datetime2"); + + b.PrimitiveCollection("RelatedTo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Severity") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending(); + + b.ToTable("EventLogItems"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ExternalIntegrationDispatchRequestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("DispatchContextJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DispatchContextTypeName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("ExternalIntegrationDispatchRequests"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FailedAt") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageId") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("FailedAt"); + + b.ToTable("FailedErrorImports"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEditEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("EditId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("EditId"); + + b.ToTable("FailedMessageEdits"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ExceptionMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("ExceptionType") + .HasColumnType("nvarchar(max)"); + + b.Property("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("LastAttemptedAt") + .HasColumnType("datetime2"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("LastTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("MessageId") + .HasColumnType("nvarchar(max)"); + + b.Property("MessageType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("int"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("StatusChangedAt") + .HasColumnType("datetime2"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("ConversationId"); + + b.HasIndex("FailingEndpointAddress"); + + b.HasIndex("ReceivingEndpointName"); + + b.HasIndex("StatusChangedAt") + .HasFilter("[Status] IN (2, 4)"); + + b.HasIndex("TimeSent"); + + b.HasIndex("Status", "LastModified"); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("Status", "LastModified"), new[] { "FirstTimeOfFailure", "LastTimeOfFailure" }); + + b.HasIndex("Status", "LastTimeOfFailure"); + + b.HasIndex("Status", "MessageType", "UniqueMessageId"); + + b.ToTable("FailedMessages"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uniqueidentifier"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("FailedMessageUniqueId", "GroupId"); + + b.HasIndex("GroupId"); + + b.HasIndex("Type", "GroupId"); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("Type", "GroupId"), new[] { "Title" }); + + b.ToTable("FailedMessageGroups"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("RetryBatchId") + .HasColumnType("uniqueidentifier"); + + b.Property("StageAttempts") + .HasColumnType("int"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("RetryBatchId"); + + b.ToTable("FailedMessageRetries"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.GroupCommentEntity", b => + { + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Comment") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("GroupId"); + + b.ToTable("GroupComments"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.HistoricRetryOperationEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CompletionTime") + .HasColumnType("datetime2"); + + b.Property("Failed") + .HasColumnType("bit"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CompletionTime", "Id") + .IsDescending(); + + b.ToTable("HistoricRetryOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("Monitored") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("KnownEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("ThroughputSource") + .HasColumnType("int"); + + b.PrimitiveCollection("EndpointIndicators") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("NormalizedSanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Scope") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("UserIndicator") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("NormalizedName", "ThroughputSource"); + + b.HasIndex("NormalizedSanitizedName"); + + b.ToTable("LicensingEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("ThroughputSource") + .HasColumnType("int"); + + b.Property("DateUtc") + .HasColumnType("date"); + + b.Property("MessageCount") + .HasColumnType("bigint"); + + b.HasKey("NormalizedName", "ThroughputSource", "DateUtc"); + + b.HasIndex("DateUtc"); + + b.ToTable("LicensingEndpointThroughput"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MessageRedirectEntity", b => + { + b.Property("FromPhysicalAddress") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("ToPhysicalAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("FromPhysicalAddress"); + + b.ToTable("MessageRedirects"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Classifier") + .HasColumnType("nvarchar(max)"); + + b.Property("Context") + .HasColumnType("nvarchar(max)"); + + b.Property("InitialBatchSize") + .HasColumnType("int"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("InitiatedByName") + .HasColumnType("nvarchar(max)"); + + b.Property("Last") + .HasColumnType("datetime2"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RetrySessionId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("StagingId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Status", "RetrySessionId"); + + b.ToTable("RetryBatches"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchNowForwardingEntity", b => + { + b.Property("Id") + .HasColumnType("int"); + + b.Property("RetryBatchId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.ToTable("RetryBatchNowForwarding"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SettingEntity", b => + { + b.Property("Key") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Value") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property("MessageType") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("TransportAddress") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("MessageType", "TransportAddress"); + + b.ToTable("Subscriptions"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.UnacknowledgedRetryOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("Classifier") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("CompletionTime") + .HasColumnType("datetime2"); + + b.Property("Failed") + .HasColumnType("bit"); + + b.Property("Last") + .HasColumnType("datetime2"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.HasKey("RequestId", "RetryType"); + + b.ToTable("UnacknowledgedRetryOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", null) + .WithMany() + .HasForeignKey("NormalizedName", "ThroughputSource") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.cs new file mode 100644 index 0000000000..3f5dc955aa --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.cs @@ -0,0 +1,56 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + /// + public partial class WidenMessageId : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "MessageId", + table: "FailedMessages", + type: "nvarchar(max)", + nullable: true, + oldClrType: typeof(string), + oldType: "nvarchar(450)", + oldMaxLength: 450, + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "MessageId", + table: "FailedErrorImports", + type: "nvarchar(max)", + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(450)", + oldMaxLength: 450); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "MessageId", + table: "FailedMessages", + type: "nvarchar(450)", + maxLength: 450, + nullable: true, + oldClrType: typeof(string), + oldType: "nvarchar(max)", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "MessageId", + table: "FailedErrorImports", + type: "nvarchar(450)", + maxLength: 450, + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(max)"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index b9e41a3681..72f4d45511 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -215,8 +215,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("MessageId") .IsRequired() - .HasMaxLength(450) - .HasColumnType("nvarchar(450)"); + .HasColumnType("nvarchar(max)"); b.HasKey("UniqueMessageId"); @@ -295,8 +294,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("datetime2"); b.Property("MessageId") - .HasMaxLength(450) - .HasColumnType("nvarchar(450)"); + .HasColumnType("nvarchar(max)"); b.Property("MessageType") .HasMaxLength(450) diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs index bf40c72369..adfcc0c73f 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs @@ -1,5 +1,9 @@ namespace ServiceControl.Persistence.EFCore.EntityConfigurations; +using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; +using System.Text; + static class ColumnLengths { // Indexed and short-by-nature values get a length so that SQL Server can index them, @@ -23,4 +27,24 @@ static class ColumnLengths // int, and the throughput key adds a date on top, so ShortTextLength would exceed SQL Server's // 900 byte index key limit. public const int LicensingEndpointNameLength = 300; + + // Cutting an indexed value short could make two different values look up as equal, so a longer value keeps as much of its start as fits, followed by # and the SHA-256 of the whole value. The result is never longer than ShortTextLength, so fitting it again returns it unchanged. + [return: NotNullIfNotNull(nameof(value))] + public static string? FitToIndex(string? value) + { + if (value is null || value.Length <= ShortTextLength) + { + return value; + } + + var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + var prefixLength = ShortTextLength - hash.Length - 1; + + if (char.IsHighSurrogate(value[prefixLength - 1])) + { + prefixLength--; + } + + return $"{value[..prefixLength]}#{hash}"; + } } diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedErrorImportConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedErrorImportConfiguration.cs index 44c46981cc..e1d806fe0f 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedErrorImportConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedErrorImportConfiguration.cs @@ -12,7 +12,7 @@ public void Configure(EntityTypeBuilder builder) builder.Property(e => e.UniqueMessageId).ValueGeneratedNever(); builder.Property(e => e.FailedAt).IsRequired(); - builder.Property(e => e.MessageId).IsRequired().HasMaxLength(ColumnLengths.ShortTextLength); + builder.Property(e => e.MessageId).IsRequired(); builder.Property(e => e.HeadersJson).IsRequired(); builder.Property(e => e.Body).IsRequired(); builder.Property(e => e.BodyStoredExternally).IsRequired(); diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs index 1bff26175c..33ea408820 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs @@ -19,9 +19,11 @@ public void Configure(EntityTypeBuilder builder) builder.Property(e => e.LastTimeOfFailure).IsRequired(); builder.Property(e => e.LastAttemptedAt).IsRequired(); - builder.Property(e => e.MessageId).HasMaxLength(ColumnLengths.ShortTextLength); builder.Property(e => e.MessageType).HasMaxLength(ColumnLengths.ShortTextLength); - builder.Property(e => e.ConversationId).HasMaxLength(ColumnLengths.ShortTextLength); + // The converter also runs on every value a query compares with the column, so a lookup by the full id finds the stored form. + builder.Property(e => e.ConversationId) + .HasMaxLength(ColumnLengths.ShortTextLength) + .HasConversion(value => ColumnLengths.FitToIndex(value), value => value); builder.Property(e => e.SendingEndpointName).HasMaxLength(ColumnLengths.ShortTextLength); builder.Property(e => e.SendingEndpointHost).HasMaxLength(ColumnLengths.ShortTextLength); builder.Property(e => e.ReceivingEndpointName).HasMaxLength(ColumnLengths.ShortTextLength); diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs index 43a0e7c927..2b9f37cab5 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs @@ -41,7 +41,8 @@ public Task RecordFailedProcessingAttempt(MessageContext context, MessageId = processingAttempt.MessageId, MessageType = TruncateTypeName(GetMetadata(processingAttempt, "MessageType")), TimeSent = GetMetadata(processingAttempt, "TimeSent"), - ConversationId = GetMetadata(processingAttempt, "ConversationId"), + // The ingestion upsert is hand-written SQL, which skips the value converter on the column. + ConversationId = ColumnLengths.FitToIndex(GetMetadata(processingAttempt, "ConversationId")), SendingEndpointName = sendingEndpoint?.Name, SendingEndpointHostId = sendingEndpoint?.HostId, SendingEndpointHost = sendingEndpoint?.Host, diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/FailedMessageQueryFilters.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/FailedMessageQueryFilters.cs index d70d643a9c..68ab57ff40 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/FailedMessageQueryFilters.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/FailedMessageQueryFilters.cs @@ -72,8 +72,8 @@ public static IQueryable FilterByLastModifiedRange(this IQu try { - from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - to = DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + from = AsUtc(DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)); + to = AsUtc(DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)); } catch (Exception) { @@ -116,17 +116,28 @@ public static IQueryable FilterBySentTimeRange(this IQuerya { if (timeSentRange?.From is { } from) { - source = source.Where(message => message.TimeSent >= from); + var fromUtc = AsUtc(from); + source = source.Where(message => message.TimeSent >= fromUtc); } if (timeSentRange?.To is { } to) { - source = source.Where(message => message.TimeSent <= to); + var toUtc = AsUtc(to); + source = source.Where(message => message.TimeSent <= toUtc); } return source; } + // Npgsql refuses to write a Local or Unspecified value to timestamptz. A value without a zone is taken as UTC, as SQL Server's UtcDateTimeConverter already does. + static DateTime AsUtc(DateTime value) => value.Kind switch + { + DateTimeKind.Utc => value, + DateTimeKind.Local => value.ToUniversalTime(), + DateTimeKind.Unspecified => DateTime.SpecifyKind(value, DateTimeKind.Utc), + _ => throw new ArgumentOutOfRangeException(nameof(value), value.Kind, "Unknown DateTimeKind") + }; + public static IQueryable IncludeSystemMessagesWhere(this IQueryable source, bool includeSystemMessages) => includeSystemMessages ? source : source.Where(message => !message.IsSystemMessage); diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs index c8c66d83c2..17621c29aa 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs @@ -1,16 +1,21 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; +using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.Unicode; // The headers of a failed message are stored verbatim as the HeadersJson column. static class MessageHeaders { public static string Write(Dictionary headers) => - JsonSerializer.Serialize(headers, HeadersJsonContext.Default.DictionaryStringString); + JsonSerializer.Serialize(headers, context.DictionaryStringString); public static Dictionary Read(string headersJson) => - JsonSerializer.Deserialize(headersJson, HeadersJsonContext.Default.DictionaryStringString) ?? []; + JsonSerializer.Deserialize(headersJson, context.DictionaryStringString) ?? []; + + // The default encoder escapes every non-ASCII character, and full text search then indexes the escape sequence instead of the word. This one writes letters outside ASCII as they are, while HTML sensitive characters and emoji stay escaped. + static readonly HeadersJsonContext context = new(new JsonSerializerOptions { Encoder = JavaScriptEncoder.Create(UnicodeRanges.All) }); } // Source generated serialization, which keeps the reflection-based serializer off the ingestion hot path. diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs index cbfeafd87b..29686a18dc 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs @@ -6,9 +6,7 @@ namespace ServiceControl.Persistence.Tests; using NUnit.Framework; /// -/// PostgreSQL only uses the GIN index of the AddFullTextSearch migration when the query expression -/// parses to the same tree as the indexed one. A mismatch is silent: search keeps working, on a -/// sequential scan of every failed message. These tests need no database. +/// PostgreSQL only uses the full text GIN index when the query expression parses to the same tree as the indexed one. A mismatch is silent: search keeps working, on a sequential scan of every failed message. These tests need no database. /// class FullTextSearchIndexTests { diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchLimitTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchLimitTests.cs new file mode 100644 index 0000000000..e52c54b805 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchLimitTests.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.Infrastructure; + +class FullTextSearchLimitTests : ErrorIngestionTestBase +{ + const int TokenCount = 150_000; + + [SetUp] + public void StoreLargeBodiesInline() => EFSettings.BodyStorage.MaxBodySizeToStore = 2 * 1024 * 1024; + + [Test] + public async Task Ingests_and_searches_a_body_with_more_distinct_words_than_a_tsvector_can_hold() + { + var body = string.Join(' ', Enumerable.Range(0, TokenCount).Select(token => $"w{token:D7}")); + var failure = new IngestedFailure { ContentType = "text/plain", Body = Encoding.UTF8.GetBytes(body) }; + + Assert.That(failure.Body, Has.Length.GreaterThan(1024 * 1024), "The body has to be larger than the 1 MB a tsvector can hold"); + + await Ingest(failure); + + var result = await MessagesViewStore.GetAllMessagesForSearch("w0000001", new PagingInfo(), new SortInfo()); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { failure.UniqueMessageIdString })); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/BodyReadTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/BodyReadTests.cs index 7a2996c262..527217eff0 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/BodyReadTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/BodyReadTests.cs @@ -79,6 +79,18 @@ public async Task Fetches_by_message_id() Assert.That(result.State, Is.EqualTo(MessageBodyState.Available)); } + [Test] + public async Task Fetches_by_an_over_length_message_id() + { + var failure = new IngestedFailure { MessageId = new string('m', 600), Body = Encoding.UTF8.GetBytes("1") }; + await Ingest(failure); + + var result = await Fetch(failure.MessageId); + + Assert.That(result, Is.Not.Null); + Assert.That(result.State, Is.EqualTo(MessageBodyState.Available)); + } + [Test] public async Task Reports_an_empty_body() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs index b926be9e25..243d01980b 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs @@ -81,6 +81,18 @@ public async Task An_over_length_message_type_is_capped_in_the_column_but_kept_i } } + [Test] + public async Task An_over_length_message_id_is_stored_in_full() + { + var failure = new IngestedFailure { MessageId = new string('m', 600) }; + + await Ingest(failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + Assert.That(row.MessageId, Is.EqualTo(failure.MessageId)); + } + [Test] public async Task Later_attempt_replaces_the_stored_attempt() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/FailedErrorImportTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/FailedErrorImportTests.cs index 259aa61e0e..995a2e83c3 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/FailedErrorImportTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/FailedErrorImportTests.cs @@ -36,6 +36,18 @@ public async Task Stores_and_replays_a_failed_import() Assert.That(await FailedImportStore.QueryContainsFailedImports(), Is.False); } + [Test] + public async Task Stores_and_replays_a_failed_import_with_an_over_length_message_id() + { + var nativeId = new string('n', 600); + + await StoreImport(WellFormedHeaders(), Encoding.UTF8.GetBytes("1"), nativeId: nativeId); + + var replayed = await Replay(); + + Assert.That(replayed.Select(message => message.Id), Is.EqualTo(new[] { nativeId })); + } + [Test] public async Task Round_trips_a_binary_body_with_nul_bytes() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs index cf680e2418..98819f4b66 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs @@ -231,6 +231,65 @@ public async Task Keeps_system_messages_in_a_conversation() Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { system.UniqueMessageIdString })); } + [Test] + public async Task Filters_by_an_over_length_conversation() + { + var inConversation = new IngestedFailure { ConversationId = new string('c', 600) }; + + await Ingest(inConversation, new IngestedFailure()); + + var result = await MessagesViewStore.GetAllMessagesByConversation(inConversation.ConversationId, new PagingInfo(), new SortInfo(), false); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inConversation.UniqueMessageIdString })); + } + + [Test] + public async Task Filters_by_the_reported_id_of_an_over_length_conversation() + { + var inConversation = new IngestedFailure { ConversationId = new string('c', 600) }; + + await Ingest(inConversation, new IngestedFailure()); + + var all = await MessagesViewStore.GetAllMessages(new PagingInfo(), new SortInfo(), true); + var reportedId = all.Results.Single(view => view.Id == inConversation.UniqueMessageIdString).ConversationId; + + var result = await MessagesViewStore.GetAllMessagesByConversation(reportedId, new PagingInfo(), new SortInfo(), false); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inConversation.UniqueMessageIdString })); + } + + [Test] + public async Task Keeps_over_length_conversations_that_share_a_prefix_apart() + { + var sharedPrefix = new string('c', 450); + var first = new IngestedFailure { ConversationId = sharedPrefix + new string('1', 150) }; + var second = new IngestedFailure { ConversationId = sharedPrefix + new string('2', 150) }; + + await Ingest(first, second); + + var firstConversation = await MessagesViewStore.GetAllMessagesByConversation(first.ConversationId, new PagingInfo(), new SortInfo(), false); + var secondConversation = await MessagesViewStore.GetAllMessagesByConversation(second.ConversationId, new PagingInfo(), new SortInfo(), false); + + using (Assert.EnterMultipleScope()) + { + Assert.That(firstConversation.Results.Select(view => view.Id), Is.EqualTo(new[] { first.UniqueMessageIdString })); + Assert.That(secondConversation.Results.Select(view => view.Id), Is.EqualTo(new[] { second.UniqueMessageIdString })); + } + } + + [Test] + public async Task Keeps_a_surrogate_pair_whole_when_shortening_a_conversation_id() + { + var conversationId = new string('c', 384) + char.ConvertFromUtf32(0x1F600) + new string('c', 214); + var failure = new IngestedFailure { ConversationId = conversationId }; + + await Ingest(failure); + + var view = await SingleMessage(); + + Assert.That(view.ConversationId, Does.StartWith(new string('c', 384) + "#"), "The pair straddles the end of the kept prefix, so it has to be left out whole"); + } + [Test] public async Task Searches_the_headers() { @@ -241,6 +300,16 @@ public async Task Searches_the_headers() await AssertSearchFinds("zarquon", matching); } + [Test] + public async Task Searches_a_non_ascii_word_in_the_headers() + { + var matching = new IngestedFailure { ExceptionMessage = "Bestellprüfung fehlgeschlagen" }; + + await Ingest(matching, new IngestedFailure()); + + await AssertSearchFinds("Bestellprüfung", matching); + } + [Test] public async Task Searches_the_body() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs new file mode 100644 index 0000000000..04f038a7e6 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs @@ -0,0 +1,65 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.Infrastructure; + +class TimeRangeFilterTests : ErrorIngestionTestBase +{ + [Test] + public async Task Filters_by_a_time_sent_range_with_an_offset() + { + var before = new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 9, 58, 0, DateTimeKind.Utc) }; + var inside = new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 10, 1, 0, DateTimeKind.Utc) }; + var after = new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 10, 4, 0, DateTimeKind.Utc) }; + + await Ingest(before, inside, after); + + var result = await MessagesViewStore.GetAllMessages(new PagingInfo(), new SortInfo(), true, + new DateTimeRange("2026-07-22T12:00:00+02:00", "2026-07-22T12:03:00+02:00")); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside.UniqueMessageIdString })); + } + + [Test] + public async Task Filters_by_a_modified_range_with_an_offset() + { + var (inside, from, to) = await IngestThreeModifiedTwoMinutesApart(); + + var result = await FailedMessageQueryStore.GetFailedMessages(null, $"{WithOffset(from)}...{WithOffset(to)}", null, new PagingInfo(), new SortInfo()); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside })); + } + + [Test] + public async Task Filters_by_a_modified_range_without_a_zone_as_utc() + { + var (inside, from, to) = await IngestThreeModifiedTwoMinutesApart(); + + var result = await FailedMessageQueryStore.GetFailedMessages(null, $"{WithoutZone(from)}...{WithoutZone(to)}", null, new PagingInfo(), new SortInfo()); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside })); + } + + async Task<(string Inside, DateTime From, DateTime To)> IngestThreeModifiedTwoMinutesApart() + { + var start = Now; + var inside = new IngestedFailure(); + + await Ingest(new IngestedFailure()); + AdvanceClock(TimeSpan.FromMinutes(2)); + await Ingest(inside); + AdvanceClock(TimeSpan.FromMinutes(2)); + await Ingest(new IngestedFailure()); + + return (inside.UniqueMessageIdString, start.AddMinutes(1), start.AddMinutes(3)); + } + + static string WithOffset(DateTime utc) => + new DateTimeOffset(utc).ToOffset(TimeSpan.FromHours(2)).ToString("yyyy-MM-ddTHH:mm:ss.fffffffzzz", CultureInfo.InvariantCulture); + + static string WithoutZone(DateTime utc) => utc.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture); +} From 8c4324123a0bb05c182e2a10da5eb1c17f02483c Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 30 Sep 2026 10:31:14 +1000 Subject: [PATCH 2/2] Fix UTC datetime handling, URL-safe conversation ID truncation, and quoted-word search Move `UtcDateTimeConverter` to the shared `ServiceControlDbContext` so both SQL Server and PostgreSQL apply it to query parameters, removing the need to manually call `AsUtc` before filtering on date ranges. Timestamps without a zone are now treated as UTC rather than rejected by Npgsql. Switch the over-length conversation ID separator from `#` to `~` so truncated IDs need no URL encoding when ServicePulse embeds them in a URL path. Switch JSON header encoding to `UnsafeRelaxedJsonEscaping` so apostrophes, plus signs, and non-ASCII letters are written literally; the previous encoder caused full text search to index escape sequences instead of the words they represented. --- .../SqlServerServiceControlDbContext.cs | 9 --- .../UtcDateTimeConverters.cs | 14 ---- .../DbContexts/ServiceControlDbContext.cs | 8 ++ .../DbContexts/UtcDateTimeConverters.cs | 16 ++++ .../EntityConfigurations/ColumnLengths.cs | 4 +- .../FailedMessageQueryFilters.cs | 19 +---- .../Infrastructure/MessageHeaders.cs | 5 +- .../EFCore/MessagesViewDataStoreTests.cs | 22 +++++- .../EFCore/TimeRangeFilterTests.cs | 77 +++++++++++++++++-- 9 files changed, 124 insertions(+), 50 deletions(-) delete mode 100644 src/ServiceControl.Persistence.EFCore.SqlServer/UtcDateTimeConverters.cs create mode 100644 src/ServiceControl.Persistence.EFCore/DbContexts/UtcDateTimeConverters.cs diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs index 90ce6a48e8..91e578f75f 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; -using System; using EntityConfigurations; using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; @@ -10,14 +9,6 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; public class SqlServerServiceControlDbContext(DbContextOptions options) : ServiceControlDbContext(options) { - protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) - { - base.ConfigureConventions(configurationBuilder); - - configurationBuilder.Properties().HaveConversion(); - configurationBuilder.Properties().HaveConversion(); - } - protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/UtcDateTimeConverters.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/UtcDateTimeConverters.cs deleted file mode 100644 index e464acc5b6..0000000000 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/UtcDateTimeConverters.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace ServiceControl.Persistence.EFCore.SqlServer; - -using System; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; - -// datetime2 stores no offset, so values come back as DateTimeKind.Unspecified and are then serialized -// by the API without the UTC marker. Everything persisted here is UTC. -sealed class UtcDateTimeConverter() : ValueConverter( - value => value.Kind == DateTimeKind.Local ? value.ToUniversalTime() : value, - value => DateTime.SpecifyKind(value, DateTimeKind.Utc)); - -sealed class NullableUtcDateTimeConverter() : ValueConverter( - value => value.HasValue && value.Value.Kind == DateTimeKind.Local ? value.Value.ToUniversalTime() : value, - value => value.HasValue ? DateTime.SpecifyKind(value.Value, DateTimeKind.Utc) : value); diff --git a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs index 3717c6a7f3..d9a40b9b14 100644 --- a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs @@ -38,6 +38,14 @@ public abstract class ServiceControlDbContext(DbContextOptions options) : DbCont protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => optionsBuilder.EnableDetailedErrors(); + protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) + { + base.ConfigureConventions(configurationBuilder); + + configurationBuilder.Properties().HaveConversion(); + configurationBuilder.Properties().HaveConversion(); + } + protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); diff --git a/src/ServiceControl.Persistence.EFCore/DbContexts/UtcDateTimeConverters.cs b/src/ServiceControl.Persistence.EFCore/DbContexts/UtcDateTimeConverters.cs new file mode 100644 index 0000000000..8f0cc514dd --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DbContexts/UtcDateTimeConverters.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.EFCore.DbContexts; + +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +// SQL Server's datetime2 has no offset, so values read back as Unspecified. Npgsql refuses to write a Local or Unspecified value to timestamptz. EF applies these converters to query parameters as well, so a date range from the API needs no conversion of its own. +sealed class UtcDateTimeConverter() : ValueConverter( + value => ToUtc(value), + value => DateTime.SpecifyKind(value, DateTimeKind.Utc)) +{ + public static DateTime ToUtc(DateTime value) => + value.Kind == DateTimeKind.Local ? value.ToUniversalTime() : DateTime.SpecifyKind(value, DateTimeKind.Utc); +} + +sealed class NullableUtcDateTimeConverter() : ValueConverter( + value => value.HasValue ? UtcDateTimeConverter.ToUtc(value.Value) : value, + value => value.HasValue ? DateTime.SpecifyKind(value.Value, DateTimeKind.Utc) : value); diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs index adfcc0c73f..f86bedace8 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs @@ -28,7 +28,7 @@ static class ColumnLengths // 900 byte index key limit. public const int LicensingEndpointNameLength = 300; - // Cutting an indexed value short could make two different values look up as equal, so a longer value keeps as much of its start as fits, followed by # and the SHA-256 of the whole value. The result is never longer than ShortTextLength, so fitting it again returns it unchanged. + // Cutting an indexed value short could make two different values look up as equal, so a longer value keeps as much of its start as fits, followed by ~ and the SHA-256 of the whole value. The result is never longer than ShortTextLength, so fitting it again returns it unchanged. [return: NotNullIfNotNull(nameof(value))] public static string? FitToIndex(string? value) { @@ -45,6 +45,6 @@ static class ColumnLengths prefixLength--; } - return $"{value[..prefixLength]}#{hash}"; + return $"{value[..prefixLength]}~{hash}"; } } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/FailedMessageQueryFilters.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/FailedMessageQueryFilters.cs index 68ab57ff40..d70d643a9c 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/FailedMessageQueryFilters.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/FailedMessageQueryFilters.cs @@ -72,8 +72,8 @@ public static IQueryable FilterByLastModifiedRange(this IQu try { - from = AsUtc(DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)); - to = AsUtc(DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)); + from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + to = DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); } catch (Exception) { @@ -116,28 +116,17 @@ public static IQueryable FilterBySentTimeRange(this IQuerya { if (timeSentRange?.From is { } from) { - var fromUtc = AsUtc(from); - source = source.Where(message => message.TimeSent >= fromUtc); + source = source.Where(message => message.TimeSent >= from); } if (timeSentRange?.To is { } to) { - var toUtc = AsUtc(to); - source = source.Where(message => message.TimeSent <= toUtc); + source = source.Where(message => message.TimeSent <= to); } return source; } - // Npgsql refuses to write a Local or Unspecified value to timestamptz. A value without a zone is taken as UTC, as SQL Server's UtcDateTimeConverter already does. - static DateTime AsUtc(DateTime value) => value.Kind switch - { - DateTimeKind.Utc => value, - DateTimeKind.Local => value.ToUniversalTime(), - DateTimeKind.Unspecified => DateTime.SpecifyKind(value, DateTimeKind.Utc), - _ => throw new ArgumentOutOfRangeException(nameof(value), value.Kind, "Unknown DateTimeKind") - }; - public static IQueryable IncludeSystemMessagesWhere(this IQueryable source, bool includeSystemMessages) => includeSystemMessages ? source : source.Where(message => !message.IsSystemMessage); diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs index 17621c29aa..fc3276dea5 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs @@ -3,7 +3,6 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; -using System.Text.Unicode; // The headers of a failed message are stored verbatim as the HeadersJson column. static class MessageHeaders @@ -14,8 +13,8 @@ public static string Write(Dictionary headers) => public static Dictionary Read(string headersJson) => JsonSerializer.Deserialize(headersJson, context.DictionaryStringString) ?? []; - // The default encoder escapes every non-ASCII character, and full text search then indexes the escape sequence instead of the word. This one writes letters outside ASCII as they are, while HTML sensitive characters and emoji stay escaped. - static readonly HeadersJsonContext context = new(new JsonSerializerOptions { Encoder = JavaScriptEncoder.Create(UnicodeRanges.All) }); + // The relaxed encoder is only unsafe for JSON embedded in HTML, and this JSON never reaches HTML. Full text search needs apostrophes, plus signs and non-ASCII letters written as they are, or it cannot find the words next to them. + static readonly HeadersJsonContext context = new(new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }); } // Source generated serialization, which keeps the reflection-based serializer off the ingestion hot path. diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs index 98819f4b66..8656464871 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs @@ -258,6 +258,16 @@ public async Task Filters_by_the_reported_id_of_an_over_length_conversation() Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inConversation.UniqueMessageIdString })); } + [Test] + public async Task Reports_an_over_length_conversation_id_that_needs_no_url_encoding() + { + await Ingest(new IngestedFailure { ConversationId = new string('c', 600) }); + + var view = await SingleMessage(); + + Assert.That(Uri.EscapeDataString(view.ConversationId), Is.EqualTo(view.ConversationId), "ServicePulse puts the conversation id into a URL path without encoding it"); + } + [Test] public async Task Keeps_over_length_conversations_that_share_a_prefix_apart() { @@ -287,7 +297,7 @@ public async Task Keeps_a_surrogate_pair_whole_when_shortening_a_conversation_id var view = await SingleMessage(); - Assert.That(view.ConversationId, Does.StartWith(new string('c', 384) + "#"), "The pair straddles the end of the kept prefix, so it has to be left out whole"); + Assert.That(view.ConversationId, Does.StartWith(new string('c', 384) + "~"), "The pair straddles the end of the kept prefix, so it has to be left out whole"); } [Test] @@ -310,6 +320,16 @@ public async Task Searches_a_non_ascii_word_in_the_headers() await AssertSearchFinds("Bestellprüfung", matching); } + [Test] + public async Task Searches_a_single_quoted_word_in_the_headers() + { + var matching = new IngestedFailure { ExceptionMessage = "The given key 'CustomerId' was not present in the dictionary." }; + + await Ingest(matching, new IngestedFailure()); + + await AssertSearchFinds("CustomerId", matching); + } + [Test] public async Task Searches_the_body() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs index 04f038a7e6..fe441f981b 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs @@ -1,10 +1,13 @@ namespace ServiceControl.Persistence.Tests; using System; +using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; using NUnit.Framework; +using ServiceControl.MessageFailures; using ServiceControl.Persistence.Infrastructure; class TimeRangeFilterTests : ErrorIngestionTestBase @@ -12,16 +15,23 @@ class TimeRangeFilterTests : ErrorIngestionTestBase [Test] public async Task Filters_by_a_time_sent_range_with_an_offset() { - var before = new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 9, 58, 0, DateTimeKind.Utc) }; - var inside = new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 10, 1, 0, DateTimeKind.Utc) }; - var after = new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 10, 4, 0, DateTimeKind.Utc) }; - - await Ingest(before, inside, after); + var inside = await IngestThreeSentThreeMinutesApart(); var result = await MessagesViewStore.GetAllMessages(new PagingInfo(), new SortInfo(), true, new DateTimeRange("2026-07-22T12:00:00+02:00", "2026-07-22T12:03:00+02:00")); - Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside.UniqueMessageIdString })); + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside })); + } + + [Test] + public async Task Filters_by_a_time_sent_range_without_a_zone_as_utc() + { + var inside = await IngestThreeSentThreeMinutesApart(); + + var result = await MessagesViewStore.GetAllMessages(new PagingInfo(), new SortInfo(), true, + new DateTimeRange("2026-07-22T10:00:00", "2026-07-22T10:03:00")); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside })); } [Test] @@ -44,6 +54,56 @@ public async Task Filters_by_a_modified_range_without_a_zone_as_utc() Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside })); } + [Test] + public async Task Unarchives_by_a_modified_range_with_an_offset() + { + var (inside, from, to) = await IngestThreeModifiedTwoMinutesApart(); + await SetEveryStatus(FailedMessageStatus.Archived); + + var unarchived = await FailedMessageLifecycleStore.UnArchiveMessagesByRange(ParsedWithOffset(from), ParsedWithOffset(to)); + + Assert.That(unarchived, Is.EqualTo(new[] { inside })); + } + + [Test] + public async Task Finds_pending_retries_by_a_modified_range_with_an_offset() + { + var (inside, from, to) = await IngestThreeModifiedTwoMinutesApart(); + await SetEveryStatus(FailedMessageStatus.RetryIssued); + + var pending = await FailedMessageRetryStore.GetRetryPendingMessages(ParsedWithOffset(from), ParsedWithOffset(to), "error"); + + Assert.That(pending, Is.EqualTo(new[] { inside }).IgnoreCase); + } + + [Test] + public async Task Processes_pending_retries_by_a_modified_range_with_an_offset() + { + var (inside, from, to) = await IngestThreeModifiedTwoMinutesApart(); + await SetEveryStatus(FailedMessageStatus.RetryIssued); + + var processed = new List(); + await FailedMessageRetryStore.ProcessPendingRetries(ParsedWithOffset(from), ParsedWithOffset(to), null, (id, _) => + { + processed.Add(id); + return Task.CompletedTask; + }); + + Assert.That(processed, Is.EqualTo(new[] { inside }).IgnoreCase); + } + + async Task IngestThreeSentThreeMinutesApart() + { + var inside = new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 10, 1, 0, DateTimeKind.Utc) }; + + await Ingest( + new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 9, 58, 0, DateTimeKind.Utc) }, + inside, + new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 10, 4, 0, DateTimeKind.Utc) }); + + return inside.UniqueMessageIdString; + } + async Task<(string Inside, DateTime From, DateTime To)> IngestThreeModifiedTwoMinutesApart() { var start = Now; @@ -58,8 +118,13 @@ public async Task Filters_by_a_modified_range_without_a_zone_as_utc() return (inside.UniqueMessageIdString, start.AddMinutes(1), start.AddMinutes(3)); } + Task SetEveryStatus(FailedMessageStatus status) => + Query(dbContext => dbContext.FailedMessages.ExecuteUpdateAsync(setters => setters.SetProperty(message => message.Status, status))); + static string WithOffset(DateTime utc) => new DateTimeOffset(utc).ToOffset(TimeSpan.FromHours(2)).ToString("yyyy-MM-ddTHH:mm:ss.fffffffzzz", CultureInfo.InvariantCulture); static string WithoutZone(DateTime utc) => utc.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture); + + static DateTime ParsedWithOffset(DateTime utc) => DateTime.Parse(WithOffset(utc), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); }