diff --git a/.github/workflows/cloud-database-tests.yml b/.github/workflows/cloud-database-tests.yml index 0ce0f35e52..db780d16e4 100644 --- a/.github/workflows/cloud-database-tests.yml +++ b/.github/workflows/cloud-database-tests.yml @@ -65,12 +65,14 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.plan.outputs.targets) }} - # The two suites that exercise the persister. Named outright rather than selected by test - # category, because the category also carries the transport tests, which test the queue - # transport against its own connection string and have no business running here. + # The suites that exercise the primary and audit persisters. Named outright rather than + # selected by test category, because the category also carries the transport tests, which test + # the queue transport against its own connection string and have no business running here. env: PERSISTENCE_PROJECT: src/ServiceControl.Persistence.Tests.${{ matrix.provider }}/ServiceControl.Persistence.Tests.${{ matrix.provider }}.csproj ACCEPTANCE_PROJECT: src/ServiceControl.AcceptanceTests.${{ matrix.provider }}/ServiceControl.AcceptanceTests.${{ matrix.provider }}.csproj + AUDIT_PERSISTENCE_PROJECT: src/ServiceControl.Audit.Persistence.Tests.${{ matrix.provider }}/ServiceControl.Audit.Persistence.Tests.${{ matrix.provider }}.csproj + AUDIT_ACCEPTANCE_PROJECT: src/ServiceControl.Audit.AcceptanceTests.${{ matrix.provider }}/ServiceControl.Audit.AcceptanceTests.${{ matrix.provider }}.csproj steps: - name: Check for secrets env: @@ -104,6 +106,8 @@ jobs: run: | dotnet build $Env:PERSISTENCE_PROJECT --configuration Release dotnet build $Env:ACCEPTANCE_PROJECT --configuration Release + dotnet build $Env:AUDIT_PERSISTENCE_PROJECT --configuration Release + dotnet build $Env:AUDIT_ACCEPTANCE_PROJECT --configuration Release - name: Azure login uses: azure/login@v3.1.0 if: startsWith(matrix.target, 'azure-') @@ -134,7 +138,7 @@ jobs: - name: Run tests uses: Particular/run-tests-action@v1.9.0 with: - projects: ${{ env.PERSISTENCE_PROJECT }};${{ env.ACCEPTANCE_PROJECT }} + projects: ${{ env.PERSISTENCE_PROJECT }};${{ env.ACCEPTANCE_PROJECT }};${{ env.AUDIT_PERSISTENCE_PROJECT }};${{ env.AUDIT_ACCEPTANCE_PROJECT }} max-parallel: 2 env: ServiceControl_TESTS_FILTER: ${{ matrix.provider }} diff --git a/src/ProjectReferences.Persisters.Audit.props b/src/ProjectReferences.Persisters.Audit.props index e729ddae3d..e54de83685 100644 --- a/src/ProjectReferences.Persisters.Audit.props +++ b/src/ProjectReferences.Persisters.Audit.props @@ -1,6 +1,8 @@ + + diff --git a/src/ServiceControl.Audit.AcceptanceTests.PostgreSql/.editorconfig b/src/ServiceControl.Audit.AcceptanceTests.PostgreSql/.editorconfig new file mode 100644 index 0000000000..ca5ad8bd2e --- /dev/null +++ b/src/ServiceControl.Audit.AcceptanceTests.PostgreSql/.editorconfig @@ -0,0 +1,5 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Audit.AcceptanceTests.PostgreSql/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.Audit.AcceptanceTests.PostgreSql/AcceptanceTestStorageConfiguration.cs new file mode 100644 index 0000000000..c6aeb6f385 --- /dev/null +++ b/src/ServiceControl.Audit.AcceptanceTests.PostgreSql/AcceptanceTestStorageConfiguration.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.Audit.AcceptanceTests.PostgreSql; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Audit.AcceptanceTests.TestSupport; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.Tests; + +public class AcceptanceTestStorageConfiguration : IAcceptanceTestStorageConfiguration +{ + public string PersistenceType { get; } = "PostgreSQL"; + + public async Task> CustomizeSettings(CancellationToken cancellationToken = default) + { + var schema = $"sc_at_{Guid.NewGuid():n}"; + + connectionString = await PostgreSqlSharedContainer.GetConnectionStringAsync(cancellationToken); + await TestSchema.Create(connectionString, schema, cancellationToken); + + schemas.Add(schema); + + return new Dictionary + { + [EFPersistenceConfigurationBase.ConnectionStringKey] = connectionString, + [EFPersistenceConfigurationBase.SchemaKey] = schema + }; + } + + public async Task Cleanup(CancellationToken cancellationToken = default) + { + while (schemas.TryTake(out var schema)) + { + await TestSchema.Drop(connectionString, schema, cancellationToken); + } + } + + // The runner holds this lock while it writes the persister settings to the process-wide AppSettings. + public async Task UseDatabaseLifecycleLock(CancellationToken cancellationToken = default) + { + await initializationLock.WaitAsync(cancellationToken); + return new Release(); + } + + sealed class Release : IDisposable + { + public void Dispose() => initializationLock.Release(); + } + + static readonly SemaphoreSlim initializationLock = new(1, 1); + readonly ConcurrentBag schemas = []; + string connectionString; +} diff --git a/src/ServiceControl.Audit.AcceptanceTests.PostgreSql/ServiceControl.Audit.AcceptanceTests.PostgreSql.csproj b/src/ServiceControl.Audit.AcceptanceTests.PostgreSql/ServiceControl.Audit.AcceptanceTests.PostgreSql.csproj new file mode 100644 index 0000000000..4836ca191d --- /dev/null +++ b/src/ServiceControl.Audit.AcceptanceTests.PostgreSql/ServiceControl.Audit.AcceptanceTests.PostgreSql.csproj @@ -0,0 +1,41 @@ + + + + net10.0 + PostgreSql + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Audit.AcceptanceTests.SqlServer/.editorconfig b/src/ServiceControl.Audit.AcceptanceTests.SqlServer/.editorconfig new file mode 100644 index 0000000000..ca5ad8bd2e --- /dev/null +++ b/src/ServiceControl.Audit.AcceptanceTests.SqlServer/.editorconfig @@ -0,0 +1,5 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Audit.AcceptanceTests.SqlServer/AcceptanceTestStorageConfiguration.cs b/src/ServiceControl.Audit.AcceptanceTests.SqlServer/AcceptanceTestStorageConfiguration.cs new file mode 100644 index 0000000000..52485fc4a1 --- /dev/null +++ b/src/ServiceControl.Audit.AcceptanceTests.SqlServer/AcceptanceTestStorageConfiguration.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.Audit.AcceptanceTests.SqlServer; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Audit.AcceptanceTests.TestSupport; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.Tests; + +public class AcceptanceTestStorageConfiguration : IAcceptanceTestStorageConfiguration +{ + public string PersistenceType { get; } = "SQLServer"; + + public async Task> CustomizeSettings(CancellationToken cancellationToken = default) + { + var schema = $"sc_at_{Guid.NewGuid():n}"; + + connectionString = await SqlServerSharedContainer.GetConnectionStringAsync(cancellationToken); + await TestSchema.Create(connectionString, schema, cancellationToken); + + schemas.Add(schema); + + return new Dictionary + { + [EFPersistenceConfigurationBase.ConnectionStringKey] = connectionString, + [EFPersistenceConfigurationBase.SchemaKey] = schema + }; + } + + public async Task Cleanup(CancellationToken cancellationToken = default) + { + while (schemas.TryTake(out var schema)) + { + await TestSchema.Drop(connectionString, schema, cancellationToken); + } + } + + // The runner holds this lock while it writes the persister settings to the process-wide AppSettings. + public async Task UseDatabaseLifecycleLock(CancellationToken cancellationToken = default) + { + await initializationLock.WaitAsync(cancellationToken); + return new Release(); + } + + sealed class Release : IDisposable + { + public void Dispose() => initializationLock.Release(); + } + + static readonly SemaphoreSlim initializationLock = new(1, 1); + readonly ConcurrentBag schemas = []; + string connectionString; +} diff --git a/src/ServiceControl.Audit.AcceptanceTests.SqlServer/ServiceControl.Audit.AcceptanceTests.SqlServer.csproj b/src/ServiceControl.Audit.AcceptanceTests.SqlServer/ServiceControl.Audit.AcceptanceTests.SqlServer.csproj new file mode 100644 index 0000000000..0294b97928 --- /dev/null +++ b/src/ServiceControl.Audit.AcceptanceTests.SqlServer/ServiceControl.Audit.AcceptanceTests.SqlServer.csproj @@ -0,0 +1,41 @@ + + + + net10.0 + SqlServer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_body_content.FullTextSearchDisabled.cs b/src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_body_content.FullTextSearchDisabled.cs new file mode 100644 index 0000000000..18fea7045a --- /dev/null +++ b/src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_body_content.FullTextSearchDisabled.cs @@ -0,0 +1,49 @@ +namespace ServiceControl.Audit.AcceptanceTests.Auditing +{ + using System.Threading.Tasks; + using AcceptanceTesting; + using Audit.Auditing.MessagesView; + using NServiceBus; + using NServiceBus.AcceptanceTesting; + using NUnit.Framework; + + partial class When_processed_message_searched_by_body_content + { + [Test] + public async Task Should_not_be_found_when_fulltext_search_disabled() + { + SetSettings = settings => settings.EnableFullTextSearchOnBodies = false; + + var searchString = "forty-two"; + + var context = await Define() + .WithEndpoint(b => b.When((bus, c) => bus.Send(new MyMessage + { + Something = "Somewhere in the body is the answer to all of the questions. forty-two" + }))) + .WithEndpoint() + .Done(async c => + { + if (c.MessageId != null && await this.TryGetMany($"/api/messages/search/{c.MessageId}")) + { + c.MessageIngested = true; + } + + if (!c.MessageIngested) + { + return false; + } + + c.MessageFound = await this.TryGetMany($"/api/messages/search/{searchString}"); + return true; + }) + .Run(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(context.MessageIngested, Is.True); + Assert.That(context.MessageFound, Is.False); + } + } + } +} diff --git a/src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_body_content.cs b/src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_body_content.cs index 6e0d0ec70e..0e9fe3bcd7 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_body_content.cs +++ b/src/ServiceControl.Audit.AcceptanceTests/Auditing/When_processed_message_searched_by_body_content.cs @@ -9,7 +9,7 @@ using NServiceBus.AcceptanceTesting.Customization; using NUnit.Framework; - class When_processed_message_searched_by_body_content : AcceptanceTest + partial class When_processed_message_searched_by_body_content : AcceptanceTest { [Test] public async Task Should_be_found_when_fulltext_search_enabled() @@ -44,43 +44,6 @@ public async Task Should_be_found_when_fulltext_search_enabled() Assert.That(context.MessageFound, Is.True); } - [Test] - public async Task Should_not_be_found_when_fulltext_search_disabled() - { - SetSettings = settings => settings.EnableFullTextSearchOnBodies = false; - - var searchString = "forty-two"; - - var context = await Define() - .WithEndpoint(b => b.When((bus, c) => bus.Send(new MyMessage - { - Something = "Somewhere in the body is the answer to all of the questions. forty-two" - }))) - .WithEndpoint() - .Done(async c => - { - if (c.MessageId != null && await this.TryGetMany($"/api/messages/search/{c.MessageId}")) - { - c.MessageIngested = true; - } - - if (!c.MessageIngested) - { - return false; - } - - c.MessageFound = await this.TryGetMany($"/api/messages/search/{searchString}"); - return true; - }) - .Run(); - - using (Assert.EnterMultipleScope()) - { - Assert.That(context.MessageIngested, Is.True); - Assert.That(context.MessageFound, Is.False); - } - } - public class Sender : EndpointConfigurationBuilder { public Sender() => diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/.editorconfig b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/.editorconfig new file mode 100644 index 0000000000..fc68ac3228 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/.editorconfig @@ -0,0 +1,9 @@ +[*.cs] + +# Justification: ServiceControl app has no synchronization context +dotnet_diagnostic.CA2007.severity = none + +# Disable style rules for auto-generated EF migrations +[Migrations/**.cs] +dotnet_diagnostic.IDE0065.severity = none +generated_code = true diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/AuditNpgsqlMigrationsSqlGenerator.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/AuditNpgsqlMigrationsSqlGenerator.cs new file mode 100644 index 0000000000..b3b15088a6 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/AuditNpgsqlMigrationsSqlGenerator.cs @@ -0,0 +1,62 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Infrastructure.Internal; +using Npgsql.EntityFrameworkCore.PostgreSQL.Migrations; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +#pragma warning disable EF1001 // Internal EF Core API usage +sealed class AuditNpgsqlMigrationsSqlGenerator( + MigrationsSqlGeneratorDependencies dependencies, + INpgsqlSingletonOptions npgsqlSingletonOptions, + IDbContextOptions contextOptions) + : NpgsqlMigrationsSqlGenerator(dependencies, npgsqlSingletonOptions) +{ + public static readonly string[] PartitionedTables = ["audit_messages", "saga_snapshots"]; + + const string PartitionKey = "created_on"; + + readonly string? schema = contextOptions.FindExtension()?.Schema; + + public override IReadOnlyList Generate( + IReadOnlyList operations, + IModel? model = null, + MigrationsSqlGenerationOptions options = MigrationsSqlGenerationOptions.Default) + { + if (schema is null) + { + return base.Generate(operations, model, options); + } + + MigrationOperation[] stamped = + [ + .. operations.Select(operation => operation is SqlOperation sql + ? FullTextSearchSql.Rewrite(sql, schema) + : MigrationSchemaStamper.Stamp(operation, schema)) + ]; + + return base.Generate(stamped, model, options); + } + + protected override void Generate(CreateTableOperation operation, IModel? model, MigrationCommandListBuilder builder, bool terminate = true) + { + if (!PartitionedTables.Contains(operation.Name)) + { + base.Generate(operation, model, builder, terminate); + return; + } + + base.Generate(operation, model, builder, terminate: false); + builder.Append($" PARTITION BY RANGE ({Dependencies.SqlGenerationHelper.DelimitIdentifier(PartitionKey)})"); + + if (terminate) + { + builder.AppendLine(Dependencies.SqlGenerationHelper.StatementTerminator); + EndStatement(builder); + } + } +} +#pragma warning restore EF1001 diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/AuditPartitionCustomCheck.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/AuditPartitionCustomCheck.cs new file mode 100644 index 0000000000..e3f6515daf --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/AuditPartitionCustomCheck.cs @@ -0,0 +1,36 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using Microsoft.Extensions.DependencyInjection; +using NServiceBus.CustomChecks; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class AuditPartitionCustomCheck(IServiceScopeFactory scopeFactory, IAuditPartitionManager partitions, TimeProvider timeProvider) + : CustomCheck("Audit partition provisioning", "ServiceControl.Audit Health", TimeSpan.FromHours(1)) +{ + public static readonly TimeSpan Threshold = TimeSpan.FromHours(12); + + public override async Task PerformCheck(CancellationToken cancellationToken = default) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var end = await partitions.ProvisionedUntil(dbContext, cancellationToken); + var now = timeProvider.GetUtcNow().UtcDateTime; + + if (end is null) + { + return CheckResult.Failed("No audit partitions are provisioned, so audit ingestion cannot store anything. Run setup for this instance."); + } + + if (end.Value - now < Threshold) + { + return CheckResult.Failed( + $"Audit partitions are provisioned only until {end:u}, less than {Threshold.TotalHours:0} hours ahead. " + + "The retention sweep provisions them and has not been able to. Audit ingestion stops once the last provisioned hour passes. " + + "Check this instance's logs for the sweep failures, and run setup for this instance to provision partitions immediately."); + } + + return CheckResult.Pass; + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs new file mode 100644 index 0000000000..26eb2cc22c --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs @@ -0,0 +1,42 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +static class FullTextSearchSql +{ + const string IndexName = "ix_audit_messages_full_text"; + const string TableName = "audit_messages"; + + public const string Configuration = "simple"; + + // to_tsvector fails once a document's lexemes pass 1 MB. + public const int IndexedBodyLength = 262144; + + // Must match what EF renders for PostgreSqlFullTextSearchDialect exactly, or PostgreSQL won't use the index. + public static readonly string IndexedExpression = + $"""to_tsvector('{Configuration}', headers_json || ' ' || substring(COALESCE(body_text, ''), 1, {IndexedBodyLength}) || ' ' || replace(replace(COALESCE(message_type, ''), '.', ' '), '+', ' '))"""; + + public static readonly string Up = CreateIndexSql(null); + + public static readonly string Down = DropIndexSql(null); + + public static MigrationOperation Rewrite(SqlOperation operation, string schema) => + operation.Sql switch + { + var sql when sql == Up => WithSql(operation, CreateIndexSql(schema)), + var sql when sql == Down => WithSql(operation, DropIndexSql(schema)), + _ => operation + }; + + public static bool IsHandled(string sql) => sql == Up || sql == Down; + + static string CreateIndexSql(string? schema) => + $"CREATE INDEX {IndexName} ON {Qualify(schema, TableName)} USING GIN ({IndexedExpression})"; + + static string DropIndexSql(string? schema) => $"DROP INDEX IF EXISTS {Qualify(schema, IndexName)}"; + + static string Qualify(string? schema, string name) => schema is null ? name : $"\"{schema}\".{name}"; + + static SqlOperation WithSql(SqlOperation operation, string sql) => + new() { Sql = sql, SuppressTransaction = operation.SuppressTransaction }; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032741_Initial.Designer.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032741_Initial.Designer.cs new file mode 100644 index 0000000000..948dba5bdc --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032741_Initial.Designer.cs @@ -0,0 +1,254 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlAuditDbContext))] + [Migration("20260928032741_Initial")] + partial class Initial + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.AuditMessageEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BodyContentType") + .HasColumnType("text") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyState") + .HasColumnType("integer") + .HasColumnName("body_state"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint") + .HasColumnName("critical_time_ticks"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint") + .HasColumnName("delivery_time_ticks"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("MessageId") + .HasColumnType("text") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasColumnType("text") + .HasColumnName("message_type"); + + b.Property("ProcessedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed_at"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint") + .HasColumnName("processing_time_ticks"); + + b.Property("ReceivingEndpointHost") + .HasColumnType("text") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasColumnType("text") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasColumnType("text") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.HasKey("CreatedOn", "Id") + .HasName("pk_audit_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_audit_messages_conversation_id"); + + b.HasIndex("ProcessedAt") + .HasDatabaseName("ix_audit_messages_processed_at"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_audit_messages_time_sent"); + + b.HasIndex("UniqueMessageId") + .HasDatabaseName("ix_audit_messages_unique_message_id"); + + b.HasIndex("ReceivingEndpointName", "CreatedOn") + .HasDatabaseName("ix_audit_messages_receiving_endpoint_name_created_on"); + + b.ToTable("audit_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.FailedAuditImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("Body") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("body"); + + b.Property("ExceptionInfo") + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("MessageId") + .HasColumnType("text") + .HasColumnName("message_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_audit_imports"); + + b.HasIndex("FailedAt", "UniqueMessageId") + .HasDatabaseName("ix_failed_audit_imports_failed_at_unique_message_id"); + + b.ToTable("failed_audit_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.SagaSnapshotEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Endpoint") + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property("FinishTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("finish_time"); + + b.Property("InitiatingMessageJson") + .HasColumnType("text") + .HasColumnName("initiating_message_json"); + + b.Property("OutgoingMessagesJson") + .HasColumnType("text") + .HasColumnName("outgoing_messages_json"); + + b.Property("ProcessedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed_at"); + + b.Property("SagaId") + .HasColumnType("uuid") + .HasColumnName("saga_id"); + + b.Property("SagaType") + .HasColumnType("text") + .HasColumnName("saga_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("StateAfterChange") + .HasColumnType("text") + .HasColumnName("state_after_change"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("CreatedOn", "Id") + .HasName("pk_saga_snapshots"); + + b.HasIndex("SagaId", "FinishTime") + .HasDatabaseName("ix_saga_snapshots_saga_id_finish_time"); + + b.ToTable("saga_snapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032741_Initial.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032741_Initial.cs new file mode 100644 index 0000000000..a9ecbc7963 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032741_Initial.cs @@ -0,0 +1,138 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql.Migrations +{ + /// + public partial class Initial : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "audit_messages", + columns: table => new + { + created_on = table.Column(type: "timestamp with time zone", nullable: false), + id = table.Column(type: "bigint", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + unique_message_id = table.Column(type: "uuid", nullable: false), + message_id = table.Column(type: "text", nullable: true), + message_type = table.Column(type: "text", nullable: true), + time_sent = table.Column(type: "timestamp with time zone", nullable: true), + processed_at = table.Column(type: "timestamp with time zone", nullable: false), + conversation_id = table.Column(type: "character varying(450)", maxLength: 450, nullable: true), + is_system_message = table.Column(type: "boolean", nullable: false), + status = table.Column(type: "integer", nullable: false), + sending_endpoint_name = table.Column(type: "text", nullable: true), + sending_endpoint_host_id = table.Column(type: "uuid", nullable: true), + sending_endpoint_host = table.Column(type: "text", nullable: true), + receiving_endpoint_name = table.Column(type: "character varying(450)", maxLength: 450, nullable: true), + receiving_endpoint_host_id = table.Column(type: "uuid", nullable: true), + receiving_endpoint_host = table.Column(type: "text", nullable: true), + critical_time_ticks = table.Column(type: "bigint", nullable: true), + processing_time_ticks = table.Column(type: "bigint", nullable: true), + delivery_time_ticks = table.Column(type: "bigint", nullable: true), + headers_json = table.Column(type: "text", nullable: false), + body_text = table.Column(type: "text", nullable: true), + body_state = table.Column(type: "integer", nullable: false), + body_size = table.Column(type: "integer", nullable: false), + body_content_type = table.Column(type: "text", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_audit_messages", x => new { x.created_on, x.id }); + }); + + migrationBuilder.CreateTable( + name: "failed_audit_imports", + columns: table => new + { + unique_message_id = table.Column(type: "uuid", nullable: false), + failed_at = table.Column(type: "timestamp with time zone", nullable: false), + message_id = table.Column(type: "text", nullable: true), + headers_json = table.Column(type: "text", nullable: false), + body = table.Column(type: "bytea", nullable: false), + exception_info = table.Column(type: "text", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_failed_audit_imports", x => x.unique_message_id); + }); + + migrationBuilder.CreateTable( + name: "saga_snapshots", + columns: table => new + { + created_on = table.Column(type: "timestamp with time zone", nullable: false), + id = table.Column(type: "bigint", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + saga_id = table.Column(type: "uuid", nullable: false), + saga_type = table.Column(type: "text", nullable: true), + status = table.Column(type: "integer", nullable: false), + start_time = table.Column(type: "timestamp with time zone", nullable: false), + finish_time = table.Column(type: "timestamp with time zone", nullable: false), + processed_at = table.Column(type: "timestamp with time zone", nullable: false), + endpoint = table.Column(type: "text", nullable: true), + state_after_change = table.Column(type: "text", nullable: true), + initiating_message_json = table.Column(type: "text", nullable: true), + outgoing_messages_json = table.Column(type: "text", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_saga_snapshots", x => new { x.created_on, x.id }); + }); + + migrationBuilder.CreateIndex( + name: "ix_audit_messages_conversation_id", + table: "audit_messages", + column: "conversation_id"); + + migrationBuilder.CreateIndex( + name: "ix_audit_messages_processed_at", + table: "audit_messages", + column: "processed_at"); + + migrationBuilder.CreateIndex( + name: "ix_audit_messages_receiving_endpoint_name_created_on", + table: "audit_messages", + columns: new[] { "receiving_endpoint_name", "created_on" }); + + migrationBuilder.CreateIndex( + name: "ix_audit_messages_time_sent", + table: "audit_messages", + column: "time_sent"); + + migrationBuilder.CreateIndex( + name: "ix_audit_messages_unique_message_id", + table: "audit_messages", + column: "unique_message_id"); + + migrationBuilder.CreateIndex( + name: "ix_failed_audit_imports_failed_at_unique_message_id", + table: "failed_audit_imports", + columns: new[] { "failed_at", "unique_message_id" }); + + migrationBuilder.CreateIndex( + name: "ix_saga_snapshots_saga_id_finish_time", + table: "saga_snapshots", + columns: new[] { "saga_id", "finish_time" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "audit_messages"); + + migrationBuilder.DropTable( + name: "failed_audit_imports"); + + migrationBuilder.DropTable( + name: "saga_snapshots"); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032745_AddFullTextSearch.Designer.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032745_AddFullTextSearch.Designer.cs new file mode 100644 index 0000000000..b164b4fcc9 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032745_AddFullTextSearch.Designer.cs @@ -0,0 +1,254 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlAuditDbContext))] + [Migration("20260928032745_AddFullTextSearch")] + partial class AddFullTextSearch + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.AuditMessageEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BodyContentType") + .HasColumnType("text") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyState") + .HasColumnType("integer") + .HasColumnName("body_state"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint") + .HasColumnName("critical_time_ticks"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint") + .HasColumnName("delivery_time_ticks"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("MessageId") + .HasColumnType("text") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasColumnType("text") + .HasColumnName("message_type"); + + b.Property("ProcessedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed_at"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint") + .HasColumnName("processing_time_ticks"); + + b.Property("ReceivingEndpointHost") + .HasColumnType("text") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasColumnType("text") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasColumnType("text") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.HasKey("CreatedOn", "Id") + .HasName("pk_audit_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_audit_messages_conversation_id"); + + b.HasIndex("ProcessedAt") + .HasDatabaseName("ix_audit_messages_processed_at"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_audit_messages_time_sent"); + + b.HasIndex("UniqueMessageId") + .HasDatabaseName("ix_audit_messages_unique_message_id"); + + b.HasIndex("ReceivingEndpointName", "CreatedOn") + .HasDatabaseName("ix_audit_messages_receiving_endpoint_name_created_on"); + + b.ToTable("audit_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.FailedAuditImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("Body") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("body"); + + b.Property("ExceptionInfo") + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("MessageId") + .HasColumnType("text") + .HasColumnName("message_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_audit_imports"); + + b.HasIndex("FailedAt", "UniqueMessageId") + .HasDatabaseName("ix_failed_audit_imports_failed_at_unique_message_id"); + + b.ToTable("failed_audit_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.SagaSnapshotEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Endpoint") + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property("FinishTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("finish_time"); + + b.Property("InitiatingMessageJson") + .HasColumnType("text") + .HasColumnName("initiating_message_json"); + + b.Property("OutgoingMessagesJson") + .HasColumnType("text") + .HasColumnName("outgoing_messages_json"); + + b.Property("ProcessedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed_at"); + + b.Property("SagaId") + .HasColumnType("uuid") + .HasColumnName("saga_id"); + + b.Property("SagaType") + .HasColumnType("text") + .HasColumnName("saga_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("StateAfterChange") + .HasColumnType("text") + .HasColumnName("state_after_change"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("CreatedOn", "Id") + .HasName("pk_saga_snapshots"); + + b.HasIndex("SagaId", "FinishTime") + .HasDatabaseName("ix_saga_snapshots_saga_id_finish_time"); + + b.ToTable("saga_snapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032745_AddFullTextSearch.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032745_AddFullTextSearch.cs new file mode 100644 index 0000000000..19c0c1a4cb --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/20260928032745_AddFullTextSearch.cs @@ -0,0 +1,22 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql.Migrations +{ + /// + public partial class AddFullTextSearch : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(FullTextSearchSql.Up); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(FullTextSearchSql.Down); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlAuditDbContextModelSnapshot.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlAuditDbContextModelSnapshot.cs new file mode 100644 index 0000000000..eb646b0c01 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlAuditDbContextModelSnapshot.cs @@ -0,0 +1,251 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlAuditDbContext))] + partial class PostgreSqlAuditDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.AuditMessageEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("BodyContentType") + .HasColumnType("text") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyState") + .HasColumnType("integer") + .HasColumnName("body_state"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint") + .HasColumnName("critical_time_ticks"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint") + .HasColumnName("delivery_time_ticks"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("MessageId") + .HasColumnType("text") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasColumnType("text") + .HasColumnName("message_type"); + + b.Property("ProcessedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed_at"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint") + .HasColumnName("processing_time_ticks"); + + b.Property("ReceivingEndpointHost") + .HasColumnType("text") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasColumnType("text") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasColumnType("text") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.HasKey("CreatedOn", "Id") + .HasName("pk_audit_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_audit_messages_conversation_id"); + + b.HasIndex("ProcessedAt") + .HasDatabaseName("ix_audit_messages_processed_at"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_audit_messages_time_sent"); + + b.HasIndex("UniqueMessageId") + .HasDatabaseName("ix_audit_messages_unique_message_id"); + + b.HasIndex("ReceivingEndpointName", "CreatedOn") + .HasDatabaseName("ix_audit_messages_receiving_endpoint_name_created_on"); + + b.ToTable("audit_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.FailedAuditImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("Body") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("body"); + + b.Property("ExceptionInfo") + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("MessageId") + .HasColumnType("text") + .HasColumnName("message_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_audit_imports"); + + b.HasIndex("FailedAt", "UniqueMessageId") + .HasDatabaseName("ix_failed_audit_imports_failed_at_unique_message_id"); + + b.ToTable("failed_audit_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.SagaSnapshotEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Endpoint") + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property("FinishTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("finish_time"); + + b.Property("InitiatingMessageJson") + .HasColumnType("text") + .HasColumnName("initiating_message_json"); + + b.Property("OutgoingMessagesJson") + .HasColumnType("text") + .HasColumnName("outgoing_messages_json"); + + b.Property("ProcessedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed_at"); + + b.Property("SagaId") + .HasColumnType("uuid") + .HasColumnName("saga_id"); + + b.Property("SagaType") + .HasColumnType("text") + .HasColumnName("saga_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("StateAfterChange") + .HasColumnType("text") + .HasColumnName("state_after_change"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("CreatedOn", "Id") + .HasName("pk_saga_snapshots"); + + b.HasIndex("SagaId", "FinishTime") + .HasDatabaseName("ix_saga_snapshots_saga_id_finish_time"); + + b.ToTable("saga_snapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlAuditDbContext.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlAuditDbContext.cs new file mode 100644 index 0000000000..4f65c76342 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlAuditDbContext.cs @@ -0,0 +1,48 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using Microsoft.EntityFrameworkCore; +using Npgsql; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; + +public class PostgreSqlAuditDbContext(DbContextOptions options) : AuditDbContext(options) +{ + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + base.OnConfiguring(optionsBuilder); + + optionsBuilder.UseSnakeCaseNamingConvention(); + } + + public override bool IsDuplicateKeyException(DbUpdateException exception) + { + var queue = new Queue([exception]); + while (queue.Count > 0) + { + var e = queue.Dequeue(); + if (e is PostgresException { SqlState: PostgresErrorCodes.UniqueViolation }) + { + return true; + } + + if (e is AggregateException aggregateException) + { + foreach (var inner in aggregateException.InnerExceptions) + { + queue.Enqueue(inner); + } + } + + if (e.InnerException != null) + { + queue.Enqueue(e.InnerException); + } + } + + return false; + } + + public override async Task SchemaExists(string schema, CancellationToken cancellationToken = default) => + await Database + .SqlQueryRaw("""SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = {0}) THEN 1 ELSE 0 END AS "Value" """, schema) + .SingleAsync(cancellationToken) == 1; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlAuditDbContextFactory.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlAuditDbContextFactory.cs new file mode 100644 index 0000000000..3c7da24a24 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlAuditDbContextFactory.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.EntityFrameworkCore.Migrations; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; + +public class PostgreSqlAuditDbContextFactory : IDesignTimeDbContextFactory +{ + public PostgreSqlAuditDbContext CreateDbContext(string[] args) + { + var connectionString = Environment.GetEnvironmentVariable("SERVICECONTROL_DATABASE_CONNECTIONSTRING") + ?? "Host=localhost;Port=5432;Database=servicecontrolaudit;Username=postgres;Password=postgres"; + + var optionsBuilder = new DbContextOptionsBuilder(); + optionsBuilder.UseNpgsql(connectionString, npgsqlOptions => npgsqlOptions.MigrationsHistoryTable(AuditDbContext.MigrationsHistoryTableName)); + optionsBuilder.ReplaceService(); + + return new PostgreSqlAuditDbContext(optionsBuilder.Options); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlAuditPartitionManager.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlAuditPartitionManager.cs new file mode 100644 index 0000000000..1c310dd52f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlAuditPartitionManager.cs @@ -0,0 +1,130 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using System.Globalization; +using System.Text; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.Logging; +using Npgsql; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class PostgreSqlAuditPartitionManager(ILogger logger) : IAuditPartitionManager +{ + public async Task EnsurePartitions(AuditDbContext dbContext, DateTime from, DateTime toExclusive, CancellationToken cancellationToken = default) + { + var sql = new StringBuilder(); + foreach (var table in Tables(dbContext)) + { + for (var day = from.Date; day < toExclusive; day = day.AddDays(1)) + { + sql.Append(CultureInfo.InvariantCulture, $"CREATE TABLE IF NOT EXISTS {table.Partition(day)} PARTITION OF {table.Parent} FOR VALUES FROM ('{Bound(day)}') TO ('{Bound(day.AddDays(1))}');\n"); + } + } + + if (sql.Length == 0) + { + return; + } + + if (!await WithShortLockWait(dbContext, sql.ToString(), cancellationToken)) + { + logger.LogWarning("Could not provision audit partitions up to {End:u} because the audit tables were too busy. The next sweep will try again", toExclusive); + } + } + + public async Task> ListExpired(AuditDbContext dbContext, DateTime keepFrom, CancellationToken cancellationToken = default) + { + var days = new SortedSet(); + foreach (var table in Tables(dbContext)) + { + foreach (var day in await PartitionDays(dbContext, table, cancellationToken)) + { + if (day.AddDays(1) <= keepFrom) + { + days.Add(day); + } + } + } + + return [.. days]; + } + + public async Task DropExpired(AuditDbContext dbContext, DateTime day, int batchSize, CancellationToken cancellationToken = default) + { + var sql = string.Concat(Tables(dbContext).Select(table => $"DROP TABLE IF EXISTS {table.Partition(day)};\n")); + + return await WithShortLockWait(dbContext, sql, cancellationToken) + ? new DropResult(RowsDeleted: 0, Completed: true) + : new DropResult(RowsDeleted: 0, Completed: true, Deferred: true); + } + + public async Task ProvisionedUntil(AuditDbContext dbContext, CancellationToken cancellationToken = default) + { + var days = await PartitionDays(dbContext, Table(dbContext), cancellationToken); + return days.Count == 0 ? null : days.Max().AddDays(1); + } + + internal static string PartitionName(string tableName, DateTime day) => + string.Create(CultureInfo.InvariantCulture, $"{tableName}_{day:yyyyMMdd}"); + + static async Task> PartitionDays(AuditDbContext dbContext, AuditTable table, CancellationToken cancellationToken) + { + var names = await dbContext.Database + .SqlQueryRaw(""" + SELECT c.relname AS "Value" + FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid + JOIN pg_class p ON p.oid = i.inhparent + JOIN pg_namespace n ON n.oid = p.relnamespace + WHERE p.relname = {0} AND n.nspname = COALESCE(CAST({1} AS text), current_schema()) + """, table.Name, dbContext.Schema ?? (object)DBNull.Value) + .ToListAsync(cancellationToken); + + var days = new List(names.Count); + foreach (var name in names) + { + if (name.Length > table.Name.Length + 1 + && DateTime.TryParseExact(name[(table.Name.Length + 1)..], "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var day)) + { + days.Add(day); + } + } + + return days; + } + + // While DDL waits for its exclusive lock, every insert and query queues behind it. + static Task WithShortLockWait(AuditDbContext dbContext, string sql, CancellationToken cancellationToken) => + dbContext.Database.CreateExecutionStrategy().ExecuteAsync(async token => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(token); + await dbContext.Database.ExecuteSqlRawAsync("SET LOCAL lock_timeout = '5s'", token); + + try + { + await dbContext.Database.ExecuteSqlRawAsync(sql, token); + } + catch (PostgresException e) when (e.SqlState == PostgresErrorCodes.LockNotAvailable) + { + return false; + } + + await transaction.CommitAsync(token); + return true; + }, cancellationToken); + + static AuditTable[] Tables(AuditDbContext dbContext) => [Table(dbContext), Table(dbContext)]; + + static AuditTable Table(AuditDbContext dbContext) => + new(dbContext.Model.FindEntityType(typeof(TEntity))!.GetTableName()!, SchemaQualifiedTableName.For(dbContext), dbContext.Schema, dbContext.GetService()); + + static string Bound(DateTime day) => day.ToString("yyyy-MM-dd 00:00:00+00", CultureInfo.InvariantCulture); + + sealed record AuditTable(string Name, string Parent, string? Schema, ISqlGenerationHelper SqlGenerationHelper) + { + public string Partition(DateTime day) => SqlGenerationHelper.DelimitIdentifier(PartitionName(Name, day), Schema); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs new file mode 100644 index 0000000000..cd48f14bb8 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using Microsoft.EntityFrameworkCore; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class PostgreSqlFullTextSearchDialect : IFullTextSearchDialect +{ + public IQueryable Search(IQueryable source, string searchTerms) => + source.Where(message => + EF.Functions.ToTsVector(FullTextSearchSql.Configuration, + message.HeadersJson + " " + + (message.BodyText ?? "").Substring(0, FullTextSearchSql.IndexedBodyLength) + " " + + (message.MessageType ?? "").Replace(".", " ").Replace("+", " ")) + .Matches(EF.Functions.WebSearchToTsQuery(FullTextSearchSql.Configuration, ToOrQuery(searchTerms)))); + + static string ToOrQuery(string searchTerms) => + string.Join(" OR ", searchTerms.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs new file mode 100644 index 0000000000..66d36b959f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -0,0 +1,64 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.DependencyInjection; +using NServiceBus; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class PostgreSqlPersistence(EFPersisterSettings settings) : EFPersistenceBase(settings) +{ + protected override void AddDbContext(IServiceCollection services, EFPersisterSettings settings) + { + services.AddDbContext(options => + { + options.UseNpgsql(settings.ConnectionString, npgsqlOptions => + { + npgsqlOptions.CommandTimeout(settings.CommandTimeout); + npgsqlOptions.MigrationsHistoryTable(AuditDbContext.MigrationsHistoryTableName, settings.Schema); + + if (settings.EnableRetryOnFailure) + { + npgsqlOptions.EnableRetryOnFailure( + maxRetryCount: settings.MaxRetryCount, + maxRetryDelay: TimeSpan.FromSeconds(settings.MaxRetryDelayInSeconds), + errorCodesToAdd: null); + } + }); + + options.ReplaceService(); + + if (settings.Schema is not null) + { + ((IDbContextOptionsBuilderInfrastructure)options).AddOrUpdateExtension(new SchemaOptionsExtension(settings.Schema)); + options.ReplaceService(); + + // HasDefaultSchema makes the model differ from the migrations snapshot on purpose. + options.ConfigureWarnings(warnings => warnings.Ignore(RelationalEventId.PendingModelChangesWarning)); + } + + if (settings.EnableSensitiveDataLogging) + { + options.EnableSensitiveDataLogging(); + } + }, ServiceLifetime.Scoped); + + services.AddScoped(provider => provider.GetRequiredService()); + } + + protected override void AddPartitionManager(IServiceCollection services) => + services.AddSingleton(); + + protected override void AddQueryServices(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + } + + protected override void AddCustomChecks(EndpointConfiguration endpointConfiguration) => + endpointConfiguration.AddCustomCheck(); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistenceConfiguration.cs new file mode 100644 index 0000000000..dabb3c7cf7 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlPersistenceConfiguration.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using ServiceControl.Audit.Persistence.EFCore.Abstractions; + +class PostgreSqlPersistenceConfiguration : EFPersistenceConfigurationBase +{ + public override string Name => "PostgreSQL"; + + public static readonly TimeSpan MinimumRetentionPeriod = TimeSpan.FromDays(1); + public static readonly TimeSpan MaximumRetentionPeriod = TimeSpan.FromDays(90); + + protected override IPersistence Create(EFPersisterSettings settings) + { + if (settings.AuditRetentionPeriod < MinimumRetentionPeriod || settings.AuditRetentionPeriod > MaximumRetentionPeriod) + { + throw new InvalidOperationException( + $"AuditRetentionPeriod is {settings.AuditRetentionPeriod}, but PostgreSQL storage keeps audit data for 1 to 90 days. Set AuditRetentionPeriod within that range."); + } + + return new PostgreSqlPersistence(settings); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlRetentionLock.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlRetentionLock.cs new file mode 100644 index 0000000000..3e6e1bdc34 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlRetentionLock.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Audit.Persistence.EFCore.PostgreSql; + +using Microsoft.Extensions.Logging; +using Npgsql; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class PostgreSqlRetentionLock(EFPersisterSettings settings, ILogger logger) : IRetentionLock +{ + // Unpooled, because a session lock would stay with a pooled connection after it returns to the pool. + readonly string connectionString = new NpgsqlConnectionStringBuilder(settings.ConnectionString) { Pooling = false }.ConnectionString; + readonly string resource = RetentionLock.ResourceName(settings.Schema); + + public async Task TryAcquire(CancellationToken cancellationToken = default) + { + var connection = new NpgsqlConnection(connectionString); + var acquired = false; + try + { + await connection.OpenAsync(cancellationToken); + + await using var command = connection.CreateCommand(); + command.CommandText = "SELECT pg_try_advisory_lock(hashtext(@resource))"; + command.Parameters.AddWithValue("resource", resource); + + acquired = await command.ExecuteScalarAsync(cancellationToken) is true; + return acquired ? new Handle(connection, resource, logger) : null; + } + finally + { + if (!acquired) + { + await connection.DisposeAsync(); + } + } + } + + sealed class Handle(NpgsqlConnection connection, string resource, ILogger logger) : IAsyncDisposable + { + public async ValueTask DisposeAsync() + { + try + { + await using var command = connection.CreateCommand(); + command.CommandText = "SELECT pg_advisory_unlock(hashtext(@resource))"; + command.Parameters.AddWithValue("resource", resource); + + if (await command.ExecuteScalarAsync() is not true) + { + logger.LogWarning("The audit retention lock was not held by this connection when releasing it. Connect to PostgreSQL directly or through a session mode pooler, or retention will stop running"); + } + } + finally + { + await connection.DisposeAsync(); + } + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/ServiceControl.Audit.Persistence.EFCore.PostgreSql.csproj b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/ServiceControl.Audit.Persistence.EFCore.PostgreSql.csproj new file mode 100644 index 0000000000..fce1f0a48a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/ServiceControl.Audit.Persistence.EFCore.PostgreSql.csproj @@ -0,0 +1,50 @@ + + + + net10.0 + enable + enable + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/persistence.manifest b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/persistence.manifest new file mode 100644 index 0000000000..abff0adcaf --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/persistence.manifest @@ -0,0 +1,21 @@ +{ + "Name": "PostgreSQL", + "DisplayName": "PostgreSQL", + "Description": "PostgreSQL ServiceControl Audit persister", + "AssemblyName": "ServiceControl.Audit.Persistence.EFCore.PostgreSql", + "TypeName": "ServiceControl.Audit.Persistence.EFCore.PostgreSql.PostgreSqlPersistenceConfiguration, ServiceControl.Audit.Persistence.EFCore.PostgreSql", + "Settings": [ + { + "Name": "ServiceControl.Audit/Database/ConnectionString", + "Mandatory": true + }, + { + "Name": "ServiceControl.Audit/Database/Schema", + "Mandatory": false + }, + { + "Name": "ServiceControl.Audit/Database/CommandTimeout", + "Mandatory": false + } + ] +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/.editorconfig b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/.editorconfig new file mode 100644 index 0000000000..fc68ac3228 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/.editorconfig @@ -0,0 +1,9 @@ +[*.cs] + +# Justification: ServiceControl app has no synchronization context +dotnet_diagnostic.CA2007.severity = none + +# Disable style rules for auto-generated EF migrations +[Migrations/**.cs] +dotnet_diagnostic.IDE0065.severity = none +generated_code = true diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/FullTextSearchSql.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/FullTextSearchSql.cs new file mode 100644 index 0000000000..c4b62a7db0 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/FullTextSearchSql.cs @@ -0,0 +1,78 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +static class FullTextSearchSql +{ + const string CatalogName = "ServiceControlFullTextCatalog"; + const string TableName = "AuditMessages"; + + const string KeyIndexName = "IX_AuditMessages_Id"; + + public const string RequireFullTextSearch = """ + IF SERVERPROPERTY('IsFullTextInstalled') <> 1 + BEGIN + THROW 50000, 'ServiceControl requires the SQL Server Full-Text Search feature, which is not installed on this instance. Install it and run setup again.', 1; + END + """; + + public const string CreateCatalog = $""" + IF NOT EXISTS (SELECT 1 FROM sys.fulltext_catalogs WHERE name = '{CatalogName}') + BEGIN + BEGIN TRY + EXEC('CREATE FULLTEXT CATALOG {CatalogName}'); + END TRY + BEGIN CATCH + IF NOT EXISTS (SELECT 1 FROM sys.fulltext_catalogs WHERE name = '{CatalogName}') THROW; + END CATCH + END + """; + + public const string DropCatalog = $""" + IF EXISTS (SELECT 1 FROM sys.fulltext_catalogs WHERE name = '{CatalogName}') + AND NOT EXISTS (SELECT 1 + FROM sys.fulltext_indexes i + JOIN sys.fulltext_catalogs c ON i.fulltext_catalog_id = c.fulltext_catalog_id + WHERE c.name = '{CatalogName}') + BEGIN + DROP FULLTEXT CATALOG {CatalogName}; + END + """; + + public static readonly string CreateIndex = CreateIndexSql(null); + + public static readonly string DropIndex = DropIndexSql(null); + + public static MigrationOperation Rewrite(SqlOperation operation, string schema) => + operation.Sql switch + { + var sql when sql == CreateIndex => WithSql(operation, CreateIndexSql(schema)), + var sql when sql == DropIndex => WithSql(operation, DropIndexSql(schema)), + _ => operation + }; + + public static bool IsHandled(string sql) => + sql == RequireFullTextSearch || sql == CreateCatalog || sql == DropCatalog || sql == CreateIndex || sql == DropIndex; + + static string CreateIndexSql(string? schema) => $""" + IF NOT EXISTS (SELECT 1 FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('{Qualify(schema)}')) + BEGIN + EXEC('CREATE FULLTEXT INDEX ON {Qualify(schema)}(HeadersJson LANGUAGE 0, BodyText LANGUAGE 0) + KEY INDEX {KeyIndexName} + ON {CatalogName} + WITH (CHANGE_TRACKING AUTO, STOPLIST = OFF)'); + END + """; + + static string DropIndexSql(string? schema) => $""" + IF EXISTS (SELECT 1 FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('{Qualify(schema)}')) + BEGIN + DROP FULLTEXT INDEX ON {Qualify(schema)}; + END + """; + + static string Qualify(string? schema) => schema is null ? TableName : $"[{schema}].[{TableName}]"; + + static SqlOperation WithSql(SqlOperation operation, string sql) => + new() { Sql = sql, SuppressTransaction = operation.SuppressTransaction }; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032748_Initial.Designer.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032748_Initial.Designer.cs new file mode 100644 index 0000000000..c7b8a0293b --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032748_Initial.Designer.cs @@ -0,0 +1,205 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Audit.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerAuditDbContext))] + [Migration("20260928032748_Initial")] + partial class Initial + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.AuditMessageEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BodyContentType") + .HasColumnType("nvarchar(max)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyState") + .HasColumnType("int"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("MessageId") + .HasColumnType("nvarchar(max)"); + + b.Property("MessageType") + .HasColumnType("nvarchar(max)"); + + b.Property("ProcessedAt") + .HasColumnType("datetime2"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint"); + + b.Property("ReceivingEndpointHost") + .HasColumnType("nvarchar(max)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasColumnType("nvarchar(max)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("CreatedOn", "Id"); + + b.HasIndex("ConversationId"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("ProcessedAt"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.HasIndex("ReceivingEndpointName", "CreatedOn"); + + b.ToTable("AuditMessages"); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.FailedAuditImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("ExceptionInfo") + .HasColumnType("nvarchar(max)"); + + b.Property("FailedAt") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageId") + .HasColumnType("nvarchar(max)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("FailedAt", "UniqueMessageId"); + + b.ToTable("FailedAuditImports"); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.SagaSnapshotEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Endpoint") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishTime") + .HasColumnType("datetime2"); + + b.Property("InitiatingMessageJson") + .HasColumnType("nvarchar(max)"); + + b.Property("OutgoingMessagesJson") + .HasColumnType("nvarchar(max)"); + + b.Property("ProcessedAt") + .HasColumnType("datetime2"); + + b.Property("SagaId") + .HasColumnType("uniqueidentifier"); + + b.Property("SagaType") + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("StateAfterChange") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("CreatedOn", "Id"); + + b.HasIndex("SagaId", "FinishTime"); + + b.ToTable("SagaSnapshots"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032748_Initial.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032748_Initial.cs new file mode 100644 index 0000000000..b756b3da9a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032748_Initial.cs @@ -0,0 +1,143 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer.Migrations +{ + /// + public partial class Initial : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "AuditMessages", + columns: table => new + { + CreatedOn = table.Column(type: "datetime2", nullable: false), + Id = table.Column(type: "bigint", nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + UniqueMessageId = table.Column(type: "uniqueidentifier", nullable: false), + MessageId = table.Column(type: "nvarchar(max)", nullable: true), + MessageType = table.Column(type: "nvarchar(max)", nullable: true), + TimeSent = table.Column(type: "datetime2", nullable: true), + ProcessedAt = table.Column(type: "datetime2", nullable: false), + ConversationId = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true), + IsSystemMessage = table.Column(type: "bit", nullable: false), + Status = table.Column(type: "int", nullable: false), + SendingEndpointName = table.Column(type: "nvarchar(max)", nullable: true), + SendingEndpointHostId = table.Column(type: "uniqueidentifier", nullable: true), + SendingEndpointHost = table.Column(type: "nvarchar(max)", nullable: true), + ReceivingEndpointName = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: true), + ReceivingEndpointHostId = table.Column(type: "uniqueidentifier", nullable: true), + ReceivingEndpointHost = table.Column(type: "nvarchar(max)", nullable: true), + CriticalTimeTicks = table.Column(type: "bigint", nullable: true), + ProcessingTimeTicks = table.Column(type: "bigint", nullable: true), + DeliveryTimeTicks = table.Column(type: "bigint", nullable: true), + HeadersJson = table.Column(type: "nvarchar(max)", nullable: false), + BodyText = table.Column(type: "nvarchar(max)", nullable: true), + BodyState = table.Column(type: "int", nullable: false), + BodySize = table.Column(type: "int", nullable: false), + BodyContentType = table.Column(type: "nvarchar(max)", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_AuditMessages", x => new { x.CreatedOn, x.Id }); + }); + + migrationBuilder.CreateTable( + name: "FailedAuditImports", + columns: table => new + { + UniqueMessageId = table.Column(type: "uniqueidentifier", nullable: false), + FailedAt = table.Column(type: "datetime2", nullable: false), + MessageId = table.Column(type: "nvarchar(max)", nullable: true), + HeadersJson = table.Column(type: "nvarchar(max)", nullable: false), + Body = table.Column(type: "varbinary(max)", nullable: false), + ExceptionInfo = table.Column(type: "nvarchar(max)", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_FailedAuditImports", x => x.UniqueMessageId); + }); + + migrationBuilder.CreateTable( + name: "SagaSnapshots", + columns: table => new + { + CreatedOn = table.Column(type: "datetime2", nullable: false), + Id = table.Column(type: "bigint", nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + SagaId = table.Column(type: "uniqueidentifier", nullable: false), + SagaType = table.Column(type: "nvarchar(max)", nullable: true), + Status = table.Column(type: "int", nullable: false), + StartTime = table.Column(type: "datetime2", nullable: false), + FinishTime = table.Column(type: "datetime2", nullable: false), + ProcessedAt = table.Column(type: "datetime2", nullable: false), + Endpoint = table.Column(type: "nvarchar(max)", nullable: true), + StateAfterChange = table.Column(type: "nvarchar(max)", nullable: true), + InitiatingMessageJson = table.Column(type: "nvarchar(max)", nullable: true), + OutgoingMessagesJson = table.Column(type: "nvarchar(max)", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_SagaSnapshots", x => new { x.CreatedOn, x.Id }); + }); + + migrationBuilder.CreateIndex( + name: "IX_AuditMessages_ConversationId", + table: "AuditMessages", + column: "ConversationId"); + + migrationBuilder.CreateIndex( + name: "IX_AuditMessages_Id", + table: "AuditMessages", + column: "Id", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_AuditMessages_ProcessedAt", + table: "AuditMessages", + column: "ProcessedAt"); + + migrationBuilder.CreateIndex( + name: "IX_AuditMessages_ReceivingEndpointName_CreatedOn", + table: "AuditMessages", + columns: new[] { "ReceivingEndpointName", "CreatedOn" }); + + migrationBuilder.CreateIndex( + name: "IX_AuditMessages_TimeSent", + table: "AuditMessages", + column: "TimeSent"); + + migrationBuilder.CreateIndex( + name: "IX_AuditMessages_UniqueMessageId", + table: "AuditMessages", + column: "UniqueMessageId"); + + migrationBuilder.CreateIndex( + name: "IX_FailedAuditImports_FailedAt_UniqueMessageId", + table: "FailedAuditImports", + columns: new[] { "FailedAt", "UniqueMessageId" }); + + migrationBuilder.CreateIndex( + name: "IX_SagaSnapshots_SagaId_FinishTime", + table: "SagaSnapshots", + columns: new[] { "SagaId", "FinishTime" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "AuditMessages"); + + migrationBuilder.DropTable( + name: "FailedAuditImports"); + + migrationBuilder.DropTable( + name: "SagaSnapshots"); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032751_AddFullTextSearch.Designer.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032751_AddFullTextSearch.Designer.cs new file mode 100644 index 0000000000..804a2522c2 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032751_AddFullTextSearch.Designer.cs @@ -0,0 +1,205 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Audit.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerAuditDbContext))] + [Migration("20260928032751_AddFullTextSearch")] + partial class AddFullTextSearch + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.AuditMessageEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BodyContentType") + .HasColumnType("nvarchar(max)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyState") + .HasColumnType("int"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("MessageId") + .HasColumnType("nvarchar(max)"); + + b.Property("MessageType") + .HasColumnType("nvarchar(max)"); + + b.Property("ProcessedAt") + .HasColumnType("datetime2"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint"); + + b.Property("ReceivingEndpointHost") + .HasColumnType("nvarchar(max)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasColumnType("nvarchar(max)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("CreatedOn", "Id"); + + b.HasIndex("ConversationId"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("ProcessedAt"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.HasIndex("ReceivingEndpointName", "CreatedOn"); + + b.ToTable("AuditMessages"); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.FailedAuditImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("ExceptionInfo") + .HasColumnType("nvarchar(max)"); + + b.Property("FailedAt") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageId") + .HasColumnType("nvarchar(max)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("FailedAt", "UniqueMessageId"); + + b.ToTable("FailedAuditImports"); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.SagaSnapshotEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Endpoint") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishTime") + .HasColumnType("datetime2"); + + b.Property("InitiatingMessageJson") + .HasColumnType("nvarchar(max)"); + + b.Property("OutgoingMessagesJson") + .HasColumnType("nvarchar(max)"); + + b.Property("ProcessedAt") + .HasColumnType("datetime2"); + + b.Property("SagaId") + .HasColumnType("uniqueidentifier"); + + b.Property("SagaType") + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("StateAfterChange") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("CreatedOn", "Id"); + + b.HasIndex("SagaId", "FinishTime"); + + b.ToTable("SagaSnapshots"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032751_AddFullTextSearch.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032751_AddFullTextSearch.cs new file mode 100644 index 0000000000..561c6c6e75 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/20260928032751_AddFullTextSearch.cs @@ -0,0 +1,25 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer.Migrations +{ + /// + public partial class AddFullTextSearch : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(FullTextSearchSql.RequireFullTextSearch, suppressTransaction: true); + migrationBuilder.Sql(FullTextSearchSql.CreateCatalog, suppressTransaction: true); + migrationBuilder.Sql(FullTextSearchSql.CreateIndex, suppressTransaction: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(FullTextSearchSql.DropIndex, suppressTransaction: true); + migrationBuilder.Sql(FullTextSearchSql.DropCatalog, suppressTransaction: true); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/SqlServerAuditDbContextModelSnapshot.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/SqlServerAuditDbContextModelSnapshot.cs new file mode 100644 index 0000000000..397af20956 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/Migrations/SqlServerAuditDbContextModelSnapshot.cs @@ -0,0 +1,202 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Audit.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerAuditDbContext))] + partial class SqlServerAuditDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.AuditMessageEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BodyContentType") + .HasColumnType("nvarchar(max)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyState") + .HasColumnType("int"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("MessageId") + .HasColumnType("nvarchar(max)"); + + b.Property("MessageType") + .HasColumnType("nvarchar(max)"); + + b.Property("ProcessedAt") + .HasColumnType("datetime2"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint"); + + b.Property("ReceivingEndpointHost") + .HasColumnType("nvarchar(max)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasColumnType("nvarchar(max)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("CreatedOn", "Id"); + + b.HasIndex("ConversationId"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("ProcessedAt"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.HasIndex("ReceivingEndpointName", "CreatedOn"); + + b.ToTable("AuditMessages"); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.FailedAuditImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("ExceptionInfo") + .HasColumnType("nvarchar(max)"); + + b.Property("FailedAt") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageId") + .HasColumnType("nvarchar(max)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("FailedAt", "UniqueMessageId"); + + b.ToTable("FailedAuditImports"); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.EFCore.Entities.SagaSnapshotEntity", b => + { + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Endpoint") + .HasColumnType("nvarchar(max)"); + + b.Property("FinishTime") + .HasColumnType("datetime2"); + + b.Property("InitiatingMessageJson") + .HasColumnType("nvarchar(max)"); + + b.Property("OutgoingMessagesJson") + .HasColumnType("nvarchar(max)"); + + b.Property("ProcessedAt") + .HasColumnType("datetime2"); + + b.Property("SagaId") + .HasColumnType("uniqueidentifier"); + + b.Property("SagaType") + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("StateAfterChange") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("CreatedOn", "Id"); + + b.HasIndex("SagaId", "FinishTime"); + + b.ToTable("SagaSnapshots"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SchemaStampingSqlServerMigrationsSqlGenerator.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SchemaStampingSqlServerMigrationsSqlGenerator.cs new file mode 100644 index 0000000000..210a05084d --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SchemaStampingSqlServerMigrationsSqlGenerator.cs @@ -0,0 +1,37 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Update; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +sealed class SchemaStampingSqlServerMigrationsSqlGenerator( + MigrationsSqlGeneratorDependencies dependencies, + ICommandBatchPreparer commandBatchPreparer, + IDbContextOptions contextOptions) + : SqlServerMigrationsSqlGenerator(dependencies, commandBatchPreparer) +{ + readonly string? schema = contextOptions.FindExtension()?.Schema; + + public override IReadOnlyList Generate( + IReadOnlyList operations, + IModel? model = null, + MigrationsSqlGenerationOptions options = MigrationsSqlGenerationOptions.Default) + { + if (schema is null) + { + return base.Generate(operations, model, options); + } + + MigrationOperation[] stamped = + [ + .. operations.Select(operation => operation is SqlOperation sql + ? FullTextSearchSql.Rewrite(sql, schema) + : MigrationSchemaStamper.Stamp(operation, schema)) + ]; + + return base.Generate(stamped, model, options); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/ServiceControl.Audit.Persistence.EFCore.SqlServer.csproj b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/ServiceControl.Audit.Persistence.EFCore.SqlServer.csproj new file mode 100644 index 0000000000..19212afd31 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/ServiceControl.Audit.Persistence.EFCore.SqlServer.csproj @@ -0,0 +1,49 @@ + + + + net10.0 + enable + enable + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerAuditDbContext.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerAuditDbContext.cs new file mode 100644 index 0000000000..829680fecc --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerAuditDbContext.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Entities; + +public class SqlServerAuditDbContext(DbContextOptions options) : AuditDbContext(options) +{ + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + // The full text index needs a unique single column key, and the primary key has two columns. + modelBuilder.Entity().HasIndex(e => e.Id).IsUnique(); + } + + public override bool IsDuplicateKeyException(DbUpdateException exception) + { + for (var inner = exception.InnerException; inner != null; inner = inner.InnerException) + { + if (inner is SqlException { Number: 2601 or 2627 }) + { + return true; + } + } + + return false; + } + + public override async Task SchemaExists(string schema, CancellationToken cancellationToken = default) => + await Database + .SqlQueryRaw("SELECT CASE WHEN SCHEMA_ID({0}) IS NULL THEN 0 ELSE 1 END AS [Value]", schema) + .SingleAsync(cancellationToken) == 1; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerAuditDbContextFactory.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerAuditDbContextFactory.cs new file mode 100644 index 0000000000..eae870967a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerAuditDbContextFactory.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; + +public class SqlServerAuditDbContextFactory : IDesignTimeDbContextFactory +{ + public SqlServerAuditDbContext CreateDbContext(string[] args) + { + var connectionString = Environment.GetEnvironmentVariable("SERVICECONTROL_DATABASE_CONNECTIONSTRING") + ?? "Server=localhost;Database=ServiceControlAudit;Trusted_Connection=True;TrustServerCertificate=True"; + + var optionsBuilder = new DbContextOptionsBuilder(); + optionsBuilder.UseSqlServer(connectionString, sqlOptions => sqlOptions.MigrationsHistoryTable(AuditDbContext.MigrationsHistoryTableName)); + + return new SqlServerAuditDbContext(optionsBuilder.Options); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerAuditPartitionManager.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerAuditPartitionManager.cs new file mode 100644 index 0000000000..65e436b7f1 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerAuditPartitionManager.cs @@ -0,0 +1,45 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +// No partitions, because a full text index cannot be aligned to a partition scheme. +class SqlServerAuditPartitionManager : IAuditPartitionManager +{ + public Task EnsurePartitions(AuditDbContext dbContext, DateTime fromHour, DateTime toHourExclusive, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public async Task> ListExpired(AuditDbContext dbContext, DateTime keepFrom, CancellationToken cancellationToken = default) + { + var messageHours = dbContext.AuditMessages.Where(message => message.CreatedOn < keepFrom).Select(message => message.CreatedOn); + var snapshotHours = dbContext.SagaSnapshots.Where(snapshot => snapshot.CreatedOn < keepFrom).Select(snapshot => snapshot.CreatedOn); + + return await messageHours.Union(snapshotHours).OrderBy(hour => hour).ToListAsync(cancellationToken); + } + + public async Task DropExpired(AuditDbContext dbContext, DateTime hour, int batchSize, CancellationToken cancellationToken = default) + { + var messagesDeleted = await dbContext.AuditMessages + .Where(message => message.CreatedOn == hour) + .OrderBy(message => message.Id) + .Take(batchSize) + .ExecuteDeleteAsync(cancellationToken); + + if (messagesDeleted == batchSize) + { + return new DropResult(messagesDeleted, Completed: false); + } + + var snapshotsDeleted = await dbContext.SagaSnapshots + .Where(snapshot => snapshot.CreatedOn == hour) + .OrderBy(snapshot => snapshot.Id) + .Take(batchSize) + .ExecuteDeleteAsync(cancellationToken); + + return new DropResult(messagesDeleted + snapshotsDeleted, Completed: snapshotsDeleted < batchSize); + } + + public Task ProvisionedUntil(AuditDbContext dbContext, CancellationToken cancellationToken = default) => + Task.FromResult(null); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs new file mode 100644 index 0000000000..a9608980a2 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class SqlServerFullTextSearchDialect : IFullTextSearchDialect +{ + public IQueryable Search(IQueryable source, string searchTerms) => + source.Where(message => + EF.Functions.FreeText(message.HeadersJson, searchTerms) || + EF.Functions.FreeText(message.BodyText!, searchTerms)); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs new file mode 100644 index 0000000000..b7db7a8bbb --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class SqlServerPersistence(EFPersisterSettings settings) : EFPersistenceBase(settings) +{ + protected override void AddDbContext(IServiceCollection services, EFPersisterSettings settings) + { + services.AddDbContext(options => + { + options.UseSqlServer(settings.ConnectionString, sqlOptions => + { + sqlOptions.CommandTimeout(settings.CommandTimeout); + sqlOptions.MigrationsHistoryTable(AuditDbContext.MigrationsHistoryTableName, settings.Schema); + + if (settings.EnableRetryOnFailure) + { + sqlOptions.EnableRetryOnFailure( + maxRetryCount: settings.MaxRetryCount, + maxRetryDelay: TimeSpan.FromSeconds(settings.MaxRetryDelayInSeconds), + errorNumbersToAdd: null); + } + }); + + if (settings.Schema is not null) + { + ((IDbContextOptionsBuilderInfrastructure)options).AddOrUpdateExtension(new SchemaOptionsExtension(settings.Schema)); + options.ReplaceService(); + options.ReplaceService(); + + // HasDefaultSchema makes the model differ from the migrations snapshot on purpose. + options.ConfigureWarnings(warnings => warnings.Ignore(RelationalEventId.PendingModelChangesWarning)); + } + + if (settings.EnableSensitiveDataLogging) + { + options.EnableSensitiveDataLogging(); + } + }, ServiceLifetime.Scoped); + + services.AddScoped(provider => provider.GetRequiredService()); + } + + protected override void AddPartitionManager(IServiceCollection services) => + services.AddSingleton(); + + protected override void AddQueryServices(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistenceConfiguration.cs new file mode 100644 index 0000000000..cb340a6e70 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerPersistenceConfiguration.cs @@ -0,0 +1,10 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using ServiceControl.Audit.Persistence.EFCore.Abstractions; + +class SqlServerPersistenceConfiguration : EFPersistenceConfigurationBase +{ + public override string Name => "SQLServer"; + + protected override IPersistence Create(EFPersisterSettings settings) => new SqlServerPersistence(settings); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerRetentionLock.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerRetentionLock.cs new file mode 100644 index 0000000000..5d1692d5b3 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerRetentionLock.cs @@ -0,0 +1,65 @@ +namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; + +using System.Data; +using Microsoft.Data.SqlClient; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +class SqlServerRetentionLock(EFPersisterSettings settings) : IRetentionLock +{ + // Unpooled, because a session lock would stay with a pooled connection after it returns to the pool. + readonly string connectionString = new SqlConnectionStringBuilder(settings.ConnectionString) { Pooling = false }.ConnectionString; + readonly string resource = RetentionLock.ResourceName(settings.Schema); + + public async Task TryAcquire(CancellationToken cancellationToken = default) + { + var connection = new SqlConnection(connectionString); + var acquired = false; + try + { + await connection.OpenAsync(cancellationToken); + + await using var command = connection.CreateCommand(); + command.CommandText = "sp_getapplock"; + command.CommandType = CommandType.StoredProcedure; + command.Parameters.AddWithValue("@Resource", resource); + command.Parameters.AddWithValue("@LockMode", "Exclusive"); + command.Parameters.AddWithValue("@LockOwner", "Session"); + command.Parameters.AddWithValue("@LockTimeout", 0); + var result = command.Parameters.Add("@Result", SqlDbType.Int); + result.Direction = ParameterDirection.ReturnValue; + + await command.ExecuteNonQueryAsync(cancellationToken); + + acquired = result.Value is int status && status >= 0; + return acquired ? new Handle(connection, resource) : null; + } + finally + { + if (!acquired) + { + await connection.DisposeAsync(); + } + } + } + + sealed class Handle(SqlConnection connection, string resource) : IAsyncDisposable + { + public async ValueTask DisposeAsync() + { + try + { + await using var command = connection.CreateCommand(); + command.CommandText = "sp_releaseapplock"; + command.CommandType = CommandType.StoredProcedure; + command.Parameters.AddWithValue("@Resource", resource); + command.Parameters.AddWithValue("@LockOwner", "Session"); + await command.ExecuteNonQueryAsync(); + } + finally + { + await connection.DisposeAsync(); + } + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/persistence.manifest b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/persistence.manifest new file mode 100644 index 0000000000..8c7628c10f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/persistence.manifest @@ -0,0 +1,21 @@ +{ + "Name": "SQLServer", + "DisplayName": "SQL Server", + "Description": "SQL Server ServiceControl Audit persister", + "AssemblyName": "ServiceControl.Audit.Persistence.EFCore.SqlServer", + "TypeName": "ServiceControl.Audit.Persistence.EFCore.SqlServer.SqlServerPersistenceConfiguration, ServiceControl.Audit.Persistence.EFCore.SqlServer", + "Settings": [ + { + "Name": "ServiceControl.Audit/Database/ConnectionString", + "Mandatory": true + }, + { + "Name": "ServiceControl.Audit/Database/Schema", + "Mandatory": false + }, + { + "Name": "ServiceControl.Audit/Database/CommandTimeout", + "Mandatory": false + } + ] +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/.editorconfig b/src/ServiceControl.Audit.Persistence.EFCore/.editorconfig new file mode 100644 index 0000000000..fc68ac3228 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/.editorconfig @@ -0,0 +1,9 @@ +[*.cs] + +# Justification: ServiceControl app has no synchronization context +dotnet_diagnostic.CA2007.severity = none + +# Disable style rules for auto-generated EF migrations +[Migrations/**.cs] +dotnet_diagnostic.IDE0065.severity = none +generated_code = true diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs new file mode 100644 index 0000000000..dea2331e99 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Abstractions; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using NServiceBus; +using ServiceControl.Audit.Persistence.EFCore.Implementation; +using ServiceControl.Audit.Persistence.EFCore.Implementation.UnitOfWork; +using ServiceControl.Audit.Persistence.UnitOfWork; + +abstract class EFPersistenceBase(EFPersisterSettings settings) : IPersistence +{ + public void AddPersistence(IServiceCollection services) + { + AddCommon(services); + AddQueryServices(services); + + services.AddSingleton(); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton(); + services.AddHostedService(); + + if (services.SingleOrDefault(s => s.ServiceType == typeof(EndpointConfiguration)) is + { + ImplementationInstance: EndpointConfiguration endpointConfiguration + }) + { + endpointConfiguration.AddCustomCheck(); + AddCustomChecks(endpointConfiguration); + } + } + + public void AddInstaller(IServiceCollection services) + { + AddCommon(services); + services.AddHostedService(); + } + + void AddCommon(IServiceCollection services) + { + services.AddSingleton(settings); + services.TryAddSingleton(TimeProvider.System); + AddDbContext(services, settings); + AddPartitionManager(services); + } + + protected abstract void AddDbContext(IServiceCollection services, EFPersisterSettings settings); + + protected abstract void AddPartitionManager(IServiceCollection services); + + protected abstract void AddQueryServices(IServiceCollection services); + + protected virtual void AddCustomChecks(EndpointConfiguration endpointConfiguration) + { + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs new file mode 100644 index 0000000000..4b1bb5da46 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs @@ -0,0 +1,74 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Abstractions; + +using Microsoft.Extensions.Logging; +using ServiceControl.Infrastructure; + +public abstract class EFPersistenceConfigurationBase : IPersistenceConfiguration +{ + public const string ConnectionStringKey = "Database/ConnectionString"; + public const string SchemaKey = "Database/Schema"; + public const string CommandTimeoutKey = "Database/CommandTimeout"; + public const string QueryTimeoutInSecondsKey = QueryTimeLimit.SettingName; + internal const string QueryTimeoutSettingName = "ServiceControl.Audit/" + QueryTimeLimit.SettingName; + + public bool SupportsMaintenanceMode => false; + + public abstract string Name { get; } + + public IEnumerable ConfigurationKeys => [ConnectionStringKey, SchemaKey, CommandTimeoutKey, QueryTimeoutInSecondsKey]; + + public IPersistence Create(PersistenceSettings settings) => Create(CreateSettings(settings)); + + protected abstract IPersistence Create(EFPersisterSettings settings); + + public static EFPersisterSettings CreateSettings(PersistenceSettings settings) + { + var specific = settings.PersisterSpecificSettings; + + if (!specific.TryGetValue(ConnectionStringKey, out var connectionString) || string.IsNullOrWhiteSpace(connectionString)) + { + throw new InvalidOperationException($"{ConnectionStringKey} must be specified."); + } + + return new EFPersisterSettings + { + ConnectionString = connectionString, + Schema = ReadSchema(specific), + CommandTimeout = ReadInt(specific, CommandTimeoutKey, EFPersisterSettings.DefaultCommandTimeout), + QueryTimeout = QueryTimeLimit.Validate(ReadInt(specific, QueryTimeoutInSecondsKey, QueryTimeLimit.DefaultSeconds), QueryTimeoutSettingName, logger), + AuditRetentionPeriod = settings.AuditRetentionPeriod, + MaxBodySizeToStore = settings.MaxBodySizeToStore + }; + } + + static string? ReadSchema(IDictionary specific) + { + if (!specific.TryGetValue(SchemaKey, out var schema) || string.IsNullOrWhiteSpace(schema)) + { + return null; + } + + try + { + return SchemaName.Validate(schema); + } + catch (ArgumentException e) + { + throw new InvalidOperationException($"{SchemaKey} is invalid. {e.Message}", e); + } + } + + static int ReadInt(IDictionary specific, string key, int defaultValue) + { + if (!specific.TryGetValue(key, out var value)) + { + return defaultValue; + } + + return int.TryParse(value, out var parsed) + ? parsed + : throw new InvalidOperationException($"{key} must be an integer."); + } + + static readonly ILogger logger = LoggerUtil.CreateStaticLogger(); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersisterSettings.cs b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersisterSettings.cs new file mode 100644 index 0000000000..f441412deb --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersisterSettings.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Abstractions; + +using ServiceControl.Infrastructure; + +public class EFPersisterSettings +{ + public static readonly TimeSpan MigrationCommandTimeout = TimeSpan.FromMinutes(40); + + public const int DefaultCommandTimeout = 30; + + public required string ConnectionString { get; init; } + + public string? Schema + { + get; + init => field = value is null ? null : SchemaName.Validate(value); + } + + public int CommandTimeout { get; init; } = DefaultCommandTimeout; + public TimeSpan QueryTimeout { get; init; } = QueryTimeLimit.Default; + public required TimeSpan AuditRetentionPeriod { get; init; } + public required int MaxBodySizeToStore { get; init; } + public int MaxRetryCount { get; init; } = 5; + public int MaxRetryDelayInSeconds { get; init; } = 30; + public bool EnableRetryOnFailure { get; init; } = true; + public bool EnableSensitiveDataLogging { get; init; } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/SchemaName.cs b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/SchemaName.cs new file mode 100644 index 0000000000..cff0fb4ccd --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Abstractions/SchemaName.cs @@ -0,0 +1,33 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Abstractions; + +using System.Text.RegularExpressions; + +// The name is interpolated into DDL and raw SQL, so this allowlist is what keeps that safe. +public static partial class SchemaName +{ + // PostgreSQL truncates identifiers at 63 bytes. + public const int MaxLength = 63; + + public static string Validate(string schema) + { + if (string.IsNullOrWhiteSpace(schema)) + { + throw new ArgumentException("A database schema name cannot be empty.", nameof(schema)); + } + + if (schema.Length > MaxLength) + { + throw new ArgumentException($"Database schema name '{schema}' is longer than the {MaxLength} character limit.", nameof(schema)); + } + + if (!AllowedName().IsMatch(schema)) + { + throw new ArgumentException($"Database schema name '{schema}' is not valid. Use a letter or an underscore followed by letters, digits or underscores.", nameof(schema)); + } + + return schema; + } + + [GeneratedRegex("^[A-Za-z_][A-Za-z0-9_]*$")] + private static partial Regex AllowedName(); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/DbContexts/AuditDbContext.cs b/src/ServiceControl.Audit.Persistence.EFCore/DbContexts/AuditDbContext.cs new file mode 100644 index 0000000000..284e90b81e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/DbContexts/AuditDbContext.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Audit.Persistence.EFCore.DbContexts; + +using Microsoft.EntityFrameworkCore; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.EntityConfigurations; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +public abstract class AuditDbContext(DbContextOptions options) : DbContext(options) +{ + // Not EF Core's default, so an audit instance can share a schema with a primary instance. + public const string MigrationsHistoryTableName = "__AuditMigrationsHistory"; + + public string? Schema { get; } = options.FindExtension()?.Schema; + + public DbSet AuditMessages { get; set; } + public DbSet SagaSnapshots { get; set; } + public DbSet FailedAuditImports { get; set; } + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.EnableDetailedErrors(); + + protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) + { + base.ConfigureConventions(configurationBuilder); + + configurationBuilder.Properties().HaveConversion(); + configurationBuilder.Properties().HaveConversion(); + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + if (Schema is not null) + { + modelBuilder.HasDefaultSchema(Schema); + } + + modelBuilder.ApplyConfiguration(new AuditMessageConfiguration()); + modelBuilder.ApplyConfiguration(new SagaSnapshotConfiguration()); + modelBuilder.ApplyConfiguration(new FailedAuditImportConfiguration()); + } + + public abstract bool IsDuplicateKeyException(DbUpdateException exception); + + public abstract Task SchemaExists(string schema, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/DbContexts/UtcDateTimeConverters.cs b/src/ServiceControl.Audit.Persistence.EFCore/DbContexts/UtcDateTimeConverters.cs new file mode 100644 index 0000000000..a052b4e49d --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/DbContexts/UtcDateTimeConverters.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Audit.Persistence.EFCore.DbContexts; + +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +// SQL Server's datetime2 has no offset, so values read back as Unspecified. Npgsql refuses to write a Local or Unspecified value to timestamptz. EF applies these converters to query parameters as well, so a date range from the API needs no conversion of its own. +sealed class UtcDateTimeConverter() : ValueConverter( + value => ToUtc(value), + value => DateTime.SpecifyKind(value, DateTimeKind.Utc)) +{ + public static DateTime ToUtc(DateTime value) => + value.Kind == DateTimeKind.Local ? value.ToUniversalTime() : DateTime.SpecifyKind(value, DateTimeKind.Utc); +} + +sealed class NullableUtcDateTimeConverter() : ValueConverter( + value => value.HasValue ? UtcDateTimeConverter.ToUtc(value.Value) : value, + value => value.HasValue ? DateTime.SpecifyKind(value.Value, DateTimeKind.Utc) : value); diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Entities/AuditMessageEntity.cs b/src/ServiceControl.Audit.Persistence.EFCore/Entities/AuditMessageEntity.cs new file mode 100644 index 0000000000..dc9090a734 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Entities/AuditMessageEntity.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Entities; + +using ServiceControl.Audit.Monitoring; + +public class AuditMessageEntity +{ + public DateTime CreatedOn { get; set; } + public long Id { get; set; } + public Guid UniqueMessageId { get; set; } + public string? MessageId { get; set; } + public string? MessageType { get; set; } + public DateTime? TimeSent { get; set; } + public DateTime ProcessedAt { get; set; } + public string? ConversationId { get; set; } + public bool IsSystemMessage { get; set; } + public MessageStatus Status { get; set; } + public string? SendingEndpointName { get; set; } + public Guid? SendingEndpointHostId { get; set; } + public string? SendingEndpointHost { get; set; } + public string? ReceivingEndpointName { get; set; } + public Guid? ReceivingEndpointHostId { get; set; } + public string? ReceivingEndpointHost { get; set; } + public long? CriticalTimeTicks { get; set; } + public long? ProcessingTimeTicks { get; set; } + public long? DeliveryTimeTicks { get; set; } + public required string HeadersJson { get; set; } + public string? BodyText { get; set; } + public BodyState BodyState { get; set; } + public int BodySize { get; set; } + public string? BodyContentType { get; set; } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Entities/BodyState.cs b/src/ServiceControl.Audit.Persistence.EFCore/Entities/BodyState.cs new file mode 100644 index 0000000000..59d773602b --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Entities/BodyState.cs @@ -0,0 +1,9 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Entities; + +public enum BodyState +{ + None = 0, + Stored = 1, + TooLarge = 2, + NotText = 3 +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Entities/FailedAuditImportEntity.cs b/src/ServiceControl.Audit.Persistence.EFCore/Entities/FailedAuditImportEntity.cs new file mode 100644 index 0000000000..9c4ae20e1f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Entities/FailedAuditImportEntity.cs @@ -0,0 +1,11 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Entities; + +public class FailedAuditImportEntity +{ + public Guid UniqueMessageId { get; set; } + public DateTime FailedAt { get; set; } + public string? MessageId { get; set; } + public required string HeadersJson { get; set; } + public required byte[] Body { get; set; } + public string? ExceptionInfo { get; set; } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Entities/SagaSnapshotEntity.cs b/src/ServiceControl.Audit.Persistence.EFCore/Entities/SagaSnapshotEntity.cs new file mode 100644 index 0000000000..2958712327 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Entities/SagaSnapshotEntity.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Entities; + +using ServiceControl.SagaAudit; + +public class SagaSnapshotEntity +{ + public DateTime CreatedOn { get; set; } + public long Id { get; set; } + public Guid SagaId { get; set; } + public string? SagaType { get; set; } + public SagaStateChangeStatus Status { get; set; } + public DateTime StartTime { get; set; } + public DateTime FinishTime { get; set; } + public DateTime ProcessedAt { get; set; } + public string? Endpoint { get; set; } + public string? StateAfterChange { get; set; } + public string? InitiatingMessageJson { get; set; } + public string? OutgoingMessagesJson { get; set; } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/AuditMessageConfiguration.cs b/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/AuditMessageConfiguration.cs new file mode 100644 index 0000000000..3d85ce3114 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/AuditMessageConfiguration.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Audit.Persistence.EFCore.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class AuditMessageConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(e => new { e.CreatedOn, e.Id }); + builder.Property(e => e.CreatedOn).ValueGeneratedNever(); + builder.Property(e => e.Id).ValueGeneratedOnAdd(); + + builder.Property(e => e.UniqueMessageId).IsRequired(); + builder.Property(e => e.ProcessedAt).IsRequired(); + builder.Property(e => e.IsSystemMessage).IsRequired(); + builder.Property(e => e.Status).IsRequired(); + builder.Property(e => e.HeadersJson).IsRequired(); + builder.Property(e => e.BodyState).IsRequired(); + builder.Property(e => e.BodySize).IsRequired(); + + builder.Property(e => e.ConversationId).HasMaxLength(ColumnLengths.ShortTextLength).HasConversion(); + builder.Property(e => e.ReceivingEndpointName).HasMaxLength(ColumnLengths.ShortTextLength); + + builder.HasIndex(e => e.UniqueMessageId); + builder.HasIndex(e => new { e.ReceivingEndpointName, e.CreatedOn }); + builder.HasIndex(e => e.ConversationId); + builder.HasIndex(e => e.ProcessedAt); + builder.HasIndex(e => e.TimeSent); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs b/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs new file mode 100644 index 0000000000..dc18ab4e1b --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Audit.Persistence.EFCore.EntityConfigurations; + +using System.Security.Cryptography; +using System.Text; + +static class ColumnLengths +{ + public const int ShortTextLength = 450; + + public static string FitToIndex(string value) + { + if (value.Length <= ShortTextLength) + { + return value; + } + + var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + var prefixLength = ShortTextLength - hash.Length - 1; + + if (char.IsHighSurrogate(value[prefixLength - 1])) + { + prefixLength--; + } + + return $"{value[..prefixLength]}~{hash}"; + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/FailedAuditImportConfiguration.cs b/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/FailedAuditImportConfiguration.cs new file mode 100644 index 0000000000..1ed8fcf384 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/FailedAuditImportConfiguration.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Audit.Persistence.EFCore.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class FailedAuditImportConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(e => e.UniqueMessageId); + builder.Property(e => e.UniqueMessageId).ValueGeneratedNever(); + builder.Property(e => e.FailedAt).IsRequired(); + builder.Property(e => e.HeadersJson).IsRequired(); + builder.Property(e => e.Body).IsRequired(); + + builder.HasIndex(e => new { e.FailedAt, e.UniqueMessageId }); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/FitToIndexConverter.cs b/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/FitToIndexConverter.cs new file mode 100644 index 0000000000..448a2ec92f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/FitToIndexConverter.cs @@ -0,0 +1,5 @@ +namespace ServiceControl.Audit.Persistence.EFCore.EntityConfigurations; + +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +sealed class FitToIndexConverter() : ValueConverter(value => ColumnLengths.FitToIndex(value), value => value); diff --git a/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/SagaSnapshotConfiguration.cs b/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/SagaSnapshotConfiguration.cs new file mode 100644 index 0000000000..c087dfe19f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/EntityConfigurations/SagaSnapshotConfiguration.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Audit.Persistence.EFCore.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class SagaSnapshotConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(e => new { e.CreatedOn, e.Id }); + builder.Property(e => e.CreatedOn).ValueGeneratedNever(); + builder.Property(e => e.Id).ValueGeneratedOnAdd(); + + builder.Property(e => e.SagaId).IsRequired(); + builder.Property(e => e.Status).IsRequired(); + builder.Property(e => e.StartTime).IsRequired(); + builder.Property(e => e.FinishTime).IsRequired(); + builder.Property(e => e.ProcessedAt).IsRequired(); + + builder.HasIndex(e => new { e.SagaId, e.FinishTime }); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/AuditDataStore.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/AuditDataStore.cs new file mode 100644 index 0000000000..4bd9e8d840 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/AuditDataStore.cs @@ -0,0 +1,185 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Audit.Auditing; +using ServiceControl.Audit.Auditing.MessagesView; +using ServiceControl.Audit.Infrastructure; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; +using ServiceControl.Infrastructure; +using ServiceControl.SagaAudit; + +sealed class AuditDataStore( + IServiceScopeFactory scopeFactory, + EFPersisterSettings settings, + IFullTextSearchDialect fullTextSearch, + TimeProvider timeProvider) : DataStoreBase(scopeFactory, settings), IAuditMessagesViewDataStore, ISagaHistoryDataStore +{ + public const int TotalCountCap = 100_000; + + const int SagaHistoryCap = 50_000; + + public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) => + Page(messages => messages + .IncludeSystemMessagesWhere(includeSystemMessages) + .FilterBySentTimeRange(timeSentRange), pagingInfo, sortInfo, cancellationToken); + + public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) => + Page(messages => Search(messages, searchParam) + .FilterBySentTimeRange(timeSentRange), pagingInfo, sortInfo, cancellationToken); + + public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) => + Page(messages => Search(messages.Where(message => message.ReceivingEndpointName == endpoint), keyword) + .FilterBySentTimeRange(timeSentRange), pagingInfo, sortInfo, cancellationToken); + + public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) => + Page(messages => messages + .Where(message => message.ReceivingEndpointName == endpointName) + .IncludeSystemMessagesWhere(includeSystemMessages) + .FilterBySentTimeRange(timeSentRange), pagingInfo, sortInfo, cancellationToken); + + public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken = default) => + Page(messages => messages.Where(message => message.ConversationId == conversationId), pagingInfo, sortInfo, cancellationToken); + + public Task GetMessageBody(string messageId, CancellationToken cancellationToken = default) => + ExecuteQueryWithDbContext(async (dbContext, token) => + { + IQueryable candidates; + if (AuditBodyId.TryParse(messageId, out var createdOn, out var uniqueMessageId)) + { + candidates = dbContext.AuditMessages.Where(message => message.CreatedOn == createdOn && message.UniqueMessageId == uniqueMessageId); + } + else if (Guid.TryParse(messageId, out uniqueMessageId)) + { + candidates = dbContext.AuditMessages.Where(message => message.UniqueMessageId == uniqueMessageId); + } + else + { + return MessageBodyView.NotFound(); + } + + var body = await candidates + .AsNoTracking() + .OrderByDescending(message => message.CreatedOn) + .ThenByDescending(message => message.Id) + .Select(message => new { message.CreatedOn, message.Id, message.BodyState, message.BodyText, message.BodyContentType, message.BodySize }) + .FirstOrDefaultAsync(token); + + if (body is null) + { + return MessageBodyView.NotFound(); + } + + return body.BodyState switch + { + BodyState.Stored => MessageBodyView.FromString(body.BodyText!, body.BodyContentType ?? "text/plain", body.BodySize, DataVersion.Compose(("row", $"{body.CreatedOn.Ticks}-{body.Id}"))), + BodyState.None or BodyState.TooLarge => MessageBodyView.NoContent(), + BodyState.NotText => MessageBodyView.NotFound(), + _ => throw new InvalidOperationException($"Unknown body state {body.BodyState}") + }; + }, cancellationToken); + + public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken = default) => + ExecuteQueryWithDbContext(async (dbContext, token) => + { + var today = timeProvider.GetUtcNow().UtcDateTime.Date; + var from = today.AddDays(-29); + var to = today.AddDays(1); + + var ingestedFrom = from.AddDays(-1); + + var received = dbContext.AuditMessages.AsNoTracking().Where(message => message.ReceivingEndpointName == endpointName); + + var days = await received + .Where(message => !message.IsSystemMessage + && message.CreatedOn >= ingestedFrom + && message.ProcessedAt >= from + && message.ProcessedAt < to) + .GroupBy(message => message.ProcessedAt.Date) + .Select(day => new { Date = day.Key, Count = day.LongCount() }) + .OrderBy(day => day.Date) + .ToListAsync(token); + + List counts = [.. days.Select(day => new AuditCount { UtcDate = DateTime.SpecifyKind(day.Date, DateTimeKind.Utc), Count = day.Count })]; + + if (counts.Count == 0 + && !await received.AnyAsync(token) + && await dbContext.AuditMessages.AnyAsync(message => message.SendingEndpointName == endpointName && message.CreatedOn >= ingestedFrom, token)) + { + counts.Add(new AuditCount { UtcDate = today, Count = 0 }); + } + + return new QueryResult>(counts, QueryStatsInfo.Zero); + }, cancellationToken); + + public Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken = default) => + ExecuteQueryWithDbContext(async (dbContext, token) => + { + var snapshots = await dbContext.SagaSnapshots + .AsNoTracking() + .Where(snapshot => snapshot.SagaId == input) + .OrderByDescending(snapshot => snapshot.FinishTime) + .ThenByDescending(snapshot => snapshot.CreatedOn) + .ThenByDescending(snapshot => snapshot.Id) + .Take(SagaHistoryCap) + .ToListAsync(token); + + if (snapshots.Count == 0) + { + return QueryResult.Empty(); + } + + var history = new SagaHistory + { + Id = input, + SagaId = input, + SagaType = snapshots[0].SagaType, + Changes = [.. snapshots.Select(ToStateChange)] + }; + + var version = DataVersion.OverRows([("changes", snapshots.Count)], snapshots, snapshot => [snapshot.CreatedOn, snapshot.Id]); + + return new QueryResult(history, new QueryStatsInfo(version, snapshots.Count)); + }, cancellationToken); + + Task>> Page( + Func, IQueryable> filter, + PagingInfo pagingInfo, + SortInfo sortInfo, + CancellationToken cancellationToken) => + ExecuteQueryWithDbContext(async (dbContext, token) => + { + var messages = filter(dbContext.AuditMessages.AsNoTracking()); + + var total = await messages.Take(TotalCountCap).LongCountAsync(token); + + var rows = await messages + .Sort(sortInfo) + .Skip(pagingInfo.Offset) + .Take(pagingInfo.PageSize) + .ToMessageRows() + .ToListAsync(token); + + IList results = [.. rows.Select(row => row.ToMessagesView())]; + + var version = DataVersion.OverRows([("messages", total)], rows, row => [row.CreatedOn, row.Id]); + + return new QueryResult>(results, new QueryStatsInfo(version, total)); + }, cancellationToken); + + IQueryable Search(IQueryable source, string searchTerms) => + string.IsNullOrWhiteSpace(searchTerms) ? source : fullTextSearch.Search(source, searchTerms); + + static SagaStateChange ToStateChange(SagaSnapshotEntity snapshot) => new() + { + StartTime = snapshot.StartTime, + FinishTime = snapshot.FinishTime, + Status = snapshot.Status, + StateAfterChange = snapshot.StateAfterChange, + InitiatingMessage = SagaSnapshotJson.ReadInitiatingMessage(snapshot.InitiatingMessageJson), + OutgoingMessages = SagaSnapshotJson.ReadOutgoingMessages(snapshot.OutgoingMessagesJson), + Endpoint = snapshot.Endpoint + }; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/AuditRetention.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/AuditRetention.cs new file mode 100644 index 0000000000..277f309d22 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/AuditRetention.cs @@ -0,0 +1,127 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +sealed class AuditRetention( + IServiceScopeFactory scopeFactory, + IAuditPartitionManager partitions, + IRetentionLock retentionLock, + AuditRetentionCustomCheck.State retentionState, + EFPersisterSettings settings, + TimeProvider timeProvider, + ILogger logger) : BackgroundService +{ + // Stays under the 5,000 locks at which SQL Server escalates a delete to a table lock. + const int BatchSize = 4_000; + static readonly TimeSpan Interval = TimeSpan.FromHours(1); + static readonly TimeSpan InitialDelay = TimeSpan.FromMinutes(1); + static readonly TimeSpan BatchPause = TimeSpan.FromSeconds(1); + + // Before ingestion starts, which cannot insert into an hour that has no partition. + public override async Task StartAsync(CancellationToken cancellationToken = default) + { + await using (var scope = scopeFactory.CreateAsyncScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + var hour = AuditHours.Truncate(timeProvider.GetUtcNow().UtcDateTime); + await partitions.EnsurePartitions(dbContext, hour.AddHours(-1), hour + AuditHours.Lookahead, cancellationToken); + } + + await base.StartAsync(cancellationToken); + } + + protected override async Task ExecuteAsync(CancellationToken cancellationToken = default) + { + try + { + await Task.Delay(InitialDelay, timeProvider, cancellationToken); + + using PeriodicTimer timer = new(Interval, timeProvider); + + do + { + try + { + await Sweep(pace: true, cancellationToken); + } +#pragma warning disable PS0019 // Filtered on the token alone because SqlClient reports a cancelled command as a SqlException. + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + logger.LogError(ex, "Error during audit retention sweep"); + } + } while (await timer.WaitForNextTickAsync(cancellationToken)); + } + catch (Exception) when (cancellationToken.IsCancellationRequested) + { + logger.LogInformation("Stopping audit retention sweep"); + } +#pragma warning restore PS0019 + } + + internal Task SweepNow(CancellationToken cancellationToken = default) => Sweep(pace: false, cancellationToken); + + async Task Sweep(bool pace, CancellationToken cancellationToken) + { + try + { + await using (var handle = await retentionLock.TryAcquire(cancellationToken)) + { + if (handle is null) + { + logger.LogWarning("Skipping the audit retention sweep because another instance holds the retention lock. Only one audit instance should own a database."); + return; + } + + await RemoveExpired(pace, cancellationToken); + } + + retentionState.SweepSucceeded(); + } +#pragma warning disable PS0019 // Filtered on the token alone because SqlClient reports a cancelled command as a SqlException. + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + retentionState.SweepFailed(ex.Message); + throw; + } +#pragma warning restore PS0019 + } + + async Task RemoveExpired(bool pace, CancellationToken cancellationToken) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var now = timeProvider.GetUtcNow().UtcDateTime; + var hour = AuditHours.Truncate(now); + await partitions.EnsurePartitions(dbContext, hour, hour + AuditHours.Lookahead, cancellationToken); + + var keepFrom = AuditHours.Truncate(now - settings.AuditRetentionPeriod); + + foreach (var expired in await partitions.ListExpired(dbContext, keepFrom, cancellationToken)) + { + DropResult drop; + do + { + drop = await partitions.DropExpired(dbContext, expired, BatchSize, cancellationToken); + + if (drop.Deferred) + { + logger.LogInformation("Left the audit rows from {Start:u} for the next sweep because the audit tables were too busy to drop them", expired); + return; + } + + if (pace && !drop.Completed) + { + await Task.Delay(BatchPause, timeProvider, cancellationToken); + } + } while (!drop.Completed); + + logger.LogDebug("Removed the expired audit rows from {Start:u}", expired); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/AuditRetentionCustomCheck.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/AuditRetentionCustomCheck.cs new file mode 100644 index 0000000000..5304e327b5 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/AuditRetentionCustomCheck.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using NServiceBus.CustomChecks; + +class AuditRetentionCustomCheck(AuditRetentionCustomCheck.State state) + : CustomCheck("Audit retention", "ServiceControl.Audit Health", TimeSpan.FromHours(1)) +{ + public override Task PerformCheck(CancellationToken cancellationToken = default) + { + var (consecutiveFailedSweeps, lastFailure) = state.Current; + + return Task.FromResult(consecutiveFailedSweeps < 3 + ? CheckResult.Pass + : CheckResult.Failed($"The last {consecutiveFailedSweeps} audit retention sweeps failed. Last failure: {lastFailure}. See https://docs.particular.net/servicecontrol/troubleshooting for guidance on resolving the issue.")); + } + + internal class State + { + readonly object gate = new(); + int consecutiveFailedSweeps; + string? lastFailure; + + public (int ConsecutiveFailedSweeps, string? LastFailure) Current + { + get + { + lock (gate) + { + return (consecutiveFailedSweeps, lastFailure); + } + } + } + + public void SweepSucceeded() + { + lock (gate) + { + consecutiveFailedSweeps = 0; + lastFailure = null; + } + } + + public void SweepFailed(string reason) + { + lock (gate) + { + consecutiveFailedSweeps++; + lastFailure = reason; + } + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/DataStoreBase.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/DataStoreBase.cs new file mode 100644 index 0000000000..b8be917fd1 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/DataStoreBase.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Infrastructure; + +abstract class DataStoreBase(IServiceScopeFactory scopeFactory, EFPersisterSettings settings) +{ + protected async Task ExecuteWithDbContext(Func> operation, CancellationToken cancellationToken = default) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + return await operation(dbContext, cancellationToken); + } + + protected async Task ExecuteWithDbContext(Func operation, CancellationToken cancellationToken = default) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + await operation(dbContext, cancellationToken); + } + + protected async Task ExecuteQueryWithDbContext(Func> query, CancellationToken cancellationToken = default) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + dbContext.Database.SetCommandTimeout(settings.QueryTimeout); + return await QueryTimeLimit.Run(token => query(dbContext, token), settings.QueryTimeout, EFPersistenceConfigurationBase.QueryTimeoutSettingName, cancellationToken); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/DatabaseSetup.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/DatabaseSetup.cs new file mode 100644 index 0000000000..e36630c426 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/DatabaseSetup.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +sealed class DatabaseSetup( + IServiceScopeFactory scopeFactory, + IAuditPartitionManager partitions, + TimeProvider timeProvider, + ILogger logger) : IHostedService +{ + public async Task StartAsync(CancellationToken cancellationToken = default) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + dbContext.Database.SetCommandTimeout(EFPersisterSettings.MigrationCommandTimeout); + + if (dbContext.Schema is not null && !await dbContext.SchemaExists(dbContext.Schema, cancellationToken)) + { + throw new InvalidOperationException( + $"The configured schema '{dbContext.Schema}' does not exist in the database. ServiceControl does not create schemas, the same way it does not create the database. Create the schema, grant the configured user rights on it, and run setup again."); + } + + logger.LogInformation("Starting audit database migration"); + await dbContext.Database.MigrateAsync(cancellationToken); + + var hour = AuditHours.Truncate(timeProvider.GetUtcNow().UtcDateTime); + await partitions.EnsurePartitions(dbContext, hour.AddHours(-1), hour + AuditHours.Lookahead, cancellationToken); + logger.LogInformation("Audit database migration completed"); + } + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/FailedAuditStorage.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/FailedAuditStorage.cs new file mode 100644 index 0000000000..9834dbe537 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/FailedAuditStorage.cs @@ -0,0 +1,105 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Audit.Auditing; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; +using ServiceControl.Audit.Persistence.Infrastructure; + +sealed class FailedAuditStorage(IServiceScopeFactory scopeFactory, EFPersisterSettings settings, TimeProvider timeProvider) + : DataStoreBase(scopeFactory, settings), IFailedAuditStorage +{ + const int BatchSize = 100; + + public Task SaveFailedAuditImport(FailedAuditImport message, CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + var key = Key(message.Id); + var failedAt = timeProvider.GetUtcNow().UtcDateTime; + var headersJson = MessageHeaders.Write(message.Message?.Headers ?? []); + var body = message.Message?.Body ?? []; + + await dbContext.UpsertAsync(key, () => new FailedAuditImportEntity + { + UniqueMessageId = key, + FailedAt = failedAt, + MessageId = message.Message?.Id, + HeadersJson = headersJson, + Body = body, + ExceptionInfo = message.ExceptionInfo + }, entity => + { + entity.FailedAt = failedAt; + entity.MessageId = message.Message?.Id; + entity.HeadersJson = headersJson; + entity.Body = body; + entity.ExceptionInfo = message.ExceptionInfo; + }, token); + }, cancellationToken); + + public async Task ProcessFailedMessages( + Func, CancellationToken, Task> onMessage, + CancellationToken cancellationToken = default) + { + var completed = new List(); + + for (var offset = 0; ; offset += BatchSize) + { + cancellationToken.ThrowIfCancellationRequested(); + + var batch = await ExecuteWithDbContext((dbContext, token) => dbContext.FailedAuditImports + .AsNoTracking() + .OrderBy(import => import.FailedAt) + .ThenBy(import => import.UniqueMessageId) + .Skip(offset) + .Take(BatchSize) + .ToListAsync(token), cancellationToken); + + foreach (var import in batch) + { + cancellationToken.ThrowIfCancellationRequested(); + + var transportMessage = new FailedTransportMessage + { + Id = import.MessageId, + Headers = MessageHeaders.Read(import.HeadersJson), + Body = import.Body + }; + + await onMessage(transportMessage, _ => + { + completed.Add(import.UniqueMessageId); + return Task.CompletedTask; + }, cancellationToken); + } + + if (batch.Count < BatchSize) + { + break; + } + } + + cancellationToken.ThrowIfCancellationRequested(); + + await ExecuteWithDbContext(async (dbContext, token) => + { + foreach (var chunk in completed.Chunk(1000)) + { + await dbContext.FailedAuditImports + .Where(import => chunk.Contains(import.UniqueMessageId)) + .ExecuteDeleteAsync(token); + } + }, cancellationToken); + } + + public Task GetFailedAuditsCount(CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, token) => dbContext.FailedAuditImports.CountAsync(token), cancellationToken); + + static Guid Key(string? id) => + id is null ? Guid.NewGuid() + : Guid.TryParse(id, out var parsed) ? parsed + : DeterministicGuid.MakeId(id); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/MessageQueries.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/MessageQueries.cs new file mode 100644 index 0000000000..98a6402ea0 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/MessageQueries.cs @@ -0,0 +1,127 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using System.Linq.Expressions; +using NServiceBus; +using ServiceControl.Audit.Auditing.MessagesView; +using ServiceControl.Audit.Infrastructure; +using ServiceControl.Audit.Monitoring; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; +using ServiceControl.SagaAudit; + +static class MessageQueries +{ + public static IQueryable IncludeSystemMessagesWhere(this IQueryable source, bool includeSystemMessages) => + includeSystemMessages ? source : source.Where(message => !message.IsSystemMessage); + + public static IQueryable FilterBySentTimeRange(this IQueryable source, DateTimeRange? timeSentRange) + { + if (timeSentRange?.From is { } from) + { + source = source.Where(message => message.TimeSent >= from); + } + + if (timeSentRange?.To is { } to) + { + source = source.Where(message => message.TimeSent <= to); + } + + return source; + } + + public static IOrderedQueryable Sort(this IQueryable source, SortInfo? sortInfo) + { + var descending = sortInfo?.Direction != "asc"; + + return sortInfo?.Sort switch + { + "id" or "message_id" => source.OrderBy(message => message.MessageId, descending), + "message_type" => source.OrderBy(message => message.MessageType, descending), + "critical_time" => source.OrderBy(message => message.CriticalTimeTicks, descending), + "delivery_time" => source.OrderBy(message => message.DeliveryTimeTicks, descending), + "processing_time" => source.OrderBy(message => message.ProcessingTimeTicks, descending), + "processed_at" => source.OrderBy(message => message.ProcessedAt, descending), + "status" => source.OrderBy(message => message.Status, descending), + _ => source.OrderBy(message => message.TimeSent, descending) + }; + } + + public static IQueryable ToMessageRows(this IQueryable source) => + source.Select(message => new MessageRow + { + CreatedOn = message.CreatedOn, + Id = message.Id, + UniqueMessageId = message.UniqueMessageId, + MessageId = message.MessageId, + MessageType = message.MessageType, + TimeSent = message.TimeSent, + ProcessedAt = message.ProcessedAt, + ConversationId = message.ConversationId, + IsSystemMessage = message.IsSystemMessage, + Status = message.Status, + SendingEndpointName = message.SendingEndpointName, + SendingEndpointHostId = message.SendingEndpointHostId, + SendingEndpointHost = message.SendingEndpointHost, + ReceivingEndpointName = message.ReceivingEndpointName, + ReceivingEndpointHostId = message.ReceivingEndpointHostId, + ReceivingEndpointHost = message.ReceivingEndpointHost, + CriticalTimeTicks = message.CriticalTimeTicks, + ProcessingTimeTicks = message.ProcessingTimeTicks, + DeliveryTimeTicks = message.DeliveryTimeTicks, + HeadersJson = message.HeadersJson, + BodyState = message.BodyState, + BodySize = message.BodySize + }); + + public static MessagesView ToMessagesView(this MessageRow row) + { + var headers = MessageHeaders.Read(row.HeadersJson); + + var sagas = new Dictionary(); + InvokedSagasParser.Parse(headers, sagas); + + return new MessagesView + { + Id = row.UniqueMessageId.ToString(), + MessageId = row.MessageId, + MessageType = row.MessageType, + SendingEndpoint = Endpoint(row.SendingEndpointName, row.SendingEndpointHostId, row.SendingEndpointHost), + ReceivingEndpoint = Endpoint(row.ReceivingEndpointName, row.ReceivingEndpointHostId, row.ReceivingEndpointHost), + TimeSent = row.TimeSent, + ProcessedAt = row.ProcessedAt, + CriticalTime = TimeSpan.FromTicks(row.CriticalTimeTicks ?? 0), + ProcessingTime = TimeSpan.FromTicks(row.ProcessingTimeTicks ?? 0), + DeliveryTime = TimeSpan.FromTicks(row.DeliveryTimeTicks ?? 0), + IsSystemMessage = row.IsSystemMessage, + ConversationId = row.ConversationId, + Headers = [.. headers], + Status = row.Status, + MessageIntent = ReadMessageIntent(headers), + BodyUrl = row.BodyState == BodyState.None ? null : $"/messages/{AuditBodyId.Format(row.CreatedOn, row.UniqueMessageId)}/body", + BodySize = row.BodySize, + InvokedSagas = sagas.TryGetValue("InvokedSagas", out var invoked) ? invoked as List : null, + OriginatesFromSaga = sagas.TryGetValue("OriginatesFromSaga", out var originates) ? originates as SagaInfo : null + }; + } + + static EndpointDetails? Endpoint(string? name, Guid? hostId, string? host) => + name is null && hostId is null && host is null + ? null + : new EndpointDetails { Name = name, HostId = hostId ?? Guid.Empty, Host = host }; + + static MessageIntent ReadMessageIntent(Dictionary headers) + { + var intent = default(MessageIntent); + if (headers.TryGetValue(Headers.MessageIntent, out var value)) + { + Enum.TryParse(value, true, out intent); + } + + return intent; + } + + static IOrderedQueryable OrderBy(this IQueryable source, Expression> keySelector, bool descending) => + descending + ? source.OrderByDescending(keySelector).ThenByDescending(message => message.CreatedOn).ThenByDescending(message => message.Id) + : source.OrderBy(keySelector).ThenBy(message => message.CreatedOn).ThenBy(message => message.Id); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/MessageRow.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/MessageRow.cs new file mode 100644 index 0000000000..5533745ee6 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/MessageRow.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using ServiceControl.Audit.Monitoring; +using ServiceControl.Audit.Persistence.EFCore.Entities; + +sealed class MessageRow +{ + public DateTime CreatedOn { get; init; } + public long Id { get; init; } + public Guid UniqueMessageId { get; init; } + public string? MessageId { get; init; } + public string? MessageType { get; init; } + public DateTime? TimeSent { get; init; } + public DateTime ProcessedAt { get; init; } + public string? ConversationId { get; init; } + public bool IsSystemMessage { get; init; } + public MessageStatus Status { get; init; } + public string? SendingEndpointName { get; init; } + public Guid? SendingEndpointHostId { get; init; } + public string? SendingEndpointHost { get; init; } + public string? ReceivingEndpointName { get; init; } + public Guid? ReceivingEndpointHostId { get; init; } + public string? ReceivingEndpointHost { get; init; } + public long? CriticalTimeTicks { get; init; } + public long? ProcessingTimeTicks { get; init; } + public long? DeliveryTimeTicks { get; init; } + public string HeadersJson { get; init; } = string.Empty; + public BodyState BodyState { get; init; } + public int BodySize { get; init; } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UnitOfWork/AuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UnitOfWork/AuditIngestionUnitOfWork.cs new file mode 100644 index 0000000000..ac5b2b928e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UnitOfWork/AuditIngestionUnitOfWork.cs @@ -0,0 +1,117 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation.UnitOfWork; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NServiceBus; +using ServiceControl.Audit.Auditing; +using ServiceControl.Audit.Monitoring; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; +using ServiceControl.Audit.Persistence.EFCore.Entities; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; +using ServiceControl.Audit.Persistence.Infrastructure; +using ServiceControl.Audit.Persistence.UnitOfWork; +using ServiceControl.SagaAudit; + +sealed class AuditIngestionUnitOfWork(IServiceScopeFactory scopeFactory, EFPersisterSettings settings, DateTime createdOn) : IAuditIngestionUnitOfWork +{ + readonly List messages = []; + readonly List snapshots = []; + + public Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body = default, CancellationToken cancellationToken = default) + { + var headers = processedMessage.Headers; + var metadata = processedMessage.MessageMetadata; + var uniqueMessageId = ToGuid(processedMessage.UniqueMessageId); + var (bodyState, bodyText) = MessageBodyClassifier.Classify(headers, body, settings.MaxBodySizeToStore); + var bodyId = AuditBodyId.Format(createdOn, uniqueMessageId); + + processedMessage.Id ??= bodyId; + metadata["ContentLength"] = body.Length; + if (!body.IsEmpty) + { + metadata["BodyUrl"] = $"/messages/{bodyId}/body"; + } + + var sendingEndpoint = GetMetadata(metadata, "SendingEndpoint"); + var receivingEndpoint = GetMetadata(metadata, "ReceivingEndpoint"); + + messages.Add(new AuditMessageEntity + { + CreatedOn = createdOn, + UniqueMessageId = uniqueMessageId, + MessageId = GetMetadata(metadata, "MessageId"), + MessageType = GetMetadata(metadata, "MessageType"), + TimeSent = GetMetadata(metadata, "TimeSent"), + ProcessedAt = processedMessage.ProcessedAt, + ConversationId = GetMetadata(metadata, "ConversationId"), + IsSystemMessage = GetMetadata(metadata, "IsSystemMessage"), + Status = GetMetadata(metadata, "IsRetried") ? MessageStatus.ResolvedSuccessfully : MessageStatus.Successful, + SendingEndpointName = sendingEndpoint?.Name, + SendingEndpointHostId = sendingEndpoint?.HostId, + SendingEndpointHost = sendingEndpoint?.Host, + ReceivingEndpointName = receivingEndpoint?.Name, + ReceivingEndpointHostId = receivingEndpoint?.HostId, + ReceivingEndpointHost = receivingEndpoint?.Host, + CriticalTimeTicks = GetMetadata(metadata, "CriticalTime")?.Ticks, + ProcessingTimeTicks = GetMetadata(metadata, "ProcessingTime")?.Ticks, + DeliveryTimeTicks = GetMetadata(metadata, "DeliveryTime")?.Ticks, + HeadersJson = MessageHeaders.Write(headers), + BodyText = bodyText, + BodyState = bodyState, + BodySize = body.Length, + BodyContentType = headers.GetValueOrDefault(Headers.ContentType, "text/plain") + }); + + return Task.CompletedTask; + } + + public Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken = default) + { + snapshots.Add(new SagaSnapshotEntity + { + CreatedOn = createdOn, + SagaId = sagaSnapshot.SagaId, + SagaType = sagaSnapshot.SagaType, + Status = sagaSnapshot.Status, + StartTime = sagaSnapshot.StartTime, + FinishTime = sagaSnapshot.FinishTime, + ProcessedAt = sagaSnapshot.ProcessedAt, + Endpoint = sagaSnapshot.Endpoint, + StateAfterChange = sagaSnapshot.StateAfterChange, + InitiatingMessageJson = SagaSnapshotJson.Write(sagaSnapshot.InitiatingMessage), + OutgoingMessagesJson = SagaSnapshotJson.Write(sagaSnapshot.OutgoingMessages) + }); + + return Task.CompletedTask; + } + + public async Task Complete(CancellationToken cancellationToken = default) + { + if (messages.Count == 0 && snapshots.Count == 0) + { + return; + } + + await using var scope = scopeFactory.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await dbContext.Database.CreateExecutionStrategy().ExecuteAsync(async token => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(token); + await BatchInsert.Rows(dbContext, messages, token); + await BatchInsert.Rows(dbContext, snapshots, token); + await transaction.CommitAsync(token); + }, cancellationToken); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + static Guid ToGuid(string? uniqueMessageId) => + uniqueMessageId is null ? Guid.NewGuid() + : Guid.TryParse(uniqueMessageId, out var parsed) ? parsed + : DeterministicGuid.MakeId(uniqueMessageId); + + static T? GetMetadata(Dictionary metadata, string key) => + metadata.TryGetValue(key, out var value) && value is T typed ? typed : default; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UnitOfWork/AuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UnitOfWork/AuditIngestionUnitOfWorkFactory.cs new file mode 100644 index 0000000000..4bae7be84d --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UnitOfWork/AuditIngestionUnitOfWorkFactory.cs @@ -0,0 +1,18 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation.UnitOfWork; + +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Audit.Persistence.EFCore.Abstractions; +using ServiceControl.Audit.Persistence.EFCore.Infrastructure; +using ServiceControl.Audit.Persistence.UnitOfWork; + +sealed class AuditIngestionUnitOfWorkFactory(IServiceScopeFactory scopeFactory, EFPersisterSettings settings, TimeProvider timeProvider) : IAuditIngestionUnitOfWorkFactory +{ + // Fixed for the whole batch, so the body ids it records match the rows it writes. + public ValueTask StartNew(int batchSize, CancellationToken cancellationToken = default) => + ValueTask.FromResult( + new AuditIngestionUnitOfWork(scopeFactory, settings, AuditHours.Truncate(timeProvider.GetUtcNow().UtcDateTime))); + + public bool CanIngestMore() => true; + + public bool SupportsConcurrentBatches => true; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs new file mode 100644 index 0000000000..6a20dab471 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs @@ -0,0 +1,92 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation.UnitOfWork; + +using System.Text; +using NServiceBus; +using ServiceControl.Audit.Persistence.EFCore.Entities; + +static class MessageBodyClassifier +{ + public static (BodyState State, string? Text) Classify(IReadOnlyDictionary headers, ReadOnlyMemory body, int maxBodySizeToStore) + { + if (body.IsEmpty) + { + return (BodyState.None, null); + } + + var tooLarge = body.Length > maxBodySizeToStore; + var notStored = tooLarge ? BodyState.TooLarge : BodyState.NotText; + + if (!MightBeText(headers)) + { + return (notStored, null); + } + + var span = body.Span; + var slice = tooLarge ? span[..Utf8SafeLength(span[..maxBodySizeToStore])] : span; + + string text; + try + { + text = strictUtf8.GetString(slice); + } + catch (DecoderFallbackException) + { + return (notStored, null); + } + + // PostgreSQL rejects NUL in a text column. + if (text.Contains('\0')) + { + return (notStored, null); + } + + return (tooLarge ? BodyState.TooLarge : BodyState.Stored, text); + } + + static bool MightBeText(IReadOnlyDictionary headers) + { + if (headers.ContainsKey("Content-Encoding")) + { + return false; + } + + if (!headers.TryGetValue(Headers.ContentType, out var contentType)) + { + return true; + } + + var isText = contentType.StartsWith("text/") + || contentType.Contains("xml") + || contentType.Contains("json"); + + return isText && !contentType.Contains("binary"); + } + + static int Utf8SafeLength(ReadOnlySpan slice) + { + var leadIndex = slice.Length - 1; + while (leadIndex >= 0 && (slice[leadIndex] & 0xC0) == 0x80) + { + leadIndex--; + } + + if (leadIndex < 0) + { + return 0; + } + + var lead = slice[leadIndex]; + var sequenceLength = lead switch + { + < 0x80 => 1, + >= 0xF0 => 4, + >= 0xE0 => 3, + >= 0xC0 => 2, + _ => 1 + }; + + return slice.Length - leadIndex >= sequenceLength ? slice.Length : leadIndex; + } + + static readonly UTF8Encoding strictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UpsertExtensions.cs b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UpsertExtensions.cs new file mode 100644 index 0000000000..1bccffe984 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Implementation/UpsertExtensions.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Implementation; + +using Microsoft.EntityFrameworkCore; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; + +static class UpsertExtensions +{ + public static async Task UpsertAsync(this AuditDbContext dbContext, object key, Func create, Action update, CancellationToken cancellationToken = default) + where TEntity : class + { + var entity = await dbContext.FindAsync([key], cancellationToken); + if (entity is null) + { + entity = create(); + try + { + dbContext.Add(entity); + await dbContext.SaveChangesAsync(cancellationToken); + return; + } + catch (DbUpdateException e) when (dbContext.IsDuplicateKeyException(e)) + { + await dbContext.Entry(entity).ReloadAsync(cancellationToken); + } + } + + update(entity); + await dbContext.SaveChangesAsync(cancellationToken); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/AuditBodyId.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/AuditBodyId.cs new file mode 100644 index 0000000000..d5483d7a64 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/AuditBodyId.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using System.Globalization; + +static class AuditBodyId +{ + const string HourFormat = "yyyyMMddHH"; + + public static string Format(DateTime createdOn, Guid uniqueMessageId) => + $"{createdOn.ToString(HourFormat, CultureInfo.InvariantCulture)}-{uniqueMessageId}"; + + public static bool TryParse(string bodyId, out DateTime createdOn, out Guid uniqueMessageId) + { + createdOn = default; + uniqueMessageId = default; + + return bodyId.Length > HourFormat.Length + && bodyId[HourFormat.Length] == '-' + && DateTime.TryParseExact(bodyId[..HourFormat.Length], HourFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out createdOn) + && Guid.TryParse(bodyId[(HourFormat.Length + 1)..], out uniqueMessageId); + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/AuditHours.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/AuditHours.cs new file mode 100644 index 0000000000..22ebc72a5a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/AuditHours.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +static class AuditHours +{ + public static readonly TimeSpan Lookahead = TimeSpan.FromHours(48); + + public static DateTime Truncate(DateTime utc) => new(utc.Year, utc.Month, utc.Day, utc.Hour, 0, 0, DateTimeKind.Utc); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/BatchInsert.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/BatchInsert.cs new file mode 100644 index 0000000000..d4ecad0d29 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/BatchInsert.cs @@ -0,0 +1,128 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using System.Collections.Concurrent; +using System.Linq.Expressions; +using System.Reflection; +using System.Text; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage; + +static class BatchInsert +{ + public static async Task Rows(DbContext dbContext, IReadOnlyCollection rows, CancellationToken cancellationToken = default) + where TEntity : class + { + if (rows.Count == 0) + { + return; + } + + var table = InsertTable.For(dbContext); + var transaction = (dbContext.Database.CurrentTransaction + ?? throw new InvalidOperationException("Batch inserts must run inside a transaction")).GetDbTransaction(); + + foreach (var chunk in rows.Chunk(table.RowsPerStatement)) + { + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.Transaction = transaction; + command.CommandTimeout = dbContext.Database.GetCommandTimeout() ?? command.CommandTimeout; + command.CommandText = table.Sql(chunk.Length); + + var index = 0; + foreach (var row in chunk) + { + foreach (var column in table.Columns) + { + command.Parameters.Add(column.TypeMapping.CreateParameter(command, $"@p{index++}", column.Read(row), column.IsNullable)); + } + } + + await command.ExecuteNonQueryAsync(cancellationToken); + } + } + + sealed class InsertTable + { + InsertTable(string qualifiedName, IReadOnlyList columns, string columnList) + { + this.qualifiedName = qualifiedName; + this.columnList = columnList; + Columns = columns; + RowsPerStatement = Math.Max(1, Math.Min(50, MaxParametersPerStatement / columns.Count)); + } + + public IReadOnlyList Columns { get; } + + public int RowsPerStatement { get; } + + public string Sql(int rowCount) + { + var sql = new StringBuilder($"INSERT INTO {qualifiedName} ({columnList}) VALUES "); + + for (var row = 0; row < rowCount; row++) + { + sql.Append(row == 0 ? "(" : ", ("); + + for (var column = 0; column < Columns.Count; column++) + { + if (column > 0) + { + sql.Append(", "); + } + + sql.Append("@p").Append((row * Columns.Count) + column); + } + + sql.Append(')'); + } + + return sql.ToString(); + } + + public static InsertTable For(DbContext dbContext) => + cache.GetOrAdd((dbContext.Model, typeof(TEntity)), static (key, context) => + { + var entityType = key.Model.FindEntityType(key.EntityType) + ?? throw new InvalidOperationException($"{key.EntityType.Name} is not part of the model."); + var tableName = entityType.GetTableName() + ?? throw new InvalidOperationException($"{key.EntityType.Name} is not mapped to a table."); + var schema = entityType.GetSchema(); + var storeObject = StoreObjectIdentifier.Table(tableName, schema); + var sqlGenerationHelper = context.GetService(); + + InsertColumn[] columns = + [ + .. entityType.GetProperties() + .Where(property => property.ValueGenerated != ValueGenerated.OnAdd) + .Select(property => new InsertColumn( + property.GetColumnName(storeObject)!, + property.GetRelationalTypeMapping(), + property.IsNullable, + CompileGetter(property.PropertyInfo!))) + ]; + + return new InsertTable( + sqlGenerationHelper.DelimitIdentifier(tableName, schema), + columns, + string.Join(", ", columns.Select(column => sqlGenerationHelper.DelimitIdentifier(column.Name)))); + }, dbContext); + + static Func CompileGetter(PropertyInfo property) + { + var entity = Expression.Parameter(typeof(object)); + var value = Expression.Property(Expression.Convert(entity, property.DeclaringType!), property); + return Expression.Lambda>(Expression.Convert(value, typeof(object)), entity).Compile(); + } + + readonly string qualifiedName; + readonly string columnList; + + const int MaxParametersPerStatement = 2000; + + static readonly ConcurrentDictionary<(IModel Model, Type EntityType), InsertTable> cache = new(); + } + + sealed record InsertColumn(string Name, RelationalTypeMapping TypeMapping, bool IsNullable, Func Read); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IAuditPartitionManager.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IAuditPartitionManager.cs new file mode 100644 index 0000000000..2b6e7de104 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IAuditPartitionManager.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using ServiceControl.Audit.Persistence.EFCore.DbContexts; + +interface IAuditPartitionManager +{ + Task EnsurePartitions(AuditDbContext dbContext, DateTime from, DateTime toExclusive, CancellationToken cancellationToken = default); + + Task> ListExpired(AuditDbContext dbContext, DateTime keepFrom, CancellationToken cancellationToken = default); + + Task DropExpired(AuditDbContext dbContext, DateTime start, int batchSize, CancellationToken cancellationToken = default); + + Task ProvisionedUntil(AuditDbContext dbContext, CancellationToken cancellationToken = default); +} + +readonly record struct DropResult(int RowsDeleted, bool Completed, bool Deferred = false); diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IFullTextSearchDialect.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IFullTextSearchDialect.cs new file mode 100644 index 0000000000..accb50caba --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IFullTextSearchDialect.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using ServiceControl.Audit.Persistence.EFCore.Entities; + +interface IFullTextSearchDialect +{ + IQueryable Search(IQueryable source, string searchTerms); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IRetentionLock.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IRetentionLock.cs new file mode 100644 index 0000000000..4f6518edc3 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/IRetentionLock.cs @@ -0,0 +1,11 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +interface IRetentionLock +{ + Task TryAcquire(CancellationToken cancellationToken = default); +} + +static class RetentionLock +{ + public static string ResourceName(string? schema) => schema is null ? "audit_retention_sweep" : $"audit_retention_sweep:{schema}"; +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs new file mode 100644 index 0000000000..4ead155690 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; + +static class MessageHeaders +{ + public static string Write(Dictionary headers) => + JsonSerializer.Serialize(headers, context.DictionaryStringString); + + public static Dictionary Read(string headersJson) => + JsonSerializer.Deserialize(headersJson, context.DictionaryStringString) ?? []; + + // The relaxed encoder is only unsafe for JSON embedded in HTML, and this JSON never reaches HTML. Full text search needs apostrophes, plus signs and non-ASCII letters written as they are, or it cannot find the words next to them. + static readonly HeadersJsonContext context = new(new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }); +} + +[JsonSerializable(typeof(Dictionary))] +partial class HeadersJsonContext : JsonSerializerContext; diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MigrationSchemaStamper.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MigrationSchemaStamper.cs new file mode 100644 index 0000000000..ae7489baea --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MigrationSchemaStamper.cs @@ -0,0 +1,73 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +public static class MigrationSchemaStamper +{ + public static MigrationOperation Stamp(MigrationOperation operation, string schema) + { + switch (operation) + { + case CreateTableOperation createTable: + createTable.Schema ??= schema; + foreach (var column in createTable.Columns) + { + column.Schema ??= schema; + } + createTable.PrimaryKey?.Schema ??= schema; + foreach (var foreignKey in createTable.ForeignKeys) + { + foreignKey.Schema ??= schema; + foreignKey.PrincipalSchema ??= schema; + } + foreach (var uniqueConstraint in createTable.UniqueConstraints) + { + uniqueConstraint.Schema ??= schema; + } + foreach (var checkConstraint in createTable.CheckConstraints) + { + checkConstraint.Schema ??= schema; + } + break; + + case DropTableOperation dropTable: + dropTable.Schema ??= schema; + break; + + case CreateIndexOperation createIndex: + createIndex.Schema ??= schema; + break; + + case DropIndexOperation dropIndex: + dropIndex.Schema ??= schema; + break; + + case AddColumnOperation addColumn: + addColumn.Schema ??= schema; + break; + + case AlterColumnOperation alterColumn: + alterColumn.Schema ??= schema; + alterColumn.OldColumn.Schema ??= schema; + break; + + case DropColumnOperation dropColumn: + dropColumn.Schema ??= schema; + break; + + case InsertDataOperation insertData: + insertData.Schema ??= schema; + break; + + case EnsureSchemaOperation: + case DropSchemaOperation: + break; + + default: + throw new InvalidOperationException( + $"Migration operation {operation.GetType().Name} is not handled by {nameof(MigrationSchemaStamper)}, so it would run against the connection's default schema instead of '{schema}'. Add a case for it."); + } + + return operation; + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SagaSnapshotJson.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SagaSnapshotJson.cs new file mode 100644 index 0000000000..82624ebca2 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SagaSnapshotJson.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using System.Text.Json; +using System.Text.Json.Serialization; +using ServiceControl.SagaAudit; + +static class SagaSnapshotJson +{ + public static string? Write(InitiatingMessage? initiatingMessage) => + initiatingMessage is null ? null : JsonSerializer.Serialize(initiatingMessage, SagaSnapshotJsonContext.Default.InitiatingMessage); + + public static string Write(List? outgoingMessages) => + JsonSerializer.Serialize(outgoingMessages ?? [], SagaSnapshotJsonContext.Default.ListResultingMessage); + + public static InitiatingMessage? ReadInitiatingMessage(string? json) => + json is null ? null : JsonSerializer.Deserialize(json, SagaSnapshotJsonContext.Default.InitiatingMessage); + + public static List ReadOutgoingMessages(string? json) => + json is null ? [] : JsonSerializer.Deserialize(json, SagaSnapshotJsonContext.Default.ListResultingMessage) ?? []; +} + +[JsonSerializable(typeof(InitiatingMessage))] +[JsonSerializable(typeof(List))] +partial class SagaSnapshotJsonContext : JsonSerializerContext; diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SchemaModelCacheKeyFactory.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SchemaModelCacheKeyFactory.cs new file mode 100644 index 0000000000..d13394e629 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SchemaModelCacheKeyFactory.cs @@ -0,0 +1,11 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using ServiceControl.Audit.Persistence.EFCore.DbContexts; + +public sealed class SchemaModelCacheKeyFactory : IModelCacheKeyFactory +{ + public object Create(DbContext context, bool designTime) => + (context.GetType(), (context as AuditDbContext)?.Schema, designTime); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SchemaOptionsExtension.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SchemaOptionsExtension.cs new file mode 100644 index 0000000000..3a0f29dd8b --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SchemaOptionsExtension.cs @@ -0,0 +1,36 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.Extensions.DependencyInjection; + +public sealed class SchemaOptionsExtension(string schema) : IDbContextOptionsExtension +{ + public string Schema { get; } = schema; + + public DbContextOptionsExtensionInfo Info => field ??= new ExtensionInfo(this); + + public void ApplyServices(IServiceCollection services) + { + } + + public void Validate(IDbContextOptions options) + { + } + + sealed class ExtensionInfo(SchemaOptionsExtension extension) : DbContextOptionsExtensionInfo(extension) + { + public override bool IsDatabaseProvider => false; + + public override string LogFragment => $"using schema {Extension.Schema} "; + + public override void PopulateDebugInfo(IDictionary debugInfo) => + debugInfo["ServiceControl:" + nameof(Schema)] = Extension.Schema; + + // Not keyed on the schema, which would build an internal service provider per schema. + public override int GetServiceProviderHashCode() => 0; + + public override bool ShouldUseSameServiceProvider(DbContextOptionsExtensionInfo other) => other is ExtensionInfo; + + new SchemaOptionsExtension Extension => (SchemaOptionsExtension)base.Extension; + } +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SchemaQualifiedTableName.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SchemaQualifiedTableName.cs new file mode 100644 index 0000000000..e04604fe2c --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/SchemaQualifiedTableName.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Audit.Persistence.EFCore.Infrastructure; + +using System.Collections.Concurrent; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage; + +public static class SchemaQualifiedTableName +{ + public static string For(DbContext dbContext) => + cache.GetOrAdd((dbContext.Model, typeof(TEntity)), static (key, context) => + { + var entityType = key.Model.FindEntityType(key.EntityType) + ?? throw new InvalidOperationException($"{key.EntityType.Name} is not part of the model."); + + var tableName = entityType.GetTableName() + ?? throw new InvalidOperationException($"{key.EntityType.Name} is not mapped to a table."); + + return context.GetService().DelimitIdentifier(tableName, entityType.GetSchema()); + }, dbContext); + + static readonly ConcurrentDictionary<(IModel Model, Type EntityType), string> cache = new(); +} diff --git a/src/ServiceControl.Audit.Persistence.EFCore/ServiceControl.Audit.Persistence.EFCore.csproj b/src/ServiceControl.Audit.Persistence.EFCore/ServiceControl.Audit.Persistence.EFCore.csproj new file mode 100644 index 0000000000..6b51feb1d8 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.EFCore/ServiceControl.Audit.Persistence.EFCore.csproj @@ -0,0 +1,37 @@ + + + + net10.0 + enable + enable + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/.editorconfig b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/.editorconfig new file mode 100644 index 0000000000..ca5ad8bd2e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/.editorconfig @@ -0,0 +1,5 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Audit.Persistence.Tests/ApprovalFiles/AuditCountingTests.ShouldCountAuditedMessages.approved.txt b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/ApprovalFiles/AuditCountingTests.ShouldCountAuditedMessages.approved.txt similarity index 100% rename from src/ServiceControl.Audit.Persistence.Tests/ApprovalFiles/AuditCountingTests.ShouldCountAuditedMessages.approved.txt rename to src/ServiceControl.Audit.Persistence.Tests.PostgreSql/ApprovalFiles/AuditCountingTests.ShouldCountAuditedMessages.approved.txt diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt new file mode 100644 index 0000000000..648f28180a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt @@ -0,0 +1,2 @@ +ServiceControl.Audit Health: Audit partition provisioning +ServiceControl.Audit Health: Audit retention \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/AuditCountsQueryTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/AuditCountsQueryTests.cs new file mode 100644 index 0000000000..9a7b13110b --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/AuditCountsQueryTests.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System.Linq; + using Microsoft.EntityFrameworkCore; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + + [TestFixture] + class AuditCountsQueryTests + { + [Test] + public void Counts_are_grouped_by_the_utc_day() + { + using var dbContext = new PostgreSqlAuditDbContextFactory().CreateDbContext([]); + + var sql = dbContext.AuditMessages.GroupBy(message => message.ProcessedAt.Date).Select(day => day.Key).ToQueryString(); + + Assert.That(sql, Does.Contain("date_trunc('day', a.processed_at, 'UTC')")); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs new file mode 100644 index 0000000000..26ffbc3253 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs @@ -0,0 +1,34 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System.Text.RegularExpressions; + using Microsoft.EntityFrameworkCore; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + + class FullTextSearchIndexTests + { + [Test] + public void Search_uses_the_indexed_expression() + { + var sql = SearchQuery("forty-two"); + var alias = Regex.Match(sql, @"FROM audit_messages AS (\w+)").Groups[1].Value; + + Assert.That(sql.Replace($"{alias}.", string.Empty), Does.Contain(FullTextSearchSql.IndexedExpression)); + } + + [Test] + public void Terms_are_ored() + { + Assert.That(SearchQuery("forty two"), Does.Contain("='forty OR two'")); + } + + static string SearchQuery(string searchTerms) + { + using var dbContext = new PostgreSqlAuditDbContextFactory().CreateDbContext([]); + + return new PostgreSqlFullTextSearchDialect() + .Search(dbContext.AuditMessages, searchTerms) + .ToQueryString(); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/LargeBodyIndexingTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/LargeBodyIndexingTests.cs new file mode 100644 index 0000000000..27265bcf53 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/LargeBodyIndexingTests.cs @@ -0,0 +1,29 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using NUnit.Framework; + using ServiceControl.Audit.Infrastructure; + + class LargeBodyIndexingTests : EFPersistenceTestFixture + { + public override Task Setup() + { + SetSettings = settings => settings.MaxBodySizeToStore = 2 * 1024 * 1024; + return base.Setup(); + } + + [Test] + public async Task Stores_and_searches_a_large_body_of_distinct_tokens() + { + var tokens = Enumerable.Range(0, 31_000).Select(_ => Guid.NewGuid().ToString("N")).ToArray(); + await Ingest(MakeMessage(), Encoding.UTF8.GetBytes(string.Join(" ", tokens))); + + var found = await MessagesViewStore.QueryMessages(tokens[0], new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(found.Results, Has.Count.EqualTo(1)); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/MigrationSqlIsSchemaAwareTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/MigrationSqlIsSchemaAwareTests.cs new file mode 100644 index 0000000000..fa027b04b5 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/MigrationSqlIsSchemaAwareTests.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System.Linq; + using Microsoft.EntityFrameworkCore; + using Microsoft.EntityFrameworkCore.Infrastructure; + using Microsoft.EntityFrameworkCore.Migrations; + using Microsoft.EntityFrameworkCore.Migrations.Operations; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + + [TestFixture] + class MigrationSqlIsSchemaAwareTests + { + [Test] + public void Every_hand_written_migration_statement_is_schema_aware() + { + using var dbContext = new PostgreSqlAuditDbContextFactory().CreateDbContext([]); + var migrations = dbContext.GetService(); + + var unrecognised = migrations.Migrations + .Select(migration => migrations.CreateMigration(migration.Value, dbContext.Database.ProviderName)) + .SelectMany(migration => migration.UpOperations.Concat(migration.DownOperations)) + .OfType() + .Select(operation => operation.Sql) + .Where(sql => !FullTextSearchSql.IsHandled(sql)) + .ToArray(); + + Assert.That(unrecognised, Is.Empty, + $"A migration runs SQL that {nameof(FullTextSearchSql)}.{nameof(FullTextSearchSql.Rewrite)} does not recognise. It would run against the default schema, whatever Database/Schema is set to. Add it to Rewrite, and to IsHandled if it needs no qualifying."); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PartitionNameTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PartitionNameTests.cs new file mode 100644 index 0000000000..50d1bce13e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PartitionNameTests.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + + [TestFixture] + class PartitionNameTests + { + [Test] + [SetCulture("th-TH")] + public void Partition_names_keep_the_gregorian_year_whatever_the_culture() => + Assert.That(PostgreSqlAuditPartitionManager.PartitionName("audit_messages", new DateTime(2026, 9, 28, 0, 0, 0, DateTimeKind.Utc)), Is.EqualTo("audit_messages_20260928")); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PartitioningTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PartitioningTests.cs new file mode 100644 index 0000000000..1679782844 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PartitioningTests.cs @@ -0,0 +1,78 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.EntityFrameworkCore; + using Microsoft.Extensions.DependencyInjection; + using NServiceBus.CustomChecks; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.DbContexts; + using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + + class PartitioningTests : EFPersistenceTestFixture + { + [TestCase("audit_messages")] + [TestCase("saga_snapshots")] + public async Task Table_is_range_partitioned_on_created_on(string table) + { + var (strategy, partitionKey) = await WithDbContext(async (dbContext, token) => ( + await QueryScalar(dbContext, $"SELECT partstrat::text FROM pg_partitioned_table WHERE partrelid = '{dbContext.Schema}.{table}'::regclass", token), + await QueryScalar(dbContext, $""" + SELECT a.attname + FROM pg_partitioned_table p + JOIN pg_attribute a ON a.attrelid = p.partrelid AND a.attnum = p.partattrs[0] + WHERE p.partrelid = '{dbContext.Schema}.{table}'::regclass + """, token))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(strategy, Is.EqualTo("r"), $"{table} is not range partitioned"); + Assert.That(partitionKey, Is.EqualTo("created_on")); + } + } + + [Test] + public async Task Setup_provisions_whole_days_through_the_lookahead() + { + var end = await ProvisionedUntil(); + var now = configuration.TimeProvider.GetUtcNow().UtcDateTime; + + using (Assert.EnterMultipleScope()) + { + Assert.That(end, Is.EqualTo(end!.Value.Date), "a partition holds a whole day"); + Assert.That(end, Is.GreaterThanOrEqualTo(now + AuditHours.Lookahead)); + Assert.That(end, Is.LessThan(now + AuditHours.Lookahead + TimeSpan.FromDays(1))); + } + } + + [Test] + public async Task The_provisioning_check_passes_while_the_window_is_ahead_and_fails_once_it_runs_short() + { + var check = ActivatorUtilities.CreateInstance(ServiceProvider); + + var ahead = await check.PerformCheck(); + var end = await ProvisionedUntil(); + configuration.TimeProvider.Advance(end!.Value - configuration.TimeProvider.GetUtcNow().UtcDateTime - AuditPartitionCustomCheck.Threshold + TimeSpan.FromHours(1)); + var runningShort = await check.PerformCheck(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(ahead.HasFailed, Is.False); + Assert.That(runningShort.HasFailed, Is.True); + } + } + + Task ProvisionedUntil() => + WithDbContext((dbContext, token) => ServiceProvider.GetRequiredService().ProvisionedUntil(dbContext, token)); + + static async Task QueryScalar(AuditDbContext dbContext, string sql, CancellationToken cancellationToken) + { + await dbContext.Database.OpenConnectionAsync(cancellationToken); + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.CommandText = sql; + return (await command.ExecuteScalarAsync(cancellationToken))?.ToString(); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PendingModelChangesTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PendingModelChangesTests.cs new file mode 100644 index 0000000000..ad3d3b68e3 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PendingModelChangesTests.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using Microsoft.EntityFrameworkCore; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + + [TestFixture] + class PendingModelChangesTests + { + [Test] + public void The_model_matches_the_migrations() + { + using var dbContext = new PostgreSqlAuditDbContextFactory().CreateDbContext([]); + + Assert.That(dbContext.Database.HasPendingModelChanges(), Is.False, + "The model has changed and no migration matches it. Run 'dotnet ef migrations add ' in ServiceControl.Audit.Persistence.EFCore.PostgreSql."); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PersistenceTestsConfiguration.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PersistenceTestsConfiguration.cs new file mode 100644 index 0000000000..b57b73346f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/PersistenceTestsConfiguration.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System.Threading.Tasks; + using ServiceControl.Audit.Persistence.EFCore.Abstractions; + using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + using ServiceControl.Persistence.Tests; + + class PersistenceTestsConfiguration : EFPersistenceTestsConfiguration + { + public override string Name => "PostgreSQL"; + + public override Task GetConnectionString() => PostgreSqlSharedContainer.GetConnectionStringAsync(); + + protected override Task CreateSchema(string connectionString, string schema) => TestSchema.Create(connectionString, schema); + + protected override Task DropSchema(string connectionString, string schema) => TestSchema.Drop(connectionString, schema); + + public override EFPersistenceConfigurationBase CreateConfiguration() => new PostgreSqlPersistenceConfiguration(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/RetentionPeriodTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/RetentionPeriodTests.cs new file mode 100644 index 0000000000..5bc10040be --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/RetentionPeriodTests.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.Abstractions; + using ServiceControl.Audit.Persistence.EFCore.PostgreSql; + + [TestFixture] + class RetentionPeriodTests + { + [TestCase("23:59:00")] + [TestCase("90.00:01:00")] + public void Refuses_a_retention_period_outside_one_to_ninety_days(string retentionPeriod) + { + var exception = Assert.Throws(() => new PostgreSqlPersistenceConfiguration().Create(SettingsWith(TimeSpan.Parse(retentionPeriod)))); + + Assert.That(exception.Message, Does.Contain("1 to 90 days")); + } + + [TestCase("1.00:00:00")] + [TestCase("90.00:00:00")] + public void Accepts_a_retention_period_of_one_to_ninety_days(string retentionPeriod) => + Assert.That(new PostgreSqlPersistenceConfiguration().Create(SettingsWith(TimeSpan.Parse(retentionPeriod))), Is.Not.Null); + + static PersistenceSettings SettingsWith(TimeSpan retentionPeriod) + { + var settings = new PersistenceSettings(retentionPeriod, true, 100000); + settings.PersisterSpecificSettings[EFPersistenceConfigurationBase.ConnectionStringKey] = "Host=localhost"; + return settings; + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/ServiceControl.Audit.Persistence.Tests.PostgreSql.csproj b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/ServiceControl.Audit.Persistence.Tests.PostgreSql.csproj new file mode 100644 index 0000000000..1ea1c0eec9 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/ServiceControl.Audit.Persistence.Tests.PostgreSql.csproj @@ -0,0 +1,43 @@ + + + + net10.0 + PostgreSql + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ServiceControl.Audit.Persistence.Tests.RavenDB.csproj b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ServiceControl.Audit.Persistence.Tests.RavenDB.csproj index 60be6df82f..d6295c26dc 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ServiceControl.Audit.Persistence.Tests.RavenDB.csproj +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ServiceControl.Audit.Persistence.Tests.RavenDB.csproj @@ -1,4 +1,4 @@ - + net10.0 @@ -27,6 +27,8 @@ approval test instead of a very expensive acceptance test. --> + + diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/.editorconfig b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/.editorconfig new file mode 100644 index 0000000000..ca5ad8bd2e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/.editorconfig @@ -0,0 +1,5 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/ApprovalFiles/AuditCountingTests.ShouldCountAuditedMessages.approved.txt b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/ApprovalFiles/AuditCountingTests.ShouldCountAuditedMessages.approved.txt new file mode 100644 index 0000000000..8438bb464c --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/ApprovalFiles/AuditCountingTests.ShouldCountAuditedMessages.approved.txt @@ -0,0 +1,22 @@ +{ + "EndpointA": [ + { + "UtcDate": "(YESTERDAY)T00:00:00Z", + "Count": 2 + }, + { + "UtcDate": "(TODAY)T00:00:00Z", + "Count": 1 + } + ], + "EndpointB": [ + { + "UtcDate": "(WEEKBEFORE)T00:00:00Z", + "Count": 4 + }, + { + "UtcDate": "(TODAY)T00:00:00Z", + "Count": 3 + } + ] +} \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt new file mode 100644 index 0000000000..cd5cf1e630 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt @@ -0,0 +1 @@ +ServiceControl.Audit Health: Audit retention \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/MigrationSqlIsSchemaAwareTests.cs b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/MigrationSqlIsSchemaAwareTests.cs new file mode 100644 index 0000000000..dd327fb97f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/MigrationSqlIsSchemaAwareTests.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System.Linq; + using Microsoft.EntityFrameworkCore; + using Microsoft.EntityFrameworkCore.Infrastructure; + using Microsoft.EntityFrameworkCore.Migrations; + using Microsoft.EntityFrameworkCore.Migrations.Operations; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.SqlServer; + + [TestFixture] + class MigrationSqlIsSchemaAwareTests + { + [Test] + public void Every_hand_written_migration_statement_is_schema_aware() + { + using var dbContext = new SqlServerAuditDbContextFactory().CreateDbContext([]); + var migrations = dbContext.GetService(); + + var unrecognised = migrations.Migrations + .Select(migration => migrations.CreateMigration(migration.Value, dbContext.Database.ProviderName)) + .SelectMany(migration => migration.UpOperations.Concat(migration.DownOperations)) + .OfType() + .Select(operation => operation.Sql) + .Where(sql => !FullTextSearchSql.IsHandled(sql)) + .ToArray(); + + Assert.That(unrecognised, Is.Empty, + $"A migration runs SQL that {nameof(FullTextSearchSql)}.{nameof(FullTextSearchSql.Rewrite)} does not recognise. It would run against the default schema, whatever Database/Schema is set to. Add it to Rewrite, and to IsHandled if it needs no qualifying."); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PendingModelChangesTests.cs b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PendingModelChangesTests.cs new file mode 100644 index 0000000000..419233f4b8 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PendingModelChangesTests.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using Microsoft.EntityFrameworkCore; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.SqlServer; + + [TestFixture] + class PendingModelChangesTests + { + [Test] + public void The_model_matches_the_migrations() + { + using var dbContext = new SqlServerAuditDbContextFactory().CreateDbContext([]); + + Assert.That(dbContext.Database.HasPendingModelChanges(), Is.False, + "The model has changed and no migration matches it. Run 'dotnet ef migrations add ' in ServiceControl.Audit.Persistence.EFCore.SqlServer."); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs new file mode 100644 index 0000000000..85143e7ba9 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/PersistenceTestsConfiguration.cs @@ -0,0 +1,89 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Data.SqlClient; + using ServiceControl.Audit.Auditing; + using ServiceControl.Audit.Auditing.MessagesView; + using ServiceControl.Audit.Infrastructure; + using ServiceControl.Audit.Persistence.EFCore.Abstractions; + using ServiceControl.Audit.Persistence.EFCore.SqlServer; + using ServiceControl.Persistence.Tests; + + class PersistenceTestsConfiguration : EFPersistenceTestsConfiguration + { + public override string Name => "SQLServer"; + + public override Task GetConnectionString() => SqlServerSharedContainer.GetConnectionStringAsync(); + + protected override Task CreateSchema(string connectionString, string schema) => TestSchema.Create(connectionString, schema); + + protected override Task DropSchema(string connectionString, string schema) => TestSchema.Drop(connectionString, schema); + + public override EFPersistenceConfigurationBase CreateConfiguration() => new SqlServerPersistenceConfiguration(); + + protected override IAuditMessagesViewDataStore DecorateMessagesViewStore(IAuditMessagesViewDataStore store) => + new SearchesWaitForFullTextIndex(store, WaitForFullTextIndex); + + async Task WaitForFullTextIndex(CancellationToken cancellationToken) + { + await using var connection = new SqlConnection(ConnectionString); + await connection.OpenAsync(cancellationToken); + + var deadline = DateTime.UtcNow.AddSeconds(30); + while (true) + { + await using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT CAST(OBJECTPROPERTYEX(OBJECT_ID(@table), 'TableFullTextPendingChanges') AS int) + + CAST(OBJECTPROPERTYEX(OBJECT_ID(@table), 'TableFullTextPopulateStatus') AS int) + """; + command.Parameters.AddWithValue("@table", $"[{Schema}].[AuditMessages]"); + + if (Convert.ToInt32(await command.ExecuteScalarAsync(cancellationToken)) == 0) + { + return; + } + + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException("The full text index did not catch up with the audit messages table."); + } + + await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken); + } + } + + sealed class SearchesWaitForFullTextIndex(IAuditMessagesViewDataStore store, Func waitForFullTextIndex) : IAuditMessagesViewDataStore + { + public async Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) + { + await waitForFullTextIndex(cancellationToken); + return await store.QueryMessages(searchParam, pagingInfo, sortInfo, timeSentRange, cancellationToken); + } + + public async Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) + { + await waitForFullTextIndex(cancellationToken); + return await store.QueryMessagesByReceivingEndpointAndKeyword(endpoint, keyword, pagingInfo, sortInfo, timeSentRange, cancellationToken); + } + + public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => + store.GetMessages(includeSystemMessages, pagingInfo, sortInfo, timeSentRange, cancellationToken); + + public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default) => + store.QueryMessagesByReceivingEndpoint(includeSystemMessages, endpointName, pagingInfo, sortInfo, timeSentRange, cancellationToken); + + public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken = default) => + store.QueryMessagesByConversationId(conversationId, pagingInfo, sortInfo, cancellationToken); + + public Task GetMessageBody(string messageId, CancellationToken cancellationToken = default) => + store.GetMessageBody(messageId, cancellationToken); + + public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken = default) => + store.QueryAuditCounts(endpointName, cancellationToken); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests.SqlServer/ServiceControl.Audit.Persistence.Tests.SqlServer.csproj b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/ServiceControl.Audit.Persistence.Tests.SqlServer.csproj new file mode 100644 index 0000000000..b2a6fe42b5 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.SqlServer/ServiceControl.Audit.Persistence.Tests.SqlServer.csproj @@ -0,0 +1,43 @@ + + + + net10.0 + SqlServer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Audit.Persistence.Tests/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt b/src/ServiceControl.Audit.Persistence.Tests/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/ServiceControl.Audit.Persistence.Tests/AuditTests.BinaryBody.cs b/src/ServiceControl.Audit.Persistence.Tests/AuditTests.BinaryBody.cs new file mode 100644 index 0000000000..b01e0b75da --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/AuditTests.BinaryBody.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Threading.Tasks; + using NUnit.Framework; + + partial class AuditTests + { + [Test] + public async Task Can_roundtrip_message_body() + { + string expectedContentType = "text/plain"; + await using var unitOfWork = await StartAuditUnitOfWork(1); + + var body = new byte[100]; + Random.Shared.NextBytes(body); + var processedMessage = MakeMessage(); + + await unitOfWork.RecordProcessedMessage(processedMessage, body); + + await unitOfWork.Complete(); + + var bodyId = GetBodyId(processedMessage); + + var retrievedMessage = await MessagesViewStore.GetMessageBody(bodyId, TestContext.CurrentContext.CancellationToken); + + Assert.That(retrievedMessage, Is.Not.Null); + using (Assert.EnterMultipleScope()) + { + Assert.That(retrievedMessage.Found, Is.True); + Assert.That(retrievedMessage.HasContent, Is.True); + Assert.That(retrievedMessage.ContentLength, Is.EqualTo(body.Length)); + Assert.That(retrievedMessage.Version.HasValue, Is.True); + Assert.That(retrievedMessage.StreamContent, Is.Not.Null); + Assert.That(retrievedMessage.ContentType, Is.EqualTo(expectedContentType)); + } + + var resultBody = new byte[body.Length]; + var readBytes = await retrievedMessage.StreamContent.ReadAsync(resultBody, 0, body.Length); + using (Assert.EnterMultipleScope()) + { + Assert.That(readBytes, Is.EqualTo(body.Length)); + Assert.That(resultBody, Is.EqualTo(body)); + } + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/AuditTests.Deduplication.cs b/src/ServiceControl.Audit.Persistence.Tests/AuditTests.Deduplication.cs new file mode 100644 index 0000000000..1727484d5f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/AuditTests.Deduplication.cs @@ -0,0 +1,56 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Threading.Tasks; + using NUnit.Framework; + using ServiceControl.Audit.Infrastructure; + + partial class AuditTests + { + [Test] + public async Task Deduplicates_messages_in_same_batch() + { + await using var unitOfWork = await StartAuditUnitOfWork(1); + var messageId = "duplicatedId"; + var processingEndpoint = "endpoint"; + var processingStarted = DateTimeOffset.UtcNow; + + var processedMessage = MakeMessage(messageId: messageId, processingEndpoint: processingEndpoint, processingStarted: processingStarted); + var duplicatedMessage = MakeMessage(messageId: messageId, processingEndpoint: processingEndpoint, processingStarted: processingStarted); + await unitOfWork.RecordProcessedMessage(processedMessage); + await unitOfWork.RecordProcessedMessage(duplicatedMessage); + + await unitOfWork.Complete(); + + await configuration.CompleteDBOperation(); + + var queryResult = await MessagesViewStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); + + Assert.That(queryResult.QueryStats.TotalCount, Is.EqualTo(1)); + } + + [Test] + public async Task Deduplicates_messages_in_different_batches() + { + var messageId = "duplicatedId"; + var processingEndpoint = "endpoint"; + var processingStarted = DateTimeOffset.UtcNow; + + var processedMessage = MakeMessage(messageId: messageId, processingEndpoint: processingEndpoint, processingStarted: processingStarted); + await using var unitOfWork1 = await StartAuditUnitOfWork(1); + await unitOfWork1.RecordProcessedMessage(processedMessage); + await unitOfWork1.Complete(); + + var duplicatedMessage = MakeMessage(messageId: messageId, processingEndpoint: processingEndpoint, processingStarted: processingStarted); + await using var unitOfWork2 = await StartAuditUnitOfWork(1); + await unitOfWork2.RecordProcessedMessage(duplicatedMessage); + await unitOfWork2.Complete(); + + await configuration.CompleteDBOperation(); + + var queryResult = await MessagesViewStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); + + Assert.That(queryResult.QueryStats.TotalCount, Is.EqualTo(1)); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs b/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs index a5e2bde0ce..83532c48c4 100644 --- a/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs @@ -2,6 +2,7 @@ { using System; using System.Collections.Generic; + using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using Auditing; @@ -10,7 +11,7 @@ using ServiceControl.Audit.Infrastructure; [TestFixture] - class AuditTests : PersistenceTestFixture + partial class AuditTests : PersistenceTestFixture { public override Task Setup() { @@ -78,51 +79,12 @@ await IngestProcessedMessagesAudits( Assert.That(queryResult.Results, Has.Count.EqualTo(2)); } - [Test] - public async Task Can_roundtrip_message_body() - { - string expectedContentType = "text/plain"; - await using var unitOfWork = await StartAuditUnitOfWork(1); - - var body = new byte[100]; - Random.Shared.NextBytes(body); - var processedMessage = MakeMessage(); - - await unitOfWork.RecordProcessedMessage(processedMessage, body); - - await unitOfWork.Complete(); - - var bodyId = GetBodyId(processedMessage); - - var retrievedMessage = await MessagesViewStore.GetMessageBody(bodyId, TestContext.CurrentContext.CancellationToken); - - Assert.That(retrievedMessage, Is.Not.Null); - using (Assert.EnterMultipleScope()) - { - Assert.That(retrievedMessage.Found, Is.True); - Assert.That(retrievedMessage.HasContent, Is.True); - Assert.That(retrievedMessage.ContentLength, Is.EqualTo(body.Length)); - Assert.That(retrievedMessage.Version.HasValue, Is.True); - Assert.That(retrievedMessage.StreamContent, Is.Not.Null); - Assert.That(retrievedMessage.ContentType, Is.EqualTo(expectedContentType)); - } - - var resultBody = new byte[body.Length]; - var readBytes = await retrievedMessage.StreamContent.ReadAsync(resultBody, 0, body.Length); - using (Assert.EnterMultipleScope()) - { - Assert.That(readBytes, Is.EqualTo(body.Length)); - Assert.That(resultBody, Is.EqualTo(body)); - } - } - [Test] public async Task Message_body_validator_is_stable_across_reads() { await using var unitOfWork = await StartAuditUnitOfWork(1); - var body = new byte[100]; - Random.Shared.NextBytes(body); + var body = Encoding.UTF8.GetBytes("{\"Text\":\"The body\"}"); var processedMessage = MakeMessage(); await unitOfWork.RecordProcessedMessage(processedMessage, body); @@ -163,52 +125,6 @@ public async Task Does_respect_max_message_body() } } - [Test] - public async Task Deduplicates_messages_in_same_batch() - { - await using var unitOfWork = await StartAuditUnitOfWork(1); - var messageId = "duplicatedId"; - var processingEndpoint = "endpoint"; - var processingStarted = DateTimeOffset.UtcNow; - - var processedMessage = MakeMessage(messageId: messageId, processingEndpoint: processingEndpoint, processingStarted: processingStarted); - var duplicatedMessage = MakeMessage(messageId: messageId, processingEndpoint: processingEndpoint, processingStarted: processingStarted); - await unitOfWork.RecordProcessedMessage(processedMessage); - await unitOfWork.RecordProcessedMessage(duplicatedMessage); - - await unitOfWork.Complete(); - - await configuration.CompleteDBOperation(); - - var queryResult = await MessagesViewStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); - - Assert.That(queryResult.QueryStats.TotalCount, Is.EqualTo(1)); - } - - [Test] - public async Task Deduplicates_messages_in_different_batches() - { - var messageId = "duplicatedId"; - var processingEndpoint = "endpoint"; - var processingStarted = DateTimeOffset.UtcNow; - - var processedMessage = MakeMessage(messageId: messageId, processingEndpoint: processingEndpoint, processingStarted: processingStarted); - await using var unitOfWork1 = await StartAuditUnitOfWork(1); - await unitOfWork1.RecordProcessedMessage(processedMessage); - await unitOfWork1.Complete(); - - var duplicatedMessage = MakeMessage(messageId: messageId, processingEndpoint: processingEndpoint, processingStarted: processingStarted); - await using var unitOfWork2 = await StartAuditUnitOfWork(1); - await unitOfWork2.RecordProcessedMessage(duplicatedMessage); - await unitOfWork2.Complete(); - - await configuration.CompleteDBOperation(); - - var queryResult = await MessagesViewStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); - - Assert.That(queryResult.QueryStats.TotalCount, Is.EqualTo(1)); - } - [Test] public async Task Does_not_deduplicate_with_different_processing_started_header() { diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFBodyTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFBodyTests.cs new file mode 100644 index 0000000000..a25ca35b7e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFBodyTests.cs @@ -0,0 +1,123 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using NUnit.Framework; + using ServiceControl.Audit.Infrastructure; + + class EFBodyTests : EFPersistenceTestFixture + { + public override Task Setup() + { + SetSettings = settings => settings.MaxBodySizeToStore = MaxBodySizeToStore; + return base.Setup(); + } + + [Test] + public async Task Serves_a_text_body() + { + var message = MakeMessage(); + await Ingest(message, Encoding.UTF8.GetBytes("""{"OrderId":"42"}""")); + + var body = await MessagesViewStore.GetMessageBody(await BodyId(), TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(body.HasContent, Is.True); + Assert.That(body.StringContent, Is.EqualTo("""{"OrderId":"42"}""")); + Assert.That(body.ContentType, Is.EqualTo("application/json")); + Assert.That(body.Version.HasValue, Is.True); + } + } + + [Test] + public async Task Finds_a_body_by_the_unique_message_id_alone() + { + var message = MakeMessage(); + await Ingest(message, Encoding.UTF8.GetBytes("text")); + + var body = await MessagesViewStore.GetMessageBody(message.UniqueMessageId, TestContext.CurrentContext.CancellationToken); + + Assert.That(body.StringContent, Is.EqualTo("text")); + } + + [Test] + public async Task A_text_body_over_the_limit_is_not_served_but_stays_searchable() + { + var text = "findme " + new string('x', MaxBodySizeToStore); + await Ingest(MakeMessage(), Encoding.UTF8.GetBytes(text)); + + var body = await MessagesViewStore.GetMessageBody(await BodyId(), TestContext.CurrentContext.CancellationToken); + var found = await MessagesViewStore.QueryMessages("findme", new PagingInfo(), new SortInfo("time_sent", "desc")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(body.Found, Is.True); + Assert.That(body.HasContent, Is.False, "the body is over MaxBodySizeToStore, which answers 204"); + Assert.That(found.Results, Has.Count.EqualTo(1)); + } + } + + [Test] + public async Task A_binary_body_is_not_stored() + { + await Ingest(MakeMessage(contentType: "application/octet-stream"), [1, 2, 3]); + + var body = await MessagesViewStore.GetMessageBody(await BodyId(), TestContext.CurrentContext.CancellationToken); + + Assert.That(body.Found, Is.False); + } + + [Test] + public async Task A_body_that_is_not_utf8_is_not_stored() + { + await Ingest(MakeMessage(contentType: null), [0xC3, 0x28]); + + var body = await MessagesViewStore.GetMessageBody(await BodyId(), TestContext.CurrentContext.CancellationToken); + + Assert.That(body.Found, Is.False); + } + + [Test] + public async Task A_binary_body_over_the_limit_answers_that_it_was_too_large() + { + await Ingest(MakeMessage(contentType: "application/octet-stream"), new byte[MaxBodySizeToStore + 1]); + + var body = await MessagesViewStore.GetMessageBody(await BodyId(), TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(body.Found, Is.True); + Assert.That(body.HasContent, Is.False); + } + } + + [Test] + public async Task A_message_without_a_body_has_no_body_url() + { + await Ingest(MakeMessage()); + + var view = (await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc"))).Results.Single(); + + Assert.That(view.BodyUrl, Is.Null); + } + + [Test] + public async Task An_unknown_body_is_not_found() + { + var body = await MessagesViewStore.GetMessageBody($"2026010100-{Guid.NewGuid()}", TestContext.CurrentContext.CancellationToken); + + Assert.That(body.Found, Is.False); + } + + async Task BodyId() + { + var bodyUrl = (await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc"))).Results.Single().BodyUrl; + return bodyUrl.Split('/')[2]; + } + + const int MaxBodySizeToStore = 1000; + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFIngestionTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFIngestionTests.cs new file mode 100644 index 0000000000..a4c9b98cda --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFIngestionTests.cs @@ -0,0 +1,178 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using Microsoft.EntityFrameworkCore; + using Monitoring; + using NServiceBus; + using NUnit.Framework; + using ServiceControl.Audit.Infrastructure; + using ServiceControl.SagaAudit; + + class EFIngestionTests : EFPersistenceTestFixture + { + [Test] + public async Task Reports_what_was_recorded() + { + var message = MakeMessage(isRetried: true, extraHeaders: new Dictionary + { + ["NServiceBus.InvokedSagas"] = $"Shipping.ShippingPolicy:{SagaId}", + ["ServiceControl.SagaStateChange"] = $"{SagaId}:Updated" + }); + + await Ingest(message); + + var view = (await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc"))).Results.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(view.Id, Is.EqualTo(message.UniqueMessageId)); + Assert.That(view.MessageId, Is.EqualTo(message.Headers[Headers.MessageId])); + Assert.That(view.MessageType, Is.EqualTo(MessageType)); + Assert.That(view.TimeSent, Is.EqualTo(TimeSent)); + Assert.That(view.TimeSent!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(view.ProcessedAt.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(view.Status, Is.EqualTo(MessageStatus.ResolvedSuccessfully)); + Assert.That(view.MessageIntent, Is.EqualTo(MessageIntent.Publish)); + Assert.That(view.SendingEndpoint.Name, Is.EqualTo("Sender")); + Assert.That(view.SendingEndpoint.HostId, Is.EqualTo(SenderHostId)); + Assert.That(view.ReceivingEndpoint.Host, Is.EqualTo("receiver-host")); + Assert.That(view.CriticalTime, Is.EqualTo(TimeSpan.FromSeconds(3))); + Assert.That(view.ProcessingTime, Is.EqualTo(TimeSpan.FromSeconds(1))); + Assert.That(view.DeliveryTime, Is.EqualTo(TimeSpan.FromSeconds(2))); + Assert.That(view.Headers, Does.Contain(new KeyValuePair(Headers.ProcessingEndpoint, "Receiver"))); + Assert.That(view.InvokedSagas.Single().SagaId, Is.EqualTo(SagaId)); + Assert.That(view.InvokedSagas.Single().ChangeStatus, Is.EqualTo("Updated")); + } + } + + [Test] + public async Task Stores_message_times_of_local_and_unspecified_kinds_as_utc() + { + var localTimeSent = new DateTime(2026, 9, 1, 8, 30, 0, DateTimeKind.Local); + var unspecifiedProcessedAt = new DateTime(2026, 9, 1, 9, 0, 0, DateTimeKind.Unspecified); + + var message = MakeMessage(); + message.MessageMetadata["TimeSent"] = localTimeSent; + message.ProcessedAt = unspecifiedProcessedAt; + + await Ingest(message); + + var view = (await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc"))).Results.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(view.TimeSent, Is.EqualTo(localTimeSent.ToUniversalTime())); + Assert.That(view.TimeSent!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(view.ProcessedAt, Is.EqualTo(unspecifiedProcessedAt)); + Assert.That(view.ProcessedAt.Kind, Is.EqualTo(DateTimeKind.Utc)); + } + } + + [Test] + public async Task Stores_saga_times_of_local_and_unspecified_kinds_as_utc() + { + var sagaId = Guid.NewGuid(); + var localStartTime = new DateTime(2026, 9, 1, 8, 30, 0, DateTimeKind.Local); + var unspecifiedFinishTime = new DateTime(2026, 9, 1, 9, 0, 0, DateTimeKind.Unspecified); + + await using (var unitOfWork = await StartAuditUnitOfWork(1)) + { + await unitOfWork.RecordSagaSnapshot(new SagaSnapshot + { + SagaId = sagaId, + SagaType = "Shipping.ShippingPolicy", + Status = SagaStateChangeStatus.Updated, + StartTime = localStartTime, + FinishTime = unspecifiedFinishTime + }); + await unitOfWork.Complete(); + } + + var change = (await SagaHistoryStore.QuerySagaHistoryById(sagaId)).Results.Changes.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(change.StartTime, Is.EqualTo(localStartTime.ToUniversalTime())); + Assert.That(change.StartTime.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(change.FinishTime, Is.EqualTo(unspecifiedFinishTime)); + Assert.That(change.FinishTime.Kind, Is.EqualTo(DateTimeKind.Utc)); + } + } + + [Test] + public async Task Stores_a_redelivered_message_again() + { + var message = MakeMessage(); + + await Ingest(message); + await Ingest(MakeMessage(messageId: message.Headers[Headers.MessageId])); + + var result = await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(result.QueryStats.TotalCount, Is.EqualTo(2)); + } + + [Test] + public async Task Writes_a_batch_larger_than_one_statement() + { + await Ingest([.. Enumerable.Range(0, 120).Select(_ => MakeMessage())]); + + var result = await MessagesViewStore.GetMessages(true, new PagingInfo(pageSize: 200), new SortInfo("time_sent", "desc")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.QueryStats.TotalCount, Is.EqualTo(120)); + Assert.That(result.Results, Has.Count.EqualTo(120)); + } + } + + [Test] + public async Task Writes_nothing_for_a_batch_that_is_not_completed() + { + await using (var unitOfWork = await StartAuditUnitOfWork(1)) + { + await unitOfWork.RecordProcessedMessage(MakeMessage()); + } + + var stored = await WithDbContext((dbContext, token) => dbContext.AuditMessages.CountAsync(token)); + + Assert.That(stored, Is.Zero); + } + + [Test] + public async Task Stores_a_redelivered_saga_snapshot_again() + { + var sagaId = Guid.NewGuid(); + + for (var delivery = 0; delivery < 2; delivery++) + { + await using var unitOfWork = await StartAuditUnitOfWork(1); + await unitOfWork.RecordSagaSnapshot(new SagaSnapshot + { + SagaId = sagaId, + SagaType = "Shipping.ShippingPolicy", + Status = SagaStateChangeStatus.Updated, + StartTime = TimeSent, + FinishTime = TimeSent.AddSeconds(1), + InitiatingMessage = new InitiatingMessage { MessageId = "initiating", MessageType = MessageType, TimeSent = TimeSent }, + OutgoingMessages = [new ResultingMessage { MessageId = "outgoing", Destination = "Billing", TimeSent = TimeSent }] + }); + await unitOfWork.Complete(); + } + + var history = (await SagaHistoryStore.QuerySagaHistoryById(sagaId)).Results; + + using (Assert.EnterMultipleScope()) + { + Assert.That(history.Changes, Has.Count.EqualTo(2)); + Assert.That(history.Changes[0].InitiatingMessage.MessageId, Is.EqualTo("initiating")); + Assert.That(history.Changes[0].OutgoingMessages.Single().Destination, Is.EqualTo("Billing")); + } + } + + static readonly Guid SagaId = Guid.NewGuid(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFPersistenceTestFixture.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFPersistenceTestFixture.cs new file mode 100644 index 0000000000..2e8bd897d5 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFPersistenceTestFixture.cs @@ -0,0 +1,94 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Auditing; + using Microsoft.Extensions.DependencyInjection; + using Monitoring; + using NServiceBus; + using ServiceControl.Audit.Persistence.EFCore.DbContexts; + + abstract class EFPersistenceTestFixture : PersistenceTestFixture + { + protected async Task Ingest(params ProcessedMessage[] messages) + { + await using var unitOfWork = await StartAuditUnitOfWork(messages.Length); + foreach (var message in messages) + { + await unitOfWork.RecordProcessedMessage(message); + } + + await unitOfWork.Complete(); + await configuration.CompleteDBOperation(); + } + + protected async Task Ingest(ProcessedMessage message, byte[] body) + { + await using var unitOfWork = await StartAuditUnitOfWork(1); + await unitOfWork.RecordProcessedMessage(message, body); + await unitOfWork.Complete(); + await configuration.CompleteDBOperation(); + } + + protected async Task WithDbContext(Func> query) + { + await using var scope = ServiceProvider.CreateAsyncScope(); + return await query(scope.ServiceProvider.GetRequiredService(), TestTimeoutCancellationToken); + } + + protected static ProcessedMessage MakeMessage( + string messageId = null, + string receivingEndpoint = "Receiver", + string contentType = "application/json", + bool isRetried = false, + Dictionary extraHeaders = null, + string conversationId = null) + { + messageId ??= Guid.NewGuid().ToString(); + conversationId ??= Guid.NewGuid().ToString(); + + var headers = new Dictionary + { + [Headers.MessageId] = messageId, + [Headers.ProcessingEndpoint] = receivingEndpoint, + [Headers.MessageIntent] = MessageIntent.Publish.ToString(), + [Headers.ConversationId] = conversationId, + [Headers.EnclosedMessageTypes] = MessageType + }; + + if (contentType != null) + { + headers[Headers.ContentType] = contentType; + } + + foreach (var header in extraHeaders ?? []) + { + headers[header.Key] = header.Value; + } + + var metadata = new Dictionary + { + ["MessageId"] = messageId, + ["MessageType"] = MessageType, + ["TimeSent"] = TimeSent, + ["ConversationId"] = conversationId, + ["IsSystemMessage"] = false, + ["IsRetried"] = isRetried, + ["SendingEndpoint"] = new EndpointDetails { Name = "Sender", HostId = SenderHostId, Host = "sender-host" }, + ["ReceivingEndpoint"] = new EndpointDetails { Name = receivingEndpoint, HostId = ReceiverHostId, Host = "receiver-host" }, + ["CriticalTime"] = TimeSpan.FromSeconds(3), + ["ProcessingTime"] = TimeSpan.FromSeconds(1), + ["DeliveryTime"] = TimeSpan.FromSeconds(2) + }; + + return new ProcessedMessage(headers, metadata); + } + + protected const string MessageType = "Shipping.OrderShipped"; + protected static readonly DateTime TimeSent = new(2026, 9, 1, 10, 30, 0, DateTimeKind.Utc); + protected static readonly Guid SenderHostId = Guid.NewGuid(); + protected static readonly Guid ReceiverHostId = Guid.NewGuid(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFPersistenceTestsConfiguration.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFPersistenceTestsConfiguration.cs new file mode 100644 index 0000000000..be78469c28 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFPersistenceTestsConfiguration.cs @@ -0,0 +1,122 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Auditing.BodyStorage; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using Microsoft.Extensions.Time.Testing; + using NServiceBus.CustomChecks; + using ServiceControl.Audit.Persistence.EFCore.Abstractions; + using ServiceControl.Audit.Persistence.EFCore.Implementation; + using UnitOfWork; + + abstract class EFPersistenceTestsConfiguration : IPersistenceTestsConfiguration + { + public IAuditMessagesViewDataStore MessagesViewStore { get; private set; } + + public ISagaHistoryDataStore SagaHistoryStore { get; private set; } + + public IFailedAuditStorage FailedAuditStorage { get; private set; } + + public IBodyStorage BodyStorage { get; private set; } + + public IAuditIngestionUnitOfWorkFactory AuditIngestionUnitOfWorkFactory { get; private set; } + + public IServiceProvider ServiceProvider => host.Services; + + public abstract string Name { get; } + + public string ConnectionString { get; private set; } + + public string Schema { get; } = $"sc_test_{Guid.NewGuid():N}"; + + // PostgreSQL keeps only microseconds, so a finer seed would not round trip. + public FakeTimeProvider TimeProvider { get; } = new(DateTimeOffset.FromUnixTimeMilliseconds(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())); + + public async Task Configure(Action setSettings) + { + ConnectionString = await GetConnectionString(); + await CreateSchema(ConnectionString, Schema); + + var settings = new PersistenceSettings(TimeSpan.FromDays(1), true, 100000); + settings.PersisterSpecificSettings[EFPersistenceConfigurationBase.ConnectionStringKey] = ConnectionString; + settings.PersisterSpecificSettings[EFPersistenceConfigurationBase.SchemaKey] = Schema; + + setSettings(settings); + + var configuration = CreateConfiguration(); + var persistence = configuration.Create(settings); + + var hostBuilder = Host.CreateApplicationBuilder(); + hostBuilder.Services.AddSingleton(TimeProvider); + + persistence.AddInstaller(hostBuilder.Services); + persistence.AddPersistence(hostBuilder.Services); + + // Tests sweep themselves, which the hourly loop would race whenever a test moves the clock. + foreach (var retention in hostBuilder.Services.Where(descriptor => descriptor.ImplementationType == typeof(AuditRetention)).ToList()) + { + hostBuilder.Services.Remove(retention); + } + + foreach (var type in configuration.GetType().Assembly.DefinedTypes.Concat(typeof(EFPersistenceConfigurationBase).Assembly.DefinedTypes)) + { + if (!type.IsAbstract && type.IsAssignableTo(typeof(ICustomCheck))) + { + hostBuilder.Services.AddTransient(typeof(ICustomCheck), type); + } + } + + host = hostBuilder.Build(); + await host.StartAsync(); + + MessagesViewStore = DecorateMessagesViewStore(host.Services.GetRequiredService()); + SagaHistoryStore = host.Services.GetRequiredService(); + FailedAuditStorage = host.Services.GetRequiredService(); + AuditIngestionUnitOfWorkFactory = host.Services.GetRequiredService(); + BodyStorage = new MessagesViewBodyStorage(MessagesViewStore); + } + + public virtual Task CompleteDBOperation() => Task.CompletedTask; + + protected virtual IAuditMessagesViewDataStore DecorateMessagesViewStore(IAuditMessagesViewDataStore store) => store; + + public async Task Cleanup() + { + if (host != null) + { + await host.StopAsync(); + host.Dispose(); + } + + if (ConnectionString != null) + { + await DropSchema(ConnectionString, Schema); + } + } + + internal AuditRetention CreateRetention() => ActivatorUtilities.CreateInstance(ServiceProvider); + + public abstract Task GetConnectionString(); + + protected abstract Task CreateSchema(string connectionString, string schema); + + protected abstract Task DropSchema(string connectionString, string schema); + + public abstract EFPersistenceConfigurationBase CreateConfiguration(); + + IHost host; + + sealed class MessagesViewBodyStorage(IAuditMessagesViewDataStore messagesViewStore) : IBodyStorage + { + public Task Store(string bodyId, string contentType, int bodySize, System.IO.Stream bodyStream, CancellationToken cancellationToken = default) => + throw new NotSupportedException("The EF Core persisters store a body with the message it belongs to."); + + public Task TryFetch(string bodyId, CancellationToken cancellationToken = default) => + messagesViewStore.GetMessageBody(bodyId, cancellationToken); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs new file mode 100644 index 0000000000..7e78b13dc7 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs @@ -0,0 +1,151 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Monitoring; + using NServiceBus; + using NUnit.Framework; + using ServiceControl.Audit.Auditing.MessagesView; + using ServiceControl.Audit.Infrastructure; + using ServiceControl.Audit.Persistence.EFCore.EntityConfigurations; + + class EFQueryTests : EFPersistenceTestFixture + { + [TestCaseSource(nameof(TimeRangeRoutes))] + public async Task Filters_on_a_time_range_given_with_an_offset(Func>>> route) + { + await Ingest(MakeMessage(extraHeaders: new Dictionary { [SearchableHeader] = SearchableWord })); + + var inRange = await route(MessagesViewStore, new DateTimeRange("2026-09-01T12:00:00+02:00", "2026-09-01T12:45:00+02:00"), TestTimeoutCancellationToken); + var outOfRange = await route(MessagesViewStore, new DateTimeRange("2026-09-01T12:45:00+02:00"), TestTimeoutCancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(inRange.Results, Has.Count.EqualTo(1)); + Assert.That(outOfRange.Results, Is.Empty); + } + } + + [TestCaseSource(nameof(TimeRangeRoutes))] + public async Task Filters_on_a_time_range_given_without_a_zone(Func>>> route) + { + await Ingest(MakeMessage(extraHeaders: new Dictionary { [SearchableHeader] = SearchableWord })); + + var inRange = await route(MessagesViewStore, new DateTimeRange("2026-09-01T10:15:00", "2026-09-01T10:45:00"), TestTimeoutCancellationToken); + var outOfRange = await route(MessagesViewStore, new DateTimeRange("2026-09-01T10:45:00"), TestTimeoutCancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(inRange.Results, Has.Count.EqualTo(1)); + Assert.That(outOfRange.Results, Is.Empty); + } + } + + static IEnumerable TimeRangeRoutes() + { + yield return Route(nameof(IAuditMessagesViewDataStore.GetMessages), + (store, range, token) => store.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc"), range, token)); + yield return Route(nameof(IAuditMessagesViewDataStore.QueryMessages), + (store, range, token) => store.QueryMessages(SearchableWord, new PagingInfo(), new SortInfo("time_sent", "desc"), range, token)); + yield return Route(nameof(IAuditMessagesViewDataStore.QueryMessagesByReceivingEndpointAndKeyword), + (store, range, token) => store.QueryMessagesByReceivingEndpointAndKeyword("Receiver", SearchableWord, new PagingInfo(), new SortInfo("time_sent", "desc"), range, token)); + yield return Route(nameof(IAuditMessagesViewDataStore.QueryMessagesByReceivingEndpoint), + (store, range, token) => store.QueryMessagesByReceivingEndpoint(true, "Receiver", new PagingInfo(), new SortInfo("time_sent", "desc"), range, token)); + } + + static TestCaseData Route(string name, Func>>> route) => + new TestCaseData(route).SetArgDisplayNames(name); + + const string SearchableHeader = "Order.Reference"; + const string SearchableWord = "meridian"; + + [Test] + public async Task Finds_a_header_value_that_is_not_ascii() + { + await Ingest(MakeMessage(extraHeaders: new Dictionary { ["Shipping.Step"] = "Bestellprüfung" })); + + var found = await MessagesViewStore.QueryMessages("Bestellprüfung", new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(found.Results, Has.Count.EqualTo(1)); + } + + [Test] + public async Task Finds_a_word_that_is_quoted_in_single_quotes() + { + await Ingest(MakeMessage(extraHeaders: new Dictionary { ["Faults.Message"] = "The given key 'CustomerId' was not present in the dictionary." })); + + var found = await MessagesViewStore.QueryMessages("CustomerId", new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(found.Results, Has.Count.EqualTo(1)); + } + + [Test] + public async Task Sorts_by_status_in_the_order_the_primary_merges_instance_pages() + { + await Ingest(MakeMessage(isRetried: true), MakeMessage()); + + var ascending = await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("status", "asc")); + + Assert.That(ascending.Results.Select(message => message.Status), Is.EqualTo(new[] { MessageStatus.Successful, MessageStatus.ResolvedSuccessfully })); + } + + [Test] + public async Task Stores_a_message_id_of_any_length() + { + var messageId = new string('m', 600); + + await Ingest(MakeMessage(messageId: messageId)); + + var view = (await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc"))).Results.Single(); + + Assert.That(view.MessageId, Is.EqualTo(messageId)); + } + + [Test] + public async Task Finds_a_conversation_whose_id_is_longer_than_an_index_allows() + { + var conversationId = new string('c', 600); + + await Ingest(MakeMessage(conversationId: conversationId)); + + var conversation = await MessagesViewStore.QueryMessagesByConversationId(conversationId, new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(conversation.Results, Has.Count.EqualTo(1)); + } + + [Test] + public async Task Keeps_apart_conversations_whose_ids_differ_only_past_the_index_length() + { + var sharedStart = new string('c', ColumnLengths.ShortTextLength); + + await Ingest( + MakeMessage(messageId: "first", conversationId: sharedStart + "-first"), + MakeMessage(messageId: "second", conversationId: sharedStart + "-second")); + + var conversation = await MessagesViewStore.QueryMessagesByConversationId(sharedStart + "-first", new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(conversation.Results.Select(view => view.MessageId), Is.EqualTo(new[] { "first" })); + } + + [Test] + public void Fits_a_long_value_to_the_index_without_splitting_a_surrogate_pair() + { + var value = new string('x', 384) + "😀" + new string('y', 100); + + Assert.That(ColumnLengths.FitToIndex(value), Does.StartWith(new string('x', 384) + "~")); + } + + [Test] + public async Task Reports_an_over_length_conversation_id_that_needs_no_url_encoding() + { + await Ingest(MakeMessage(conversationId: new string('c', 600))); + + var view = (await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc"))).Results.Single(); + + Assert.That(Uri.EscapeDataString(view.ConversationId), Is.EqualTo(view.ConversationId), "ServicePulse puts the conversation id into a URL path without encoding it"); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFRetentionCustomCheckTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFRetentionCustomCheckTests.cs new file mode 100644 index 0000000000..66b698c25f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFRetentionCustomCheckTests.cs @@ -0,0 +1,80 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using NServiceBus.CustomChecks; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.Implementation; + using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + + class EFRetentionCustomCheckTests : EFPersistenceTestFixture + { + AuditRetentionCustomCheck Check => ServiceProvider.GetServices().OfType().Single(); + + [Test] + public async Task Check_fails_once_three_consecutive_sweeps_have_failed() + { + var retention = CreateRetentionWithUnreachableDatabase(); + + Assert.ThrowsAsync(() => retention.SweepNow()); + Assert.ThrowsAsync(() => retention.SweepNow()); + var afterTwo = await Check.PerformCheck(); + Assert.ThrowsAsync(() => retention.SweepNow()); + var afterThree = await Check.PerformCheck(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(afterTwo.HasFailed, Is.False); + Assert.That(afterThree.HasFailed, Is.True); + Assert.That(afterThree.FailureReason, Does.Contain(UnreachableRetentionLock.Reason)); + } + } + + [Test] + public async Task A_successful_sweep_clears_the_failure() + { + FailThreeSweeps(); + + await configuration.CreateRetention().SweepNow(); + + Assert.That((await Check.PerformCheck()).HasFailed, Is.False); + } + + [Test] + public async Task A_sweep_skipped_for_the_lock_leaves_the_failure_in_place() + { + FailThreeSweeps(); + + await using (await ServiceProvider.GetRequiredService().TryAcquire()) + { + await configuration.CreateRetention().SweepNow(); + } + + Assert.That((await Check.PerformCheck()).HasFailed, Is.True); + } + + void FailThreeSweeps() + { + var retention = CreateRetentionWithUnreachableDatabase(); + + for (var sweep = 0; sweep < 3; sweep++) + { + Assert.ThrowsAsync(() => retention.SweepNow()); + } + } + + AuditRetention CreateRetentionWithUnreachableDatabase() => + ActivatorUtilities.CreateInstance(ServiceProvider, new UnreachableRetentionLock()); + + sealed class UnreachableRetentionLock : IRetentionLock + { + public const string Reason = "The database is unreachable"; + + public Task TryAcquire(CancellationToken cancellationToken = default) => + Task.FromException(new InvalidOperationException(Reason)); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFRetentionTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFRetentionTests.cs new file mode 100644 index 0000000000..d06a996705 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFRetentionTests.cs @@ -0,0 +1,88 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Linq; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using NServiceBus; + using NUnit.Framework; + using ServiceControl.Audit.Infrastructure; + using ServiceControl.Audit.Persistence.EFCore.Infrastructure; + + class EFRetentionTests : EFPersistenceTestFixture + { + public override Task Setup() + { + SetSettings = settings => settings.AuditRetentionPeriod = TimeSpan.FromDays(1); + return base.Setup(); + } + + [Test] + public async Task Removes_rows_older_than_the_retention_period() + { + var expired = MakeMessage(); + await Ingest(expired); + + configuration.TimeProvider.Advance(TimeSpan.FromDays(3)); + var retention = configuration.CreateRetention(); + await retention.SweepNow(); + + var live = MakeMessage(); + await Ingest(live); + await retention.SweepNow(); + + var remaining = await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(remaining.Results.Select(message => message.MessageId), Is.EqualTo(new[] { live.Headers[Headers.MessageId] })); + } + + [Test] + public async Task Keeps_rows_until_the_retention_period_has_passed() + { + await Ingest(MakeMessage()); + + configuration.TimeProvider.Advance(TimeSpan.FromDays(1)); + + await configuration.CreateRetention().SweepNow(); + + var remaining = await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(remaining.Results, Has.Count.EqualTo(1)); + } + + [Test] + public async Task Skips_the_sweep_while_another_instance_holds_the_lock() + { + await Ingest(MakeMessage()); + configuration.TimeProvider.Advance(TimeSpan.FromDays(3)); + + await using (await ServiceProvider.GetRequiredService().TryAcquire()) + { + await configuration.CreateRetention().SweepNow(); + } + + var remaining = await MessagesViewStore.GetMessages(true, new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(remaining.Results, Has.Count.EqualTo(1)); + } + + [Test] + public async Task The_retention_lock_is_released_with_its_handle() + { + var retentionLock = ServiceProvider.GetRequiredService(); + + var first = await retentionLock.TryAcquire(); + var whileHeld = await retentionLock.TryAcquire(); + await first!.DisposeAsync(); + var afterRelease = await retentionLock.TryAcquire(); + await afterRelease!.DisposeAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first, Is.Not.Null); + Assert.That(whileHeld, Is.Null); + Assert.That(afterRelease, Is.Not.Null); + } + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/SchemaMustExistTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/SchemaMustExistTests.cs new file mode 100644 index 0000000000..6ef1503937 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/SchemaMustExistTests.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Audit.Persistence.Tests +{ + using System; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using NUnit.Framework; + using ServiceControl.Audit.Persistence.EFCore.Abstractions; + + [TestFixture] + class SchemaMustExistTests + { + [Test] + public async Task Setup_fails_when_the_configured_schema_does_not_exist() + { + var testsConfiguration = new PersistenceTestsConfiguration(); + var schema = $"sc_absent_{Guid.NewGuid():n}"; + + var settings = new PersistenceSettings(TimeSpan.FromDays(1), true, 100000); + settings.PersisterSpecificSettings[EFPersistenceConfigurationBase.ConnectionStringKey] = await testsConfiguration.GetConnectionString(); + settings.PersisterSpecificSettings[EFPersistenceConfigurationBase.SchemaKey] = schema; + + var hostBuilder = Host.CreateApplicationBuilder(); + testsConfiguration.CreateConfiguration().Create(settings).AddInstaller(hostBuilder.Services); + using var host = hostBuilder.Build(); + + var exception = Assert.ThrowsAsync(() => host.StartAsync()); + + Assert.That(exception.Message, Does.Contain(schema).And.Contain("does not exist")); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence/DevelopmentPersistenceLocations.cs b/src/ServiceControl.Audit.Persistence/DevelopmentPersistenceLocations.cs index 5c533739b7..4caae022c9 100644 --- a/src/ServiceControl.Audit.Persistence/DevelopmentPersistenceLocations.cs +++ b/src/ServiceControl.Audit.Persistence/DevelopmentPersistenceLocations.cs @@ -18,6 +18,8 @@ static DevelopmentPersistenceLocations() if (!string.IsNullOrWhiteSpace(srcFolder) && srcFolder.EndsWith("src")) { ManifestFiles.Add(BuildManifestPath(srcFolder, "ServiceControl.Audit.Persistence.RavenDB")); + ManifestFiles.Add(BuildManifestPath(srcFolder, "ServiceControl.Audit.Persistence.EFCore.SqlServer")); + ManifestFiles.Add(BuildManifestPath(srcFolder, "ServiceControl.Audit.Persistence.EFCore.PostgreSql")); } } diff --git a/src/ServiceControl.Audit.UnitTests/Infrastructure/When_instance_is_setup.cs b/src/ServiceControl.Audit.UnitTests/Infrastructure/When_instance_is_setup.cs index 7deb11df5b..af61ccd47a 100644 --- a/src/ServiceControl.Audit.UnitTests/Infrastructure/When_instance_is_setup.cs +++ b/src/ServiceControl.Audit.UnitTests/Infrastructure/When_instance_is_setup.cs @@ -16,6 +16,7 @@ using Persistence; using Transports; + [NonParallelizable] class When_instance_is_setup { [Test] diff --git a/src/ServiceControl.Audit/App.config b/src/ServiceControl.Audit/App.config index 2e6b60036a..5ea329b21c 100644 --- a/src/ServiceControl.Audit/App.config +++ b/src/ServiceControl.Audit/App.config @@ -20,6 +20,9 @@ These settings are only here so that we can debug ServiceControl while developin + + + diff --git a/src/ServiceControl.Audit/ServiceControl.Audit.csproj b/src/ServiceControl.Audit/ServiceControl.Audit.csproj index 9df69e711f..ae81341595 100644 --- a/src/ServiceControl.Audit/ServiceControl.Audit.csproj +++ b/src/ServiceControl.Audit/ServiceControl.Audit.csproj @@ -57,7 +57,9 @@ + + diff --git a/src/ServiceControl.Persistence/InternalCustomCheckClassification.cs b/src/ServiceControl.Persistence/InternalCustomCheckClassification.cs index 3c314eaed9..3e72cdc03e 100644 --- a/src/ServiceControl.Persistence/InternalCustomCheckClassification.cs +++ b/src/ServiceControl.Persistence/InternalCustomCheckClassification.cs @@ -51,6 +51,8 @@ public static class InternalCustomCheckClassification "Audit Database Index Lag", "Audit Database Search Engine", "ServiceControl.Audit database", + "Audit partition provisioning", // EF Core PostgreSQL persister + "Audit retention", // EF Core persisters }; /// diff --git a/src/ServiceControl.UnitTests/CustomChecks/InternalCustomCheckClassificationTests.cs b/src/ServiceControl.UnitTests/CustomChecks/InternalCustomCheckClassificationTests.cs index 6f83521d4b..5646f12565 100644 --- a/src/ServiceControl.UnitTests/CustomChecks/InternalCustomCheckClassificationTests.cs +++ b/src/ServiceControl.UnitTests/CustomChecks/InternalCustomCheckClassificationTests.cs @@ -34,6 +34,7 @@ public void Internal_checks_are_flagged_internal() [TestCase("Dead Letter Queue")] [TestCase("ServiceControl body storage")] [TestCase("ServiceControl Retention")] + [TestCase("Audit retention")] [TestCase("Audit Message Ingestion Process")] public void Every_shipped_check_is_internal(string id) { diff --git a/src/ServiceControl.slnx b/src/ServiceControl.slnx index 5abc0cecec..e9a507b944 100644 --- a/src/ServiceControl.slnx +++ b/src/ServiceControl.slnx @@ -18,13 +18,20 @@ + + + + + + + diff --git a/src/ServiceControl/Notifications/Email/CustomChecksMailNotification.cs b/src/ServiceControl/Notifications/Email/CustomChecksMailNotification.cs index ef697065c2..738a628fcc 100644 --- a/src/ServiceControl/Notifications/Email/CustomChecksMailNotification.cs +++ b/src/ServiceControl/Notifications/Email/CustomChecksMailNotification.cs @@ -20,12 +20,23 @@ class CustomChecksMailNotification : IDomainHandler, IDomainH "Audit Message Ingestion Process", "Audit Message Ingestion", "ServiceControl.Audit database", + "Audit Database Index Lag", + "Audit Database Search Engine", + "Audit partition provisioning", + "Audit retention", + "RavenDB dirty memory", "Dead Letter Queue", "ServiceControl Primary Instance", "ServiceControl database", "ServiceControl Remotes", "Error Message Ingestion Process", - "Error Message Ingestion" + "Error Message Ingestion", + "Error Database Index Errors", + "Error Database Index Lag", + "Error Database Search Engine", + "Message Ingestion Process", + "ServiceControl body storage", + "ServiceControl Retention" }; public CustomChecksMailNotification(IMessageSession messageSession, Settings settings, EmailThrottlingState throttlingState, ILogger logger) diff --git a/src/ServiceControlInstaller.Engine.UnitTests/ApprovalFiles/PersistenceManifestTests.ApproveAuditInstanceManifests.approved.txt b/src/ServiceControlInstaller.Engine.UnitTests/ApprovalFiles/PersistenceManifestTests.ApproveAuditInstanceManifests.approved.txt index 59a50c03c0..0489c1d713 100644 --- a/src/ServiceControlInstaller.Engine.UnitTests/ApprovalFiles/PersistenceManifestTests.ApproveAuditInstanceManifests.approved.txt +++ b/src/ServiceControlInstaller.Engine.UnitTests/ApprovalFiles/PersistenceManifestTests.ApproveAuditInstanceManifests.approved.txt @@ -1,4 +1,6 @@ [ + "PostgreSQL: PostgreSQL", "RavenDB: RavenDB", - "RavenDB35: RavenDB 3.5 (Legacy)" + "RavenDB35: RavenDB 3.5 (Legacy)", + "SQLServer: SQL Server" ] \ No newline at end of file diff --git a/src/ServiceControlInstaller.Engine.UnitTests/InstallationTests.cs b/src/ServiceControlInstaller.Engine.UnitTests/InstallationTests.cs index 33bb8f0193..b6516ac3a2 100644 --- a/src/ServiceControlInstaller.Engine.UnitTests/InstallationTests.cs +++ b/src/ServiceControlInstaller.Engine.UnitTests/InstallationTests.cs @@ -2,9 +2,11 @@ { using System.Collections.Generic; using System.IO; + using System.Linq; using System.Text.Json; using NUnit.Framework; using Particular.Approvals; + using ServiceControlInstaller.Engine.Configuration.ServiceControl; using ServiceControlInstaller.Engine.FileSystem; using ServiceControlInstaller.Engine.Instances; @@ -33,6 +35,11 @@ public void Audit_install_should_write_expected_config_file(string manifestPath) Assert.Ignore("Don't care about config for unsupported persistence types."); } + if (persistenceManifest.Settings.Any(setting => setting.Mandatory && setting.DefaultValue == null && !InstallerSuppliedSettings.Contains(setting.Name))) + { + Assert.Ignore("The installer cannot create an instance on this persister yet: it has no value for one of its mandatory settings."); + } + var newInstance = new ServiceControlAuditNewInstance(persistenceManifest); @@ -64,6 +71,13 @@ public void Audit_install_should_write_expected_config_file(string manifestPath) } } + static readonly string[] InstallerSuppliedSettings = + [ + AuditInstanceSettingsList.DBPath.Name, + AuditInstanceSettingsList.DatabaseMaintenancePort.Name, + AuditInstanceSettingsList.LogPath.Name + ]; + public static IEnumerable GetAuditPersistenceManifestPaths() { var deployPath = Path.GetFullPath(Path.Combine(TestContext.CurrentContext.TestDirectory, "..", "..", "..", "..", "..", "deploy")); diff --git a/src/ServiceControlInstaller.Engine.UnitTests/Validation/StorageEngineUpgradeTests.cs b/src/ServiceControlInstaller.Engine.UnitTests/Validation/StorageEngineUpgradeTests.cs index f5b24e2919..54a1268282 100644 --- a/src/ServiceControlInstaller.Engine.UnitTests/Validation/StorageEngineUpgradeTests.cs +++ b/src/ServiceControlInstaller.Engine.UnitTests/Validation/StorageEngineUpgradeTests.cs @@ -19,7 +19,9 @@ public class StorageEngineUpgradeTests static readonly Dictionary AuditUpgradableInPlace = new() { ["RavenDB"] = true, - ["RavenDB35"] = false + ["RavenDB35"] = false, + ["SQLServer"] = true, + ["PostgreSQL"] = true }; [Test] diff --git a/src/ServiceControlInstaller.Packaging.UnitTests/AuditDeploymentPackageTests.cs b/src/ServiceControlInstaller.Packaging.UnitTests/AuditDeploymentPackageTests.cs index 7860b0e4e1..5f1ff4ff57 100644 --- a/src/ServiceControlInstaller.Packaging.UnitTests/AuditDeploymentPackageTests.cs +++ b/src/ServiceControlInstaller.Packaging.UnitTests/AuditDeploymentPackageTests.cs @@ -17,7 +17,9 @@ public void Should_package_storages_individually() { var expectedPersisters = new[] { "RavenDB35", // Still must exist, as Raven35 persistence.manifest file must be available for SCMU to understand old versions - "RavenDB" + "RavenDB", + "SQLServer", + "PostgreSQL" }; var persisters = deploymentPackage.DeploymentUnits.Where(u => u.Category == "Persisters");