Add the missing usage report keys for the primary instance, its storage and its transport

src/ServiceControl.AcceptanceTests/Licensing/When_reporting_the_environment.cs

@@ -16,6 +16,11 @@ namespace ServiceControl.AcceptanceTests.Licensing
     using Particular.LicensingComponent.Contracts;
     using Particular.LicensingComponent.MonitoringThroughput;
     using Particular.LicensingComponent.Shared;
+    using Recoverability.MessageRedirects;
+    using ServiceBus.Management.Infrastructure.Settings;
+    using ServiceControl.Monitoring;
+    using ServiceControl.Persistence;
+    using Conventions = NServiceBus.AcceptanceTesting.Customization.Conventions;
 
     class When_reporting_the_environment : AcceptanceTest
     {
@@ -58,9 +63,7 @@ await Define<Context>()
                 Assert.That(data["Host.Model"], Is.AnyOf("Container", "WindowsService", "Console"));
                 Assert.That(data["Storage.Type"], Is.Not.Empty);
                 Assert.That(data["Storage.BodyStorage.Type"], Is.Not.Empty);
-                Assert.That(data["Storage.BodyStorage.Auth"], Is.AnyOf("ManagedIdentity", "SharedKeyOrSas", "IamRole", "StaticCredentials", "NotApplicable"));
                 Assert.That(data["Storage.HostingSource"], Is.AnyOf("Probe", "Configuration", "ConnectionString", "None"));
-                Assert.That(data["Security.Authentication"], Is.AnyOf("Enabled", "Disabled"));
                 Assert.That(data["Features.EmailNotifications"], Is.AnyOf("Enabled", "Disabled", "NotConfigured", "ReadFailed"));
                 Assert.That(int.Parse(data["Retention.ErrorHours"]), Is.GreaterThan(0));
 
@@ -85,19 +88,125 @@ await Define<Context>()
             "Storage.HostingSource",
             "Storage.FullTextSearch",
             "Storage.BodyStorage.Type",
-            "Storage.BodyStorage.Auth",
-            "Security.Authentication",
-            "Security.RoleBasedAuthorization",
-            "Security.Https",
+            "Storage.QueryTimeoutSeconds",
+            "Storage.FreeSpaceThresholdPercent",
+            "Transport.Type",
             "Features.IntegratedServicePulse",
             "Features.MessageEditing",
             "Features.ExternalIntegrationsPublishing",
             "Features.ForwardErrorMessages",
+            "Features.ErrorIngestion",
+            "Features.ConfigurationValidation",
             "Features.EmailNotifications",
+            "ServicePulse.MonitoringUrl",
+            "ServicePulse.DefaultRoute",
+            "ServicePulse.ShowPendingRetry",
+            "Host.VirtualDirectory",
+            "Host.ShutdownTimeoutSeconds",
+            "Logging.Providers",
+            "Logging.Level",
+            "Telemetry.OtlpMetrics",
+            "Limits.ExternalIntegrationsBatchSize",
+            "Ingestion.Error.MaxConcurrency",
+            "Ingestion.Error.BatchSize",
+            "Ingestion.Error.MaxParallelWriters",
+            "Ingestion.Error.BatchTimeoutMs",
+            "Ingestion.Error.RestartAfterFailureSeconds",
+            "Heartbeats.TrackInstancesDefault",
+            "Heartbeats.TrackInstancesOverrides",
+            "Heartbeats.KnownInstances",
+            "Heartbeats.MonitoredInstances",
+            "Heartbeats.GracePeriodSeconds",
+            "Recoverability.Redirects",
+            "Recoverability.RetryHistoryDepth",
+            "Licensing.ReportMasks",
             "Retention.ErrorHours",
             "Retention.EventsHours"
         ];
 
+        [Test]
+        public async Task Should_count_the_choices_made_in_servicepulse_without_revealing_them()
+        {
+            JsonDocument report = null;
+
+            await Define<Context>()
+                .WithEndpoint<MonitoringInstance>()
+                .WithEndpoint<ScalingOut>()
+                .Do("Wait for the first heartbeat of the endpoint that scales out", async _ =>
+                {
+                    var endpoints = await this.TryGetMany<EndpointsView>("/api/endpoints", endpoint => endpoint.Name == ScalingOutEndpoint && endpoint.Monitored);
+
+                    return endpoints.HasResult;
+                })
+                .Do("Stop tracking instances by default", async _ =>
+                {
+                    await this.Patch("/api/endpointssettings", new { track_instances = false });
+
+                    var settings = await this.TryGetMany<SettingsData>("/api/endpointssettings",
+                        setting => setting.Name == string.Empty && !setting.TrackInstances);
+
+                    return settings.HasResult;
+                })
+                .Do("Keep tracking the endpoint that scales out", async _ =>
+                {
+                    await this.Patch($"/api/endpointssettings/{ScalingOutEndpoint}", new { track_instances = true });
+
+                    var settings = await this.TryGetMany<SettingsData>("/api/endpointssettings",
+                        setting => setting.Name == ScalingOutEndpoint && setting.TrackInstances);
+
+                    return settings.HasResult;
+                })
+                .Do("Redirect a retired queue", async _ =>
+                {
+                    await this.Post("/api/redirects", new RedirectRequest { fromphysicaladdress = RetiredQueue, tophysicaladdress = ReplacementQueue });
+
+                    var redirects = await this.TryGetMany<MessageRedirectFromJson>("/api/redirects");
+
+                    return redirects.HasResult;
+                })
+                .Do("Mask the customer's name in the report", async _ =>
+                {
+                    await this.Post("/api/licensing/settings/masks/update", new[] { MaskedWord });
+
+                    return true;
+                })
+                .Do("Wait for the throughput data to be recorded", async _ =>
+                {
+                    var available = await this.TryGet<ReportGenerationState>(
+                        "/api/licensing/report/available", state => state.ReportCanBeGenerated);
+
+                    return available.HasResult;
+                })
+                .Do("Download the report", async _ =>
+                {
+                    var archive = await this.DownloadData("/api/licensing/report/file?spVersion=1.2.3");
+
+                    report = ReadReport(archive);
+
+                    return true;
+                })
+                .Done(_ => true)
+                .Run();
+
+            var data = report.RootElement
+                .GetProperty("ReportData")
+                .GetProperty("EnvironmentInformation")
+                .GetProperty("EnvironmentData")
+                .EnumerateObject()
+                .ToDictionary(entry => entry.Name, entry => entry.Value.GetString());
+
+            using (Assert.EnterMultipleScope())
+            {
+                Assert.That(data["Heartbeats.TrackInstancesDefault"], Is.EqualTo("Disabled"));
+                Assert.That(data["Heartbeats.TrackInstancesOverrides"], Is.EqualTo("1"));
+                Assert.That(int.Parse(data["Heartbeats.MonitoredInstances"]), Is.GreaterThanOrEqualTo(1));
+                Assert.That(int.Parse(data["Heartbeats.KnownInstances"]), Is.GreaterThanOrEqualTo(int.Parse(data["Heartbeats.MonitoredInstances"])));
+                Assert.That(data["Recoverability.Redirects"], Is.EqualTo("1"));
+                Assert.That(data["Licensing.ReportMasks"], Is.EqualTo("1"));
+                Assert.That(string.Join("|", data.Values), Does.Not.Contain(RetiredQueue).And.Not.Contain(ReplacementQueue).And.Not.Contain(MaskedWord).And.Not.Contain(ScalingOutEndpoint));
+            }
+        }
+
         static JsonDocument ReadReport(byte[] archive)
         {
             using var zip = new ZipArchive(new MemoryStream(archive), ZipArchiveMode.Read);
@@ -107,12 +216,23 @@ static JsonDocument ReadReport(byte[] archive)
         }
 
         const string SalesEndpoint = "Particular.Sales";
+        const string RetiredQueue = "Contoso.Billing.Legacy@machine1";
+        const string ReplacementQueue = "Contoso.Billing@machine2";
+        const string MaskedWord = "Contoso";
 
         class Context : ScenarioContext, ISequenceContext
         {
             public int Step { get; set; }
         }
 
+        static string ScalingOutEndpoint => Conventions.EndpointNamingConvention(typeof(ScalingOut));
+
+        class ScalingOut : EndpointConfigurationBuilder
+        {
+            public ScalingOut() =>
+                EndpointSetup<DefaultServerWithoutAudit>(c => c.SendHeartbeatTo(Settings.DEFAULT_INSTANCE_NAME));
+        }
+
         class MonitoringInstance : EndpointConfigurationBuilder
         {
             public MonitoringInstance() =>

src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDatabaseHostingProbe.cs

@@ -24,7 +24,7 @@ public async Task<DatabaseHosting> Probe(CancellationToken cancellationToken = d
             await using var command = dbContext.Database.GetDbConnection().CreateCommand();
             // ProductVersion rather than ProductMajorVersion: the latter is documented as SQL Server
             // only and comes back null on Azure SQL Database, Managed Instance and Synapse.
-            command.CommandText = "SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('ProductVersion')";
+            command.CommandText = "SELECT SERVERPROPERTY('EngineEdition'), SERVERPROPERTY('ProductVersion'), DB_ID('rdsadmin')";
             command.CommandTimeout = ProbeTimeoutSeconds;
 
             await dbContext.Database.OpenConnectionAsync(cancellationToken);
@@ -38,7 +38,9 @@ public async Task<DatabaseHosting> Probe(CancellationToken cancellationToken = d
 
             var engineEdition = Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture);
 
-            return new DatabaseHosting(HostingFor(engineEdition, ConfiguredHost), MajorVersion(reader), DatabaseHostingSource.Probe);
+            var rds = !reader.IsDBNull(2);
+
+            return new DatabaseHosting(HostingFor(engineEdition, rds, ConfiguredHost), MajorVersion(reader), DatabaseHostingSource.Probe);
         }
         catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
         {
@@ -59,12 +61,12 @@ public async Task<DatabaseHosting> Probe(CancellationToken cancellationToken = d
     /// Synapse and Fabric are not mapped: ServiceControl does not run on them, and an edition we do
     /// not recognise is not evidence of an ordinary SQL Server, so it falls to the host name.
     /// </summary>
-    internal static string HostingFor(int engineEdition, string? host) => engineEdition switch
+    internal static string HostingFor(int engineEdition, bool rds, string? host) => engineEdition switch
     {
         AzureSqlDatabase => "AzureSql",
         AzureSqlManagedInstance => "AzureSqlManagedInstance",
         AzureSqlEdge => "AzureSqlEdge",
-        PersonalOrDesktop or Standard or Enterprise or Express => ManagedOrSelfHosted(host),
+        PersonalOrDesktop or Standard or Enterprise or Express => rds ? "AwsRds" : ManagedOrSelfHosted(host),
         _ => DatabaseHostClassifier.Classify(host)
     };
 

src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs

@@ -7,11 +7,11 @@ namespace ServiceControl.Persistence.EFCore.Abstractions;
 public abstract class EFPersistenceConfigurationBase : PersistenceConfiguration, IPersistenceConfiguration
 {
     const string ConnectionStringKey = "Database/ConnectionString";
-    const string SchemaKey = "Database/Schema";
-    const string CommandTimeoutKey = "Database/CommandTimeout";
+    internal const string SchemaKey = "Database/Schema";
+    internal const string CommandTimeoutKey = "Database/CommandTimeout";
     const string BodyStorageTypeKey = "MessageBody/StorageType";
     const string FileSystemStoragePathKey = "MessageBody/FileSystem/StoragePath";
-    const string FileSystemDataSpaceRemainingThresholdKey = "MessageBody/FileSystem/DataSpaceRemainingThreshold";
+    internal const string FileSystemDataSpaceRemainingThresholdKey = "MessageBody/FileSystem/DataSpaceRemainingThreshold";
     const string AzureConnectionStringKey = "MessageBody/Azure/ConnectionString";
     const string AzureServiceUriKey = "MessageBody/Azure/ServiceUri";
     const string AzureManagedIdentityClientIdKey = "MessageBody/Azure/ManagedIdentityClientId";
@@ -23,11 +23,11 @@ public abstract class EFPersistenceConfigurationBase : PersistenceConfiguration,
     const string S3ServiceUrlKey = "MessageBody/S3/ServiceUrl";
     const string S3AccessKeyIdKey = "MessageBody/S3/AccessKeyId";
     const string S3SecretAccessKeyKey = "MessageBody/S3/SecretAccessKey";
-    const string MinBodySizeForCompressionKey = "MessageBody/MinCompressionSize";
+    internal const string MinBodySizeForCompressionKey = "MessageBody/MinCompressionSize";
     const string MaxBodySizeToStoreKey = "MaxBodySizeToStore";
     const string ErrorRetentionPeriodKey = "ErrorRetentionPeriod";
     const string EventsRetentionPeriodKey = "EventsRetentionPeriod";
-    const string SubscriptionCacheDurationKey = "SubscriptionCacheDuration";
+    internal const string SubscriptionCacheDurationKey = "SubscriptionCacheDuration";
     const string ExternalIntegrationsDispatchingBatchSizeKey = "ExternalIntegrationsDispatchingBatchSize";
 
     public bool SupportsMaintenanceMode => false;

src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs

@@ -4,6 +4,8 @@ namespace ServiceControl.Persistence.EFCore.Implementation;
 using Abstractions;
 using Infrastructure;
 using Particular.LicensingComponent.Contracts;
+using ServiceControl.Configuration;
+using ServiceControl.Infrastructure;
 using static Particular.LicensingComponent.Contracts.EnvironmentDatum;
 
 class EFEnvironmentDataProvider(EFPersisterSettings settings, IDatabaseHostingProbe hostingProbe) : IEnvironmentDataProvider
@@ -23,11 +25,28 @@ public IEnumerable<EnvironmentDatum> GetData()
             Deferred("Storage.HostingSource", async cancellationToken => (await Hosting(cancellationToken)).Source),
             Value("Storage.FullTextSearch", () => settings.EnableFullTextSearchOnBodies ? "Enabled" : "Disabled"),
             Value("Storage.BodyStorage.Type", () => BodyStorageType(settings.BodyStorage)),
-            Value("Storage.BodyStorage.Auth", () => BodyStorageAuth(settings.BodyStorage)),
-            Value("Limits.MaxBodySizeToStore", () => settings.BodyStorage.MaxBodySizeToStore.ToString(CultureInfo.InvariantCulture))
+            Value("Limits.MaxBodySizeToStore", () => settings.BodyStorage.MaxBodySizeToStore.ToString(CultureInfo.InvariantCulture)),
+            Value("Storage.Schema", () => settings.Schema is null ? "Default" : "Custom"),
+            Value("Storage.CommandTimeoutSeconds", () => WhenConfigured(EFPersistenceConfigurationBase.CommandTimeoutKey, () => Number(settings.CommandTimeout))),
+            Value("Storage.QueryTimeoutSeconds", () => WhenConfigured(QueryTimeLimit.SettingName, () => Seconds(settings.QueryTimeout))),
+            Value("Storage.SubscriptionCacheSeconds", () => WhenConfigured(EFPersistenceConfigurationBase.SubscriptionCacheDurationKey, () => Seconds(settings.SubscriptionCacheDuration))),
+            Value("Storage.BodyStorage.MinCompressionBytes", () => WhenConfigured(EFPersistenceConfigurationBase.MinBodySizeForCompressionKey, () => Number(settings.BodyStorage.MinCompressionSize))),
+            Value("Storage.FreeSpaceThresholdPercent", FreeSpaceThreshold)
         ];
     }
 
+    string FreeSpaceThreshold() => settings.BodyStorage is FileSystemBodyStorageSettings fileSystem
+        ? WhenConfigured(EFPersistenceConfigurationBase.FileSystemDataSpaceRemainingThresholdKey, () => Number(fileSystem.DataSpaceRemainingThreshold))
+        : "NotApplicable";
+
+    static string WhenConfigured(string key, Func<string> readValue) =>
+        SettingsReader.TryRead<string>(SettingsNamespace, key, out _) ? readValue() : "Default";
+
+    static string Number(int value) => value.ToString(CultureInfo.InvariantCulture);
+
+    static string Seconds(TimeSpan value) =>
+        Math.Round(value.TotalSeconds, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture);
+
     static string BodyStorageType(BodyStorageSettings bodyStorage) => bodyStorage switch
     {
         FileSystemBodyStorageSettings => nameof(Abstractions.BodyStorageType.FileSystem),
@@ -36,15 +55,5 @@ public IEnumerable<EnvironmentDatum> GetData()
         _ => "Unknown"
     };
 
-    static string BodyStorageAuth(BodyStorageSettings bodyStorage) => bodyStorage switch
-    {
-        AzureBlobBodyStorageSettings azureBlob => azureBlob.Authentication switch
-        {
-            AzureBlobManagedIdentityAuthentication => "ManagedIdentity",
-            AzureBlobSharedKeyAuthentication => "SharedKeyOrSas",
-            _ => "Unknown"
-        },
-        S3BodyStorageSettings s3 => s3.Credentials is null ? "IamRole" : "StaticCredentials",
-        _ => "NotApplicable"
-    };
+    static readonly SettingsRootNamespace SettingsNamespace = new("ServiceControl");
 }

src/ServiceControl.Persistence.RavenDB/RavenEnvironmentDataProvider.cs

@@ -2,10 +2,13 @@ namespace ServiceControl.Persistence.RavenDB;
 
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Threading;
 using System.Threading.Tasks;
 using Particular.LicensingComponent.Contracts;
 using Raven.Client.ServerWide.Operations;
+using ServiceControl.Configuration;
+using ServiceControl.Infrastructure;
 using static Particular.LicensingComponent.Contracts.EnvironmentDatum;
 
 class RavenEnvironmentDataProvider(RavenPersisterSettings settings, IRavenDocumentStoreProvider documentStoreProvider) : IEnvironmentDataProvider
@@ -19,7 +22,11 @@ public IEnumerable<EnvironmentDatum> GetData() =>
         Value("Storage.HostingSource", () => Hosting().Source),
         Value("Storage.FullTextSearch", () => settings.EnableFullTextSearchOnBodies ? "Enabled" : "Disabled"),
         Value("Storage.BodyStorage.Type", () => "RavenAttachments"),
-        Value("Storage.BodyStorage.Auth", () => "NotApplicable")
+        Value("Storage.LogLevel", () => settings.LogsMode),
+        Value("Storage.QueryTimeoutSeconds", () => WhenConfigured(QueryTimeLimit.SettingName, () => Number((int)Math.Round(settings.QueryTimeout.TotalSeconds, MidpointRounding.AwayFromZero)))),
+        Value("Storage.FreeSpaceThresholdPercent", () => WhenConfigured(RavenPersistenceConfiguration.DataSpaceRemainingThresholdKey, () => Number(settings.DataSpaceRemainingThreshold))),
+        Value("Storage.MinimumFreeSpaceForIngestionPercent", () => WhenConfigured(RavenBootstrapper.MinimumStorageLeftRequiredForIngestionKey, () => Number(settings.MinimumStorageLeftRequiredForIngestion))),
+        Value("Storage.ExpirationIntervalSeconds", () => WhenConfigured(RavenBootstrapper.ExpirationProcessTimerInSecondsKey, () => Number(settings.ExpirationProcessTimerInSeconds)))
     ];
 
     (string Hosting, string Source) Hosting()
@@ -42,4 +49,11 @@ async ValueTask<string> ServerVersion(CancellationToken cancellationToken)
 
         return buildNumber.ProductVersion ?? DatabaseHostClassifier.Unknown;
     }
+
+    static string WhenConfigured(string key, Func<string> readValue) =>
+        SettingsReader.TryRead<string>(SettingsNamespace, key, out _) ? readValue() : "Default";
+
+    static string Number(int value) => value.ToString(CultureInfo.InvariantCulture);
+
+    static readonly SettingsRootNamespace SettingsNamespace = new("ServiceControl");
 }