Hystrix tests updates (#98)

* Reset more things in more places when performing resets in Hystrix
* Limit dotnet test to a single CPU, un-skip the "FlakyOnHostedAgents" test category
* Make tests less sensitive to available CPU resources

Author: dtillman

azure-pipelines.yml

@@ -47,7 +47,7 @@ jobs:
     inputs:
       command: test
       projects: '**/*.Test/*.csproj'
-      arguments: '-c $(buildConfiguration) --filter "Category!=SkipOnLinux&Category!=FlakyOnHostedAgents"'
+      arguments: '-c $(buildConfiguration) -maxcpucount:1 --filter "Category!=SkipOnLinux"'
 - job: MacOS_Build_and_Test
   pool:
     vmImage: 'macOS-10.14'
@@ -76,7 +76,7 @@ jobs:
     inputs:
       command: test
       projects: '**/*.Test/*.csproj'
-      arguments: '-c $(buildConfiguration) --filter "Category!=SkipOnMacOS&Category!=FlakyOnHostedAgents"'
+      arguments: '-c $(buildConfiguration) -maxcpucount:1 --filter "Category!=SkipOnMacOS"'
 - job: Windows_Build_Test_and_Package
   pool:
     vmImage: 'windows-2019'
@@ -130,7 +130,8 @@ jobs:
     inputs:
       command: test
       projects: 'src/Steeltoe.All.sln'
-      arguments: '-c $(buildConfiguration) /p:TreatWarningsAsErrors=True /p:CopyLocalLockFileAssemblies=true --filter "Category!=Integration&Category!=FlakyOnHostedAgents" /p:CollectCoverage=true /p:CoverletOutputFormat="opencover" /p:Include="[Steeltoe.*]*" /p:Exclude="[*.Test]*"'
+#      arguments: '-c $(buildConfiguration) /p:TreatWarningsAsErrors=True /p:CopyLocalLockFileAssemblies=true --filter "Category!=Integration&Category!=FlakyOnHostedAgents" /p:CollectCoverage=true /p:CoverletOutputFormat="opencover" /p:Include="[Steeltoe.*]*" /p:Exclude="[*.Test]*"'
+      arguments: '-c $(buildConfiguration) -maxcpucount:1 /p:TreatWarningsAsErrors=True /p:CopyLocalLockFileAssemblies=true --filter "Category!=Integration" /p:CollectCoverage=true /p:CoverletOutputFormat="opencover" /p:Include="[Steeltoe.*]*" /p:Exclude="[*.Test]*"'
   # Generate the report using ReportGenerator (https://github.com/danielpalme/ReportGenerator)
   # First install the tool on the machine, then run it
   - pwsh: |
@@ -174,4 +175,4 @@ jobs:
       PathtoPublish: $(Build.ArtifactStagingDirectory)
       ArtifactName: Packages
       publishLocation: Container
-    condition: always()
+    condition: always()

src/CircuitBreaker/src/HystrixBase/HystrixCollapserMetrics.cs

@@ -67,6 +67,9 @@ public static ICollection<HystrixCollapserMetrics> GetInstances()
 
         internal static void Reset()
         {
+            RollingCollapserEventCounterStream.Reset();
+            CumulativeCollapserEventCounterStream.Reset();
+            RollingCollapserBatchSizeDistributionStream.Reset();
             Metrics.Clear();
         }
 

src/CircuitBreaker/src/HystrixBase/HystrixCommandMetrics.cs

@@ -117,6 +117,14 @@ internal static void Reset()
                 metricsInstance.UnsubscribeAll();
             }
 
+            RollingCommandEventCounterStream.Reset();
+            CumulativeCommandEventCounterStream.Reset();
+            RollingCommandLatencyDistributionStream.Reset();
+            RollingCommandUserLatencyDistributionStream.Reset();
+            RollingCommandMaxConcurrencyStream.Reset();
+            HystrixThreadEventStream.Reset();
+            HealthCountsStream.Reset();
+
             Metrics.Clear();
         }
 

src/CircuitBreaker/src/HystrixBase/HystrixThreadPoolMetrics.cs

@@ -90,6 +90,10 @@ private static bool HasExecutedCommandsOnThread(HystrixThreadPoolMetrics threadP
 
         internal static void Reset()
         {
+            RollingThreadPoolEventCounterStream.Reset();
+            CumulativeThreadPoolEventCounterStream.Reset();
+            RollingThreadPoolMaxConcurrencyStream.Reset();
+
             Metrics.Clear();
         }
 

src/CircuitBreaker/src/HystrixBase/Metric/CachedValuesHistogram.cs

@@ -150,5 +150,10 @@ public long GetTotalCount()
         {
             return totalCount;
         }
+
+        public override string ToString()
+        {
+            return "[Mean: " + GetMean() + "/Total: " + GetTotalCount() + "]";
+        }
     }
 }

src/CircuitBreaker/src/HystrixBase/Metric/Consumer/BucketedCounterStream.cs

@@ -16,6 +16,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Reactive.Concurrency;
 using System.Reactive.Linq;
 
 namespace Steeltoe.CircuitBreaker.Hystrix.Metric.Consumer
@@ -50,7 +51,7 @@ protected BucketedCounterStream(IHystrixEventStream<Event> inputEventStream, int
             {
                 return inputEventStream
                     .Observe()
-                    .Window(TimeSpan.FromMilliseconds(bucketSizeInMs)) // bucket it by the counter window so we can emit to the next operator in time chunks, not on every OnNext
+                    .Window(TimeSpan.FromMilliseconds(bucketSizeInMs), NewThreadScheduler.Default) // bucket it by the counter window so we can emit to the next operator in time chunks, not on every OnNext
                     .SelectMany((b) =>
                     {
                         return reduceBucketToSummary(b);

src/CircuitBreaker/src/HystrixBase/Metric/Consumer/BucketedCumulativeCounterStream.cs

@@ -68,7 +68,6 @@ public Output Latest
         {
             get
             {
-                StartCachingStreamValuesIfUnstarted();
                 if (counterSubject.TryGetValue(out Output v))
                 {
                     return v;

src/CircuitBreaker/src/HystrixBase/Metric/Consumer/BucketedRollingCounterStream.cs

@@ -90,7 +90,6 @@ public Output Latest
         {
             get
             {
-                StartCachingStreamValuesIfUnstarted();
                 if (counterSubject.TryGetValue(out Output v))
                 {
                     return v;

src/CircuitBreaker/src/HystrixBase/Metric/Consumer/CumulativeCollapserEventCounterStream.cs

@@ -35,11 +35,22 @@ public static CumulativeCollapserEventCounterStream GetInstance(IHystrixCollapse
 
         public static CumulativeCollapserEventCounterStream GetInstance(IHystrixCollapserKey collapserKey, int numBuckets, int bucketSizeInMs)
         {
-            return Streams.GetOrAddEx(collapserKey.Name, (k) => new CumulativeCollapserEventCounterStream(collapserKey, numBuckets, bucketSizeInMs, HystrixCollapserMetrics.AppendEventToBucket, HystrixCollapserMetrics.BucketAggregator));
+            return Streams.GetOrAddEx(collapserKey.Name, (k) =>
+            {
+                var stream = new CumulativeCollapserEventCounterStream(collapserKey, numBuckets, bucketSizeInMs, HystrixCollapserMetrics.AppendEventToBucket, HystrixCollapserMetrics.BucketAggregator);
+                stream.StartCachingStreamValuesIfUnstarted();
+                return stream;
+            });
         }
 
         public static void Reset()
         {
+            foreach (var stream in Streams.Values)
+            {
+                stream.Unsubscribe();
+            }
+
+            HystrixCollapserEventStream.Reset();
             Streams.Clear();
         }
 

src/CircuitBreaker/src/HystrixBase/Metric/Consumer/CumulativeCommandEventCounterStream.cs

@@ -35,12 +35,24 @@ public static CumulativeCommandEventCounterStream GetInstance(IHystrixCommandKey
 
         public static CumulativeCommandEventCounterStream GetInstance(IHystrixCommandKey commandKey, int numBuckets, int bucketSizeInMs)
         {
-            var result = Streams.GetOrAddEx(commandKey.Name, (k) => new CumulativeCommandEventCounterStream(commandKey, numBuckets, bucketSizeInMs, HystrixCommandMetrics.AppendEventToBucket, HystrixCommandMetrics.BucketAggregator));
+            var result = Streams.GetOrAddEx(commandKey.Name, (k) =>
+            {
+                var stream = new CumulativeCommandEventCounterStream(commandKey, numBuckets, bucketSizeInMs, HystrixCommandMetrics.AppendEventToBucket, HystrixCommandMetrics.BucketAggregator);
+                stream.StartCachingStreamValuesIfUnstarted();
+                return stream;
+            });
             return result;
         }
 
         public static void Reset()
         {
+            foreach (var stream in Streams.Values)
+            {
+                stream.Unsubscribe();
+            }
+
+            HystrixCommandCompletionStream.Reset();
+
             Streams.Clear();
         }