use sdk pagination for accurate stack status instrumentation (#15)

Author: xh3b4sd

pkg/worker/handler/stack/detail.go

@@ -2,97 +2,68 @@ package stack
 
 import (
 	"context"
-	"regexp"
 
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi"
-	tagtypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types"
+	"github.com/aws/aws-sdk-go-v2/service/cloudformation"
+	"github.com/aws/aws-sdk-go-v2/service/cloudformation/types"
 	"github.com/xh3b4sd/tracer"
 )
 
-var (
-	// exp defines a regular expression for the ARN suffixes of nested
-	// CloudFormation stacks. Note that there is no precise definition for the
-	// amount of characters that a random hack suffix for the stack names ought to
-	// look like. So we are testing against at least 10 characters, but also
-	// accept e.g. 13 or 14.
-	//
-	//     arn:aws:cloudformation:us-west-2:995626699990:stack/server-test-FargateStack-QGXQ9XZ4J44K/165deb30-30d0-11f0-9eeb-023c6b26cb57
-	//
-	exp = regexp.MustCompile(`-[A-Z0-9]{10,}/[a-f0-9-]{36}$`)
-)
-
 type detail struct {
-	// arn is the well defined Amazon Resource Name of the CloudFormation stack.
-	arn string
+	// nam is the name of the CloudFormation stack.
+	nam string
+	// sta is the raw CloudFormation stack status, e.g. UPDATE_COMPLETE.
+	sta types.StackStatus
 }
 
-// detail finds all CloudFormation stack ARNs that are tagged with the
+// detail finds all CloudFormation stack statuses that are tagged with the
 // "environment" that matches Specta's runtime configuration. In other words, if
 // Specta is running in "staging", then detail() will find all CloudFormation
 // stacks labelled with the resource tags environment=staging.
 func (h *Handler) detail() ([]detail, error) {
 	var err error
 
-	var inp *resourcegroupstaggingapi.GetResourcesInput
+	// Create a new paginator for each instrumentation cycle in order to guarantee
+	// data integrity per loop.
+	var pag *cloudformation.DescribeStacksPaginator
 	{
-		inp = &resourcegroupstaggingapi.GetResourcesInput{
-			ResourceTypeFilters: []string{"cloudformation:stack"},
-			TagFilters: []tagtypes.TagFilter{
-				{
-					Key:    aws.String("environment"),
-					Values: []string{h.env.Environment},
-				},
-			},
-		}
-	}
-
-	var out *resourcegroupstaggingapi.GetResourcesOutput
-	{
-		out, err = h.tag.GetResources(context.Background(), inp)
-		if err != nil {
-			return nil, tracer.Mask(err)
-		}
+		pag = cloudformation.NewDescribeStacksPaginator(h.cfc, &cloudformation.DescribeStacksInput{})
 	}
 
 	var det []detail
-	for _, x := range out.ResourceTagMappingList {
-		var roo bool
+	for pag.HasMorePages() {
+		var out *cloudformation.DescribeStacksOutput
 		{
-			roo = rooSta(*x.ResourceARN)
+			out, err = pag.NextPage(context.Background())
+			if err != nil {
+				return nil, tracer.Mask(err)
+			}
 		}
 
-		if !roo {
-			continue
-		}
+		for _, x := range out.Stacks {
+			if !hasEnv(x.Tags, h.env.Environment) {
+				continue
+			}
 
-		det = append(det, detail{
-			arn: *x.ResourceARN,
-		})
+			det = append(det, detail{
+				nam: *x.StackName,
+				sta: x.StackStatus,
+			})
+		}
 	}
 
 	if len(det) == 0 {
-		return nil, tracer.Maskf(missingRootStackError, "%v", allArn(out.ResourceTagMappingList))
-	}
-	if len(det) > 1 {
-		return nil, tracer.Maskf(tooManyRootStacksError, "%v", det)
+		return nil, tracer.Maskf(missingRootStackError, "%s", h.env.Environment)
 	}
 
 	return det, nil
 }
 
-func allArn(lis []tagtypes.ResourceTagMapping) []string {
-	var all []string
-
-	for _, x := range lis {
-		all = append(all, *x.ResourceARN)
+func hasEnv(tags []types.Tag, env string) bool {
+	for _, t := range tags {
+		if t.Key != nil && t.Value != nil && *t.Key == "environment" && *t.Value == env {
+			return true
+		}
 	}
 
-	return all
-}
-
-// rooSta identifies whether the given CloudFormation stack ARN is considered a
-// root stack ARN.
-func rooSta(arn string) bool {
-	return !exp.MatchString(arn)
+	return false
 }

pkg/worker/handler/stack/detail_test.go

@@ -6,10 +6,5 @@ import (
 
 var missingRootStackError = &tracer.Error{
 	Kind: "missingRootStackError",
-	Desc: "The exporter expected to find exactly one root stack, but no stack name matched against the internal mapping.",
-}
-
-var tooManyRootStacksError = &tracer.Error{
-	Kind: "tooManyRootStacksError",
-	Desc: "The exporter expected to find exactly one root stack, but more than one stack names matched against the internal mapping.",
+	Desc: "The exporter expected to find exactly one root stack, but no stack was found for teh given environment.",
 }

pkg/worker/handler/stack/error.go

@@ -1,47 +1,51 @@
 package stack
 
 import (
-	"context"
+	"regexp"
 	"strings"
 
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/service/cloudformation"
 	"github.com/aws/aws-sdk-go-v2/service/cloudformation/types"
-	"github.com/xh3b4sd/tracer"
 )
 
-var status = map[types.ResourceStatus]float64{
-	types.ResourceStatusCreateFailed:             0.0,
-	types.ResourceStatusDeleteFailed:             0.0,
-	types.ResourceStatusDeleteSkipped:            0.0,
-	types.ResourceStatusExportFailed:             0.0,
-	types.ResourceStatusExportRollbackComplete:   0.0,
-	types.ResourceStatusExportRollbackFailed:     0.0,
-	types.ResourceStatusExportRollbackInProgress: 0.0,
-	types.ResourceStatusImportFailed:             0.0,
-	types.ResourceStatusImportRollbackComplete:   0.0,
-	types.ResourceStatusImportRollbackFailed:     0.0,
-	types.ResourceStatusImportRollbackInProgress: 0.0,
-	types.ResourceStatusRollbackComplete:         0.0,
-	types.ResourceStatusRollbackFailed:           0.0,
-	types.ResourceStatusRollbackInProgress:       0.0,
-	types.ResourceStatusUpdateFailed:             0.0,
-	types.ResourceStatusUpdateRollbackComplete:   0.0,
-	types.ResourceStatusUpdateRollbackFailed:     0.0,
-	types.ResourceStatusUpdateRollbackInProgress: 0.0,
-
-	types.ResourceStatusCreateInProgress: 0.5,
-	types.ResourceStatusDeleteInProgress: 0.5,
-	types.ResourceStatusExportInProgress: 0.5,
-	types.ResourceStatusImportInProgress: 0.5,
-	types.ResourceStatusUpdateInProgress: 0.5,
-
-	types.ResourceStatusCreateComplete: 1.0,
-	types.ResourceStatusDeleteComplete: 1.0,
-	types.ResourceStatusExportComplete: 1.0,
-	types.ResourceStatusImportComplete: 1.0,
-	types.ResourceStatusUpdateComplete: 1.0,
-}
+var (
+	// exp defines a regular expression for the stack names of CloudFormation
+	// stacks. Note that there is no precise definition for the amount of
+	// characters that a random hash suffix for the stack names ought to look
+	// like. So we are testing against at least 10 characters, but also accept
+	// e.g. 13 or 14.
+	//
+	//     server-test-FargateStack-QGXQ9XZ4J44K
+	//
+	exp = regexp.MustCompile(`-[A-Z0-9]{10,}$`)
+
+	status = map[types.StackStatus]float64{
+		types.StackStatusCreateFailed:                            0.0,
+		types.StackStatusDeleteFailed:                            0.0,
+		types.StackStatusImportRollbackComplete:                  0.0,
+		types.StackStatusImportRollbackFailed:                    0.0,
+		types.StackStatusImportRollbackInProgress:                0.0,
+		types.StackStatusRollbackComplete:                        0.0,
+		types.StackStatusRollbackFailed:                          0.0,
+		types.StackStatusRollbackInProgress:                      0.0,
+		types.StackStatusUpdateFailed:                            0.0,
+		types.StackStatusUpdateRollbackComplete:                  0.0,
+		types.StackStatusUpdateRollbackCompleteCleanupInProgress: 0.0,
+		types.StackStatusUpdateRollbackFailed:                    0.0,
+		types.StackStatusUpdateRollbackInProgress:                0.0,
+
+		types.StackStatusCreateInProgress:                0.5,
+		types.StackStatusDeleteInProgress:                0.5,
+		types.StackStatusImportInProgress:                0.5,
+		types.StackStatusReviewInProgress:                0.5,
+		types.StackStatusUpdateCompleteCleanupInProgress: 0.5,
+		types.StackStatusUpdateInProgress:                0.5,
+
+		types.StackStatusCreateComplete: 1.0,
+		types.StackStatusDeleteComplete: 1.0,
+		types.StackStatusImportComplete: 1.0,
+		types.StackStatusUpdateComplete: 1.0,
+	}
+)
 
 type stack struct {
 	// hlt is the CloudFormation stack status, either 0.0, 0.5 or 1.0.
@@ -57,65 +61,57 @@ type stack struct {
 // assigned the stack status 0.5, otherwise the unhealthy stack status 0 is
 // assigned.
 func (h *Handler) stack(det []detail) ([]stack, error) {
-	var err error
-
 	var sta []stack
+
 	for _, x := range det {
-		var inp *cloudformation.DescribeStackResourcesInput
+		var tag string
 		{
-			inp = &cloudformation.DescribeStackResourcesInput{
-				StackName: aws.String(x.arn),
-			}
+			tag = staTag(x.nam)
 		}
 
-		var out *cloudformation.DescribeStackResourcesOutput
-		{
-			out, err = h.cfc.DescribeStackResources(context.Background(), inp)
-			if err != nil {
-				return nil, tracer.Mask(err)
-			}
-		}
+		if tag == "" {
+			h.log.Log(
+				"level", "warning",
+				"message", "skipping instrumentation for CloudFormation stack",
+				"reason", "CloudFormation stack name pattern is unrecognizable",
+				"name", x.nam,
+			)
 
-		for _, y := range out.StackResources {
-			var tag string
 			{
-				tag = staTag(*y.LogicalResourceId)
-			}
-
-			if tag == "" {
-				h.log.Log(
-					"level", "warning",
-					"message", "skipping instrumentation for CloudFormation stack",
-					"reason", "CloudFormation stack name pattern is unrecognizable",
-					"name", *y.LogicalResourceId,
-				)
-
-				{
-					continue
-				}
-			}
-
-			var hlt float64
-			{
-				hlt = status[y.ResourceStatus]
+				continue
 			}
+		}
 
-			sta = append(sta, stack{
-				hlt: hlt,
-				lab: tag,
-			})
+		var hlt float64
+		{
+			hlt = status[x.sta]
 		}
+
+		sta = append(sta, stack{
+			hlt: hlt,
+			lab: tag,
+		})
 	}
 
 	return sta, nil
 }
 
+// rooSta identifies whether the given CloudFormation stack name is considered a
+// root stack name.
+func rooSta(arn string) bool {
+	return !exp.MatchString(arn)
+}
+
 func staTag(nam string) string {
 	for k, v := range mapping {
 		if strings.Contains(nam, k) {
 			return v
 		}
 	}
 
+	if rooSta(nam) {
+		return "root"
+	}
+
 	return ""
 }