From 41df9e4ecfcc077a5eab2a2322615a597c4df2b3 Mon Sep 17 00:00:00 2001 From: Mohammed Firdous <124298708+mohammedfirdouss@users.noreply.github.com> Date: Thu, 7 May 2026 22:38:39 +0000 Subject: [PATCH 1/5] [Multi_K8s-Plugin] Prune orphaned resources on rollback Signed-off-by: Mohammed Firdous <124298708+mohammedfirdouss@users.noreply.github.com> --- .../deployment/misc.go | 33 ++++ .../deployment/misc_test.go | 153 ++++++++++++++++++ .../deployment/rollback.go | 14 +- 3 files changed, 198 insertions(+), 2 deletions(-) diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go index d73b587eb0..233d146c13 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go @@ -282,3 +282,36 @@ func deleteResources(ctx context.Context, lp sdk.StageLogPersister, applier *pro return deletedCount } + +// findOrphanedKeys returns the keys of resources present in targetManifests +// but absent from runningManifests. +func findOrphanedKeys(runningManifests, targetManifests []provider.Manifest) []provider.ResourceKey { + runningKeys := make(map[provider.ResourceKey]struct{}, len(runningManifests)) + for _, m := range runningManifests { + runningKeys[m.Key()] = struct{}{} + } + + orphans := make([]provider.ResourceKey, 0) + for _, m := range targetManifests { + if _, exists := runningKeys[m.Key()]; !exists { + orphans = append(orphans, m.Key()) + } + } + return orphans +} + +// pruneOrphanedResources deletes resources that exist in targetManifests but not in runningManifests. +// This handles the case where a new resource was applied during the failed deployment and must be +// removed during rollback to restore the cluster to the last known good state. +func pruneOrphanedResources(ctx context.Context, lp sdk.StageLogPersister, applier *provider.Applier, runningManifests, targetManifests []provider.Manifest) { + orphans := findOrphanedKeys(runningManifests, targetManifests) + + if len(orphans) == 0 { + lp.Info("No orphaned resources to prune") + return + } + + lp.Infof("Found %d orphaned resource(s) to prune", len(orphans)) + deleted := deleteResources(ctx, lp, applier, orphans) + lp.Successf("Successfully pruned %d orphaned resource(s)", deleted) +} diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go index c175608730..40b9d64ff8 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go @@ -23,6 +23,159 @@ import ( "github.com/pipe-cd/pipecd/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider" ) +func TestFindOrphanedKeys(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + runningManifests string + targetManifests string + wantCount int + wantNames []string + }{ + { + name: "no orphans when target equals running", + runningManifests: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: default +`, + targetManifests: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: default +`, + wantCount: 0, + wantNames: []string{}, + }, + { + name: "all target resources are orphaned when running is empty", + runningManifests: "", + targetManifests: ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: new-config + namespace: default +`, + wantCount: 1, + wantNames: []string{"new-config"}, + }, + { + name: "new resource in target not present in running is orphaned", + runningManifests: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: default +`, + targetManifests: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: default +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: new-config + namespace: default +`, + wantCount: 1, + wantNames: []string{"new-config"}, + }, + { + name: "resource only in running is not pruned", + runningManifests: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: default +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: old-config + namespace: default +`, + targetManifests: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: default +`, + wantCount: 0, + wantNames: []string{}, + }, + { + name: "multiple orphaned resources", + runningManifests: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: default +`, + targetManifests: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: default +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: new-config + namespace: default +--- +apiVersion: v1 +kind: Service +metadata: + name: new-svc + namespace: default +`, + wantCount: 2, + wantNames: []string{"new-config", "new-svc"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var running []provider.Manifest + if tt.runningManifests != "" { + running = mustParseManifests(t, tt.runningManifests) + } + var target []provider.Manifest + if tt.targetManifests != "" { + target = mustParseManifests(t, tt.targetManifests) + } + + orphans := findOrphanedKeys(running, target) + + assert.Len(t, orphans, tt.wantCount) + + gotNames := make([]string, 0, len(orphans)) + for _, k := range orphans { + gotNames = append(gotNames, k.Name()) + } + for _, name := range tt.wantNames { + assert.Contains(t, gotNames, name) + } + }) + } +} + func TestCheckVariantSelectorInWorkload(t *testing.T) { t.Parallel() diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go index 3bfc82d6f8..ec828f9b66 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go @@ -211,8 +211,18 @@ func (p *Plugin) rollback(ctx context.Context, input *sdk.ExecuteStageInput[kube failed = true } - // TODO: prune resources which don't exist in the running manifests but exist in the target manifests. - // This occurs when the user adds a new resource and the deployment pipeline fails. + lp.Info("Start pruning resources that do not exist in the running manifests") + targetCfg, err := input.Request.TargetDeploymentSource.AppConfig() + if err != nil { + lp.Infof("Failed to load target app config for pruning, skipping: %v", err) + } else { + targetManifests, err := p.loadManifests(ctx, &input.Request.Deployment, targetCfg.Spec, &input.Request.TargetDeploymentSource, provider.NewLoader(toolRegistry), input.Logger, multiTarget) + if err != nil { + lp.Infof("Failed to load target manifests for pruning, skipping: %v", err) + } else { + pruneOrphanedResources(ctx, lp, applier, manifests, targetManifests) + } + } if failed { return sdk.StageStatusFailure From 949a564e2e272537b4072af32a11eaee9e9f7f62 Mon Sep 17 00:00:00 2001 From: Mohammed Firdous <124298708+mohammedfirdouss@users.noreply.github.com> Date: Sun, 24 May 2026 09:36:37 +0000 Subject: [PATCH 2/5] fix(kubernetes_multicluster): use live resources for rollback pruning Signed-off-by: Mohammed Firdous <124298708+mohammedfirdouss@users.noreply.github.com> --- .../deployment/misc.go | 33 ---- .../deployment/misc_test.go | 152 ------------------ .../deployment/rollback.go | 24 ++- 3 files changed, 16 insertions(+), 193 deletions(-) diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go index 7d6bf84900..2780a515ca 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc.go @@ -311,36 +311,3 @@ func deleteResources(ctx context.Context, lp sdk.StageLogPersister, applier *pro return deletedCount, errors.Join(errs...) } - -// findOrphanedKeys returns the keys of resources present in targetManifests -// but absent from runningManifests. -func findOrphanedKeys(runningManifests, targetManifests []provider.Manifest) []provider.ResourceKey { - runningKeys := make(map[provider.ResourceKey]struct{}, len(runningManifests)) - for _, m := range runningManifests { - runningKeys[m.Key()] = struct{}{} - } - - orphans := make([]provider.ResourceKey, 0) - for _, m := range targetManifests { - if _, exists := runningKeys[m.Key()]; !exists { - orphans = append(orphans, m.Key()) - } - } - return orphans -} - -// pruneOrphanedResources deletes resources that exist in targetManifests but not in runningManifests. -// This handles the case where a new resource was applied during the failed deployment and must be -// removed during rollback to restore the cluster to the last known good state. -func pruneOrphanedResources(ctx context.Context, lp sdk.StageLogPersister, applier *provider.Applier, runningManifests, targetManifests []provider.Manifest) { - orphans := findOrphanedKeys(runningManifests, targetManifests) - - if len(orphans) == 0 { - lp.Info("No orphaned resources to prune") - return - } - - lp.Infof("Found %d orphaned resource(s) to prune", len(orphans)) - deleted := deleteResources(ctx, lp, applier, orphans) - lp.Successf("Successfully pruned %d orphaned resource(s)", deleted) -} diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go index 40b9d64ff8..9be24ddd02 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go @@ -23,158 +23,6 @@ import ( "github.com/pipe-cd/pipecd/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider" ) -func TestFindOrphanedKeys(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - runningManifests string - targetManifests string - wantCount int - wantNames []string - }{ - { - name: "no orphans when target equals running", - runningManifests: ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app - namespace: default -`, - targetManifests: ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app - namespace: default -`, - wantCount: 0, - wantNames: []string{}, - }, - { - name: "all target resources are orphaned when running is empty", - runningManifests: "", - targetManifests: ` -apiVersion: v1 -kind: ConfigMap -metadata: - name: new-config - namespace: default -`, - wantCount: 1, - wantNames: []string{"new-config"}, - }, - { - name: "new resource in target not present in running is orphaned", - runningManifests: ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app - namespace: default -`, - targetManifests: ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app - namespace: default ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: new-config - namespace: default -`, - wantCount: 1, - wantNames: []string{"new-config"}, - }, - { - name: "resource only in running is not pruned", - runningManifests: ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app - namespace: default ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: old-config - namespace: default -`, - targetManifests: ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app - namespace: default -`, - wantCount: 0, - wantNames: []string{}, - }, - { - name: "multiple orphaned resources", - runningManifests: ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app - namespace: default -`, - targetManifests: ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app - namespace: default ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: new-config - namespace: default ---- -apiVersion: v1 -kind: Service -metadata: - name: new-svc - namespace: default -`, - wantCount: 2, - wantNames: []string{"new-config", "new-svc"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - var running []provider.Manifest - if tt.runningManifests != "" { - running = mustParseManifests(t, tt.runningManifests) - } - var target []provider.Manifest - if tt.targetManifests != "" { - target = mustParseManifests(t, tt.targetManifests) - } - - orphans := findOrphanedKeys(running, target) - - assert.Len(t, orphans, tt.wantCount) - - gotNames := make([]string, 0, len(orphans)) - for _, k := range orphans { - gotNames = append(gotNames, k.Name()) - } - for _, name := range tt.wantNames { - assert.Contains(t, gotNames, name) - } - }) - } -} func TestCheckVariantSelectorInWorkload(t *testing.T) { t.Parallel() diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go index c681f97ec2..df3e88d0dd 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go @@ -210,16 +210,24 @@ func (p *Plugin) rollback(ctx context.Context, input *sdk.ExecuteStageInput[kube failed = true } - lp.Info("Start pruning resources that do not exist in the running manifests") - targetCfg, err := input.Request.TargetDeploymentSource.AppConfig() + lp.Info("Start finding and pruning resources that no longer exist in the running manifests") + namespacedLiveResources, clusterScopedLiveResources, err := provider.GetLiveResources(ctx, kubectl, deployTargetConfig.KubeConfigPath, input.Request.Deployment.ApplicationID) if err != nil { - lp.Infof("Failed to load target app config for pruning, skipping: %v", err) - } else { - targetManifests, err := p.loadManifests(ctx, &input.Request.Deployment, targetCfg.Spec, &input.Request.TargetDeploymentSource, provider.NewLoader(toolRegistry), input.Logger, multiTarget) - if err != nil { - lp.Infof("Failed to load target manifests for pruning, skipping: %v", err) + lp.Errorf("Failed while getting live resources (%v)", err) + failed = true + } else if len(namespacedLiveResources)+len(clusterScopedLiveResources) > 0 { + removeKeys := provider.FindRemoveResources(manifests, namespacedLiveResources, clusterScopedLiveResources) + if len(removeKeys) == 0 { + lp.Info("There are no live resources to prune") } else { - pruneOrphanedResources(ctx, lp, applier, manifests, targetManifests) + lp.Infof("Start pruning %d resources", len(removeKeys)) + deletedCount, err := deleteResources(ctx, lp, applier, removeKeys) + if err != nil { + lp.Errorf("Failed to delete some resources, %d/%d resources were deleted (%v)", deletedCount, len(removeKeys), err) + failed = true + } else { + lp.Successf("Successfully deleted %d resources", deletedCount) + } } } From b48240711f882533cad2a35ae9da94a2c86c8cfa Mon Sep 17 00:00:00 2001 From: Mohammed Firdous <124298708+mohammedfirdouss@users.noreply.github.com> Date: Tue, 26 May 2026 01:04:16 +0000 Subject: [PATCH 3/5] fix(kubernetes_multicluster): fix gofmt formatting in misc_test.go Signed-off-by: Mohammed Firdous <124298708+mohammedfirdouss@users.noreply.github.com> --- .../plugin/kubernetes_multicluster/deployment/misc_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go index f8da2fa3d3..c3971e491e 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/misc_test.go @@ -26,7 +26,6 @@ import ( "github.com/pipe-cd/pipecd/pkg/app/pipedv1/plugin/kubernetes_multicluster/provider" ) - func TestCheckVariantSelectorInWorkload(t *testing.T) { t.Parallel() From 63a833f93f060ef61e2e6d5de9a56b2280fdd7d0 Mon Sep 17 00:00:00 2001 From: Mohammed Firdous <124298708+mohammedfirdouss@users.noreply.github.com> Date: Tue, 26 May 2026 22:22:10 +0000 Subject: [PATCH 4/5] fix(kubernetes_multicluster): log pruning info when no live resources are found Signed-off-by: Mohammed Firdous <124298708+mohammedfirdouss@users.noreply.github.com> --- .../plugin/kubernetes_multicluster/deployment/rollback.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go index df3e88d0dd..35031b58ef 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go @@ -229,6 +229,8 @@ func (p *Plugin) rollback(ctx context.Context, input *sdk.ExecuteStageInput[kube lp.Successf("Successfully deleted %d resources", deletedCount) } } + } else { + lp.Info("There are no live resources to prune") } if failed { From b4b3f95d8c6d1d83cba97df1b9606fd404398a13 Mon Sep 17 00:00:00 2001 From: Mohammed Firdous <124298708+mohammedfirdouss@users.noreply.github.com> Date: Tue, 26 May 2026 22:25:36 +0000 Subject: [PATCH 5/5] fix(kubernetes_multicluster): log pruning info when no live resources are found Signed-off-by: Mohammed Firdous <124298708+mohammedfirdouss@users.noreply.github.com> --- .../plugin/kubernetes_multicluster/deployment/rollback.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go index 35031b58ef..bc2faf6aef 100644 --- a/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go +++ b/pkg/app/pipedv1/plugin/kubernetes_multicluster/deployment/rollback.go @@ -212,10 +212,11 @@ func (p *Plugin) rollback(ctx context.Context, input *sdk.ExecuteStageInput[kube lp.Info("Start finding and pruning resources that no longer exist in the running manifests") namespacedLiveResources, clusterScopedLiveResources, err := provider.GetLiveResources(ctx, kubectl, deployTargetConfig.KubeConfigPath, input.Request.Deployment.ApplicationID) - if err != nil { + switch { + case err != nil: lp.Errorf("Failed while getting live resources (%v)", err) failed = true - } else if len(namespacedLiveResources)+len(clusterScopedLiveResources) > 0 { + case len(namespacedLiveResources)+len(clusterScopedLiveResources) > 0: removeKeys := provider.FindRemoveResources(manifests, namespacedLiveResources, clusterScopedLiveResources) if len(removeKeys) == 0 { lp.Info("There are no live resources to prune") @@ -229,7 +230,7 @@ func (p *Plugin) rollback(ctx context.Context, input *sdk.ExecuteStageInput[kube lp.Successf("Successfully deleted %d resources", deletedCount) } } - } else { + default: lp.Info("There are no live resources to prune") }