Skip to content

Commit 78725dc

Browse files
shivaprasadmbshivaprasadmb
committed
fixes 2
1 parent de83b74 commit 78725dc

1 file changed

Lines changed: 69 additions & 29 deletions

File tree

‎automation/util/test_suite_manager_common.go‎

Lines changed: 69 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,17 @@ func provisionOrClone(ctx context.Context, st *SetupTypes, clientset *kubernetes
6666
st.DbSecret.StringData[common.SECRET_DATA_KEY_PASSWORD] = os.Getenv(automation.DB_SECRET_PASSWORD_ENV)
6767
_, err = clientset.CoreV1().Secrets(ns).Create(ctx, st.DbSecret, metav1.CreateOptions{})
6868
if err != nil {
69-
logger.Printf("Error while creating db secret %s: %s\n", st.DbSecret.Name, err)
69+
if k8serrors.IsAlreadyExists(err) {
70+
logger.Printf("DB Secret %s already exists, updating.\n", st.DbSecret.Name)
71+
_, err = clientset.CoreV1().Secrets(ns).Update(ctx, st.DbSecret, metav1.UpdateOptions{})
72+
if err != nil {
73+
logger.Printf("Error while updating db secret %s: %s\n", st.DbSecret.Name, err)
74+
} else {
75+
logger.Printf("DB Secret %s updated.\n", st.DbSecret.Name)
76+
}
77+
} else {
78+
logger.Printf("Error while creating db secret %s: %s\n", st.DbSecret.Name, err)
79+
}
7080
} else {
7181
logger.Printf("DB Secret %s created.\n", st.DbSecret.Name)
7282
}
@@ -80,7 +90,17 @@ func provisionOrClone(ctx context.Context, st *SetupTypes, clientset *kubernetes
8090
st.NdbSecret.StringData[common.SECRET_DATA_KEY_PASSWORD] = os.Getenv(automation.NDB_SECRET_PASSWORD_ENV)
8191
_, err = clientset.CoreV1().Secrets(ndbCredsNs).Create(context.TODO(), st.NdbSecret, metav1.CreateOptions{})
8292
if err != nil {
83-
logger.Printf("Error while creating ndb secret %s: %s\n", st.NdbSecret.Name, err)
93+
if k8serrors.IsAlreadyExists(err) {
94+
logger.Printf("NDB Secret %s already exists, updating.\n", st.NdbSecret.Name)
95+
_, err = clientset.CoreV1().Secrets(ndbCredsNs).Update(context.TODO(), st.NdbSecret, metav1.UpdateOptions{})
96+
if err != nil {
97+
logger.Printf("Error while updating ndb secret %s: %s\n", st.NdbSecret.Name, err)
98+
} else {
99+
logger.Printf("NDB Secret %s updated in namespace %s.\n", st.NdbSecret.Name, ndbCredsNs)
100+
}
101+
} else {
102+
logger.Printf("Error while creating ndb secret %s: %s\n", st.NdbSecret.Name, err)
103+
}
84104
} else {
85105
logger.Printf("NDB Secret %s created in namespace %s.\n", st.NdbSecret.Name, ndbCredsNs)
86106
}
@@ -96,9 +116,20 @@ func provisionOrClone(ctx context.Context, st *SetupTypes, clientset *kubernetes
96116
st.NdbServer.Spec.CredentialSecretRef.Name = st.NdbSecret.Name
97117
}
98118
st.NdbServer.Namespace = "" // cluster-scoped has no namespace
119+
ndbServerName := st.NdbServer.Name
99120
st.NdbServer, err = v1alpha1ClientSet.NDBServers().Create(st.NdbServer)
100121
if err != nil {
101-
logger.Printf("Error while creating NDBServer %s: %s\n", st.NdbServer.Name, err)
122+
if k8serrors.IsAlreadyExists(err) {
123+
logger.Printf("NDBServer %s already exists, fetching existing.\n", ndbServerName)
124+
st.NdbServer, err = v1alpha1ClientSet.NDBServers().Get(ndbServerName, metav1.GetOptions{})
125+
if err != nil {
126+
logger.Printf("Error while fetching existing NDBServer %s: %s\n", ndbServerName, err)
127+
} else {
128+
logger.Printf("NDBServer %s fetched successfully.\n", st.NdbServer.Name)
129+
}
130+
} else {
131+
logger.Printf("Error while creating NDBServer %s: %s\n", ndbServerName, err)
132+
}
102133
} else {
103134
logger.Printf("NDBServer %s created.\n", st.NdbServer.Name)
104135
}
@@ -142,9 +173,21 @@ func provisionOrClone(ctx context.Context, st *SetupTypes, clientset *kubernetes
142173
st.Database.Spec.Instance.ClusterId = nxClusterId
143174
}
144175
}
145-
st.Database, err = v1alpha1ClientSet.Databases(st.Database.Namespace).Create(st.Database)
176+
databaseName := st.Database.Name
177+
databaseNamespace := st.Database.Namespace
178+
st.Database, err = v1alpha1ClientSet.Databases(databaseNamespace).Create(st.Database)
146179
if err != nil {
147-
logger.Printf("Error while creating Database %s: %s\n", st.Database.Name, err)
180+
if k8serrors.IsAlreadyExists(err) {
181+
logger.Printf("Database %s already exists, fetching existing.\n", databaseName)
182+
st.Database, err = v1alpha1ClientSet.Databases(databaseNamespace).Get(databaseName, metav1.GetOptions{})
183+
if err != nil {
184+
logger.Printf("Error while fetching existing Database %s: %s\n", databaseName, err)
185+
} else {
186+
logger.Printf("Database %s fetched successfully.\n", st.Database.Name)
187+
}
188+
} else {
189+
logger.Printf("Error while creating Database %s: %s\n", databaseName, err)
190+
}
148191
} else {
149192
logger.Printf("Database %s created.\n", st.Database.Name)
150193
}
@@ -387,48 +430,45 @@ func getDatabaseOrCloneResponse(ctx context.Context, st *SetupTypes, clientset *
387430
func getAppResponse(ctx context.Context, st *SetupTypes, clientset *kubernetes.Clientset, localPort string) (res http.Response, err error) {
388431
logger := GetLogger(ctx)
389432
logger.Println("getAppResponse() started...")
390-
errBaseMsg := "getAppResponse() ended"
391433

392434
// Retrieve the pod name and targetPort
393435
podName := st.AppPod.Name
394436
podTargetPort := st.AppPod.Spec.Containers[0].Ports[0].ContainerPort
395437

396-
// Run port-forward command using kubectl
397-
cmd := exec.Command("kubectl", "port-forward", podName, fmt.Sprintf("%s:%d", localPort, podTargetPort))
398-
err = cmd.Start()
399-
if err != nil {
400-
return http.Response{}, fmt.Errorf("%s! kubectl port-forward %s %s:%d failed! %v. ", errBaseMsg, podName, localPort, podTargetPort, err)
401-
} else {
402-
logger.Printf("kubectl port-forward %s %s:%d started.", podName, localPort, podTargetPort)
403-
}
404-
405-
// Wait for port-forwarding to establish
406-
// Increased from 2s to 10s to ensure port-forward fully establishes before HTTP attempts
407-
logger.Println("Waiting 10 seconds for port-forward to establish...")
408-
time.Sleep(10 * time.Second)
409-
410-
// Verify the forwarded port by making an HTTP request with retry logic
411438
url := fmt.Sprintf("http://localhost:%s", localPort)
412439
maxRetries := 10
413440
retryDelay := 2 * time.Second
414441
var resp *http.Response
415442

416443
for i := 0; i < maxRetries; i++ {
444+
// Start a fresh port-forward on every attempt to handle cases where
445+
// the previous port-forward process died (e.g. after an EOF)
446+
cmd := exec.Command("kubectl", "port-forward", podName, fmt.Sprintf("%s:%d", localPort, podTargetPort))
447+
if startErr := cmd.Start(); startErr != nil {
448+
logger.Printf("Attempt %d: kubectl port-forward failed to start: %v", i+1, startErr)
449+
time.Sleep(retryDelay)
450+
continue
451+
}
452+
logger.Printf("Attempt %d: kubectl port-forward %s %s:%d started.", i+1, podName, localPort, podTargetPort)
453+
454+
// Give port-forward time to establish before making the HTTP request
455+
time.Sleep(retryDelay)
456+
417457
resp, err = http.Get(url)
418458
if err == nil && resp.StatusCode == 200 {
419459
logger.Printf("http://localhost:%s successful on attempt %d.", localPort, i+1)
460+
cmd.Process.Kill()
420461
break
421462
}
422-
if i < maxRetries-1 {
423-
logger.Printf("Attempt %d failed (err: %v), retrying in %v...", i+1, err, retryDelay)
424-
if resp != nil {
425-
resp.Body.Close()
426-
}
427-
time.Sleep(retryDelay)
463+
464+
logger.Printf("Attempt %d failed (err: %v), retrying...", i+1, err)
465+
if resp != nil {
466+
resp.Body.Close()
428467
}
468+
cmd.Process.Kill()
429469
}
430470

431-
if err != nil || resp.StatusCode != 200 {
471+
if err != nil || resp == nil || resp.StatusCode != 200 {
432472
return http.Response{}, fmt.Errorf("http://localhost:%s failed after %d attempts! Last error: %v", localPort, maxRetries, err)
433473
}
434474

@@ -442,7 +482,7 @@ func getAppResponse(ctx context.Context, st *SetupTypes, clientset *kubernetes.C
442482
logger.Println("Response: ", string(body))
443483
}
444484

445-
logger.Printf("%s!", errBaseMsg)
485+
logger.Println("getAppResponse() ended!")
446486

447487
return *resp, nil
448488
}

0 commit comments

Comments
 (0)