feat(go-experiments): Updating failure condition for helper pod (#57) (#58)

Signed-off-by: shubhamchaudhary <shubham.chaudhary@mayadata.io>

Author: ispeakc0de

chaoslib/litmus/container_kill/container-kill.go

@@ -82,25 +82,12 @@ func PrepareContainerKill(experimentsDetails *experimentTypes.ExperimentDetails,
 		return errors.Errorf("helper pod is not in running state, err: %v", err)
 	}
 
-	// Recording the chaos start timestamp
-	ChaosStartTimeStamp := time.Now().Unix()
-
 	// Wait till the completion of the helper pod
 	// set an upper limit for the waiting time
 	log.Info("[Wait]: waiting till the completion of the helper pod")
-	err = status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=container-kill-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+experimentsDetails.ChaosInterval+60)
-	if err != nil {
-		return err
-	}
-
-	//ChaosCurrentTimeStamp contains the current timestamp
-	ChaosCurrentTimeStamp := time.Now().Unix()
-	//ChaosDiffTimeStamp contains the difference of current timestamp and start timestamp
-	//It will helpful to track the total chaos duration
-	chaosDiffTimeStamp := ChaosCurrentTimeStamp - ChaosStartTimeStamp
-
-	if int(chaosDiffTimeStamp) < experimentsDetails.ChaosDuration {
-		return errors.Errorf("The helper pod failed, check the logs of helper pod for more details")
+	podStatus, err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=container-kill-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+experimentsDetails.ChaosInterval+60)
+	if err != nil || podStatus == "Failed" {
+		return errors.Errorf("helper pod failed due to, err: %v", err)
 	}
 
 	//Deleting the helper pod for container-kill chaos

chaoslib/litmus/kubelet_service_kill/kubelet-service-kill.go

@@ -72,9 +72,9 @@ func PrepareKubeletKill(experimentsDetails *experimentTypes.ExperimentDetails, c
 	// Wait till the completion of helper pod
 	log.Infof("[Wait]: Waiting for %vs till the completion of the helper pod", strconv.Itoa(experimentsDetails.ChaosDuration+30))
 
-	err = status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=kubelet-service-kill-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+30)
-	if err != nil {
-		return err
+	podStatus, err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=kubelet-service-kill-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+30)
+	if err != nil || podStatus == "Failed" {
+		return errors.Errorf("helper pod failed due to, err: %v", err)
 	}
 
 	// Checking the status of application node

chaoslib/litmus/node_cpu_hog/node-cpu-hog.go

@@ -72,9 +72,9 @@ func PrepareNodeCPUHog(experimentsDetails *experimentTypes.ExperimentDetails, cl
 	// Wait till the completion of helper pod
 	log.Infof("[Wait]: Waiting for %vs till the completion of the helper pod", strconv.Itoa(experimentsDetails.ChaosDuration+30))
 
-	err = status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=node-cpu-hog-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+30)
-	if err != nil {
-		return err
+	podStatus, err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=node-cpu-hog-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+30)
+	if err != nil || podStatus == "Failed" {
+		return errors.Errorf("helper pod failed due to, err: %v", err)
 	}
 
 	// Checking the status of application node

chaoslib/litmus/node_memory_hog/node-memory-hog.go

@@ -62,9 +62,9 @@ func PrepareNodeMemoryHog(experimentsDetails *experimentTypes.ExperimentDetails,
 	// Wait till the completion of helper pod
 	log.Infof("[Wait]: Waiting for %vs till the completion of the helper pod", strconv.Itoa(experimentsDetails.ChaosDuration+30))
 
-	err = status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=node-memory-hog-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+30)
-	if err != nil {
-		return err
+	podStatus, err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=node-memory-hog-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+30)
+	if err != nil || podStatus == "Failed" {
+		return errors.Errorf("helper pod failed due to, err: %v", err)
 	}
 
 	// Checking the status of application node

chaoslib/litmus/pod_delete/pod-delete.go

@@ -52,24 +52,11 @@ func PreparePodDelete(experimentsDetails *experimentTypes.ExperimentDetails, cli
 		return errors.Errorf("helper pod is not in running state, err: %v", err)
 	}
 
-	// Recording the chaos start timestamp
-	ChaosStartTimeStamp := time.Now().Unix()
-
 	// Wait till the completion of helper pod
 	log.Info("[Wait]: waiting till the completion of the helper pod")
-	err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=pod-delete"+runID, clients, experimentsDetails.ChaosDuration+experimentsDetails.ChaosInterval+60)
-	if err != nil {
-		return err
-	}
-
-	//ChaosCurrentTimeStamp contains the current timestamp
-	ChaosCurrentTimeStamp := time.Now().Unix()
-	//ChaosDiffTimeStamp contains the difference of current timestamp and start timestamp
-	//It will helpful to track the total chaos duration
-	chaosDiffTimeStamp := ChaosCurrentTimeStamp - ChaosStartTimeStamp
-
-	if int(chaosDiffTimeStamp) < experimentsDetails.ChaosDuration {
-		return errors.Errorf("The helper pod failed, check the logs of helper pod for more details")
+	podStatus, err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=pod-delete"+runID, clients, experimentsDetails.ChaosDuration+experimentsDetails.ChaosInterval+60)
+	if err != nil || podStatus == "Failed" {
+		return errors.Errorf("helper pod failed due to, err: %v", err)
 	}
 
 	//Deleting the helper pod

chaoslib/pumba/network_chaos/network-chaos.go

@@ -73,9 +73,9 @@ func PreparePodNetworkChaos(experimentsDetails *experimentTypes.ExperimentDetail
 	// Wait till the completion of helper pod
 	log.Infof("[Wait]: Waiting for %vs till the completion of the helper pod", strconv.Itoa(experimentsDetails.ChaosDuration))
 
-	err = status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=pumba-netem-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+30)
-	if err != nil {
-		return err
+	podStatus, err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=pumba-netem-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+30)
+	if err != nil || podStatus == "Failed" {
+		return errors.Errorf("helper pod failed due to, err: %v", err)
 	}
 
 	//Deleting the helper pod

pkg/status/application.go

@@ -101,7 +101,9 @@ func CheckContainerStatus(appNs string, appLabel string, clients clients.ClientS
 }
 
 // WaitForCompletion wait until the completion of pod
-func WaitForCompletion(appNs string, appLabel string, clients clients.ClientSets, duration int) error {
+func WaitForCompletion(appNs string, appLabel string, clients clients.ClientSets, duration int) (string, error) {
+	var podStatus string
+
 	err := retry.
 		Times(uint(duration)).
 		Wait(1 * time.Second).
@@ -112,17 +114,18 @@ func WaitForCompletion(appNs string, appLabel string, clients clients.ClientSets
 			}
 			err = nil
 			for _, pod := range podSpec.Items {
-				log.Infof("helper pod status: %v", string(pod.Status.Phase))
-				if string(pod.Status.Phase) != "Succeeded" && string(pod.Status.Phase) != "Failed" {
+				podStatus = string(pod.Status.Phase)
+				log.Infof("helper pod status: %v", podStatus)
+				if podStatus != "Succeeded" && podStatus != "Failed" {
 					return errors.Errorf("Helper pod is not yet completed yet")
 				}
 				log.InfoWithValues("The running status of Pods are as follows", logrus.Fields{
-					"Pod": pod.Name, "Status": pod.Status.Phase})
+					"Pod": pod.Name, "Status": podStatus})
 			}
 			return nil
 		})
 	if err != nil {
-		return err
+		return "", err
 	}
-	return nil
+	return podStatus, nil
 }