Compare commits
4 Commits
Author | SHA1 | Date |
---|---|---|
|
3fe9f5b2fb | |
|
ed4d0f56a9 | |
|
4b771b2190 | |
|
db11337200 |
|
@ -26,12 +26,13 @@ RUN apk --update add \
|
||||||
# Packaging stage
|
# Packaging stage
|
||||||
# Image source: https://github.com/litmuschaos/test-tools/blob/master/custom/hardened-alpine/experiment/Dockerfile
|
# Image source: https://github.com/litmuschaos/test-tools/blob/master/custom/hardened-alpine/experiment/Dockerfile
|
||||||
# The base image is non-root (have litmus user) with default litmus directory.
|
# The base image is non-root (have litmus user) with default litmus directory.
|
||||||
FROM litmuschaos/experiment-alpine
|
FROM litmuschaos/experiment-alpine:2.14.0
|
||||||
|
|
||||||
LABEL maintainer="LitmusChaos"
|
LABEL maintainer="LitmusChaos"
|
||||||
|
|
||||||
COPY --from=builder /output/ /litmus
|
COPY --from=builder /output/ /litmus
|
||||||
COPY --from=dep /usr/bin/sudo /usr/bin/
|
COPY --from=dep /usr/bin/sudo /usr/bin/
|
||||||
|
COPY --from=dep /usr/lib/sudo /usr/lib/sudo
|
||||||
COPY --from=dep /sbin/tc /sbin/
|
COPY --from=dep /sbin/tc /sbin/
|
||||||
COPY --from=dep /sbin/iptables /sbin/
|
COPY --from=dep /sbin/iptables /sbin/
|
||||||
|
|
||||||
|
|
|
@ -301,6 +301,7 @@ func getRunID() string {
|
||||||
// triggerInlineContinuousCmdProbe trigger the inline continuous cmd probes
|
// triggerInlineContinuousCmdProbe trigger the inline continuous cmd probes
|
||||||
func triggerInlineContinuousCmdProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
func triggerInlineContinuousCmdProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
||||||
var isExperimentFailed bool
|
var isExperimentFailed bool
|
||||||
|
probeDetails := getProbeByName(probe.Name, chaosresult.ProbeDetails)
|
||||||
// waiting for initial delay
|
// waiting for initial delay
|
||||||
if probe.RunProperties.InitialDelaySeconds != 0 {
|
if probe.RunProperties.InitialDelaySeconds != 0 {
|
||||||
log.Infof("[Wait]: Waiting for %vs before probe execution", probe.RunProperties.InitialDelaySeconds)
|
log.Infof("[Wait]: Waiting for %vs before probe execution", probe.RunProperties.InitialDelaySeconds)
|
||||||
|
@ -314,15 +315,13 @@ loop:
|
||||||
err = triggerInlineCmdProbe(probe, chaosresult)
|
err = triggerInlineCmdProbe(probe, chaosresult)
|
||||||
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
||||||
if err != nil {
|
if err != nil {
|
||||||
for index := range chaosresult.ProbeDetails {
|
probeDetails.IsProbeFailedWithError = err
|
||||||
if chaosresult.ProbeDetails[index].Name == probe.Name {
|
log.Errorf("%v http probe has Failed, err: %v", probe.Name, err)
|
||||||
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
|
isExperimentFailed = true
|
||||||
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
isExperimentFailed = true
|
break loop
|
||||||
break loop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
// waiting for the probe polling interval
|
// waiting for the probe polling interval
|
||||||
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
||||||
}
|
}
|
||||||
|
@ -339,6 +338,7 @@ loop:
|
||||||
// triggerInlineOnChaosCmdProbe trigger the inline onchaos cmd probes
|
// triggerInlineOnChaosCmdProbe trigger the inline onchaos cmd probes
|
||||||
func triggerInlineOnChaosCmdProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
func triggerInlineOnChaosCmdProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
||||||
var isExperimentFailed bool
|
var isExperimentFailed bool
|
||||||
|
probeDetails := getProbeByName(probe.Name, chaosresult.ProbeDetails)
|
||||||
duration := chaosDetails.ChaosDuration
|
duration := chaosDetails.ChaosDuration
|
||||||
// waiting for initial delay
|
// waiting for initial delay
|
||||||
if probe.RunProperties.InitialDelaySeconds != 0 {
|
if probe.RunProperties.InitialDelaySeconds != 0 {
|
||||||
|
@ -363,15 +363,13 @@ loop:
|
||||||
default:
|
default:
|
||||||
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
||||||
if err = triggerInlineCmdProbe(probe, chaosresult); err != nil {
|
if err = triggerInlineCmdProbe(probe, chaosresult); err != nil {
|
||||||
for index := range chaosresult.ProbeDetails {
|
probeDetails.IsProbeFailedWithError = err
|
||||||
if chaosresult.ProbeDetails[index].Name == probe.Name {
|
log.Errorf("%v http probe has Failed, err: %v", probe.Name, err)
|
||||||
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
|
isExperimentFailed = true
|
||||||
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
isExperimentFailed = true
|
break loop
|
||||||
break loop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
// waiting for the probe polling interval
|
// waiting for the probe polling interval
|
||||||
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
||||||
}
|
}
|
||||||
|
@ -390,6 +388,7 @@ loop:
|
||||||
func triggerSourceOnChaosCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails litmusexec.PodDetails, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
func triggerSourceOnChaosCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails litmusexec.PodDetails, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
||||||
|
|
||||||
var isExperimentFailed bool
|
var isExperimentFailed bool
|
||||||
|
probeDetails := getProbeByName(probe.Name, chaosresult.ProbeDetails)
|
||||||
duration := chaosDetails.ChaosDuration
|
duration := chaosDetails.ChaosDuration
|
||||||
// waiting for initial delay
|
// waiting for initial delay
|
||||||
if probe.RunProperties.InitialDelaySeconds != 0 {
|
if probe.RunProperties.InitialDelaySeconds != 0 {
|
||||||
|
@ -412,15 +411,13 @@ loop:
|
||||||
default:
|
default:
|
||||||
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
||||||
if err = triggerSourceCmdProbe(probe, execCommandDetails, clients, chaosresult); err != nil {
|
if err = triggerSourceCmdProbe(probe, execCommandDetails, clients, chaosresult); err != nil {
|
||||||
for index := range chaosresult.ProbeDetails {
|
probeDetails.IsProbeFailedWithError = err
|
||||||
if chaosresult.ProbeDetails[index].Name == probe.Name {
|
log.Errorf("%v http probe has Failed, err: %v", probe.Name, err)
|
||||||
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
|
isExperimentFailed = true
|
||||||
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
isExperimentFailed = true
|
break loop
|
||||||
break loop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
// waiting for the probe polling interval
|
// waiting for the probe polling interval
|
||||||
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
||||||
}
|
}
|
||||||
|
@ -440,6 +437,7 @@ loop:
|
||||||
func triggerSourceContinuousCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails litmusexec.PodDetails, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
func triggerSourceContinuousCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails litmusexec.PodDetails, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
||||||
|
|
||||||
var isExperimentFailed bool
|
var isExperimentFailed bool
|
||||||
|
probeDetails := getProbeByName(probe.Name, chaosresult.ProbeDetails)
|
||||||
// waiting for initial delay
|
// waiting for initial delay
|
||||||
if probe.RunProperties.InitialDelaySeconds != 0 {
|
if probe.RunProperties.InitialDelaySeconds != 0 {
|
||||||
log.Infof("[Wait]: Waiting for %vs before probe execution", probe.RunProperties.InitialDelaySeconds)
|
log.Infof("[Wait]: Waiting for %vs before probe execution", probe.RunProperties.InitialDelaySeconds)
|
||||||
|
@ -453,15 +451,13 @@ loop:
|
||||||
err = triggerSourceCmdProbe(probe, execCommandDetails, clients, chaosresult)
|
err = triggerSourceCmdProbe(probe, execCommandDetails, clients, chaosresult)
|
||||||
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
||||||
if err != nil {
|
if err != nil {
|
||||||
for index := range chaosresult.ProbeDetails {
|
probeDetails.IsProbeFailedWithError = err
|
||||||
if chaosresult.ProbeDetails[index].Name == probe.Name {
|
log.Errorf("%v http probe has Failed, err: %v", probe.Name, err)
|
||||||
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
|
isExperimentFailed = true
|
||||||
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
isExperimentFailed = true
|
break loop
|
||||||
break loop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
// waiting for the probe polling interval
|
// waiting for the probe polling interval
|
||||||
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
||||||
}
|
}
|
||||||
|
@ -648,14 +644,14 @@ func postChaosCmdProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Resu
|
||||||
case "Continuous", "OnChaos":
|
case "Continuous", "OnChaos":
|
||||||
if reflect.DeepEqual(probe.CmdProbeInputs.Source, v1alpha1.SourceDetails{}) {
|
if reflect.DeepEqual(probe.CmdProbeInputs.Source, v1alpha1.SourceDetails{}) {
|
||||||
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
||||||
err = checkForErrorInContinuousProbe(resultDetails, probe.Name)
|
err = checkForErrorInContinuousProbe(resultDetails, chaosDetails.Timeout, chaosDetails.Delay, probe.Name)
|
||||||
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
||||||
if err = markedVerdictInEnd(err, resultDetails, probe, "PostChaos"); err != nil {
|
if err = markedVerdictInEnd(err, resultDetails, probe, "PostChaos"); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
||||||
err = checkForErrorInContinuousProbe(resultDetails, probe.Name)
|
err = checkForErrorInContinuousProbe(resultDetails, chaosDetails.Timeout, chaosDetails.Delay, probe.Name)
|
||||||
|
|
||||||
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
||||||
if err = markedVerdictInEnd(err, resultDetails, probe, "PostChaos"); err != nil {
|
if err = markedVerdictInEnd(err, resultDetails, probe, "PostChaos"); err != nil {
|
||||||
|
|
|
@ -198,6 +198,7 @@ func getHTTPBody(httpBody v1alpha1.PostMethod) (string, error) {
|
||||||
// triggerContinuousHTTPProbe trigger the continuous http probes
|
// triggerContinuousHTTPProbe trigger the continuous http probes
|
||||||
func triggerContinuousHTTPProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
func triggerContinuousHTTPProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
||||||
var isExperimentFailed bool
|
var isExperimentFailed bool
|
||||||
|
probeDetails := getProbeByName(probe.Name, chaosresult.ProbeDetails)
|
||||||
// waiting for initial delay
|
// waiting for initial delay
|
||||||
if probe.RunProperties.InitialDelaySeconds != 0 {
|
if probe.RunProperties.InitialDelaySeconds != 0 {
|
||||||
log.Infof("[Wait]: Waiting for %vs before probe execution", probe.RunProperties.InitialDelaySeconds)
|
log.Infof("[Wait]: Waiting for %vs before probe execution", probe.RunProperties.InitialDelaySeconds)
|
||||||
|
@ -211,15 +212,13 @@ loop:
|
||||||
err = triggerHTTPProbe(probe, chaosresult)
|
err = triggerHTTPProbe(probe, chaosresult)
|
||||||
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
||||||
if err != nil {
|
if err != nil {
|
||||||
for index := range chaosresult.ProbeDetails {
|
probeDetails.IsProbeFailedWithError = err
|
||||||
if chaosresult.ProbeDetails[index].Name == probe.Name {
|
log.Errorf("%v http probe has Failed, err: %v", probe.Name, err)
|
||||||
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
|
isExperimentFailed = true
|
||||||
log.Errorf("The %v http probe has been Failed, err: %v", probe.Name, err)
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
isExperimentFailed = true
|
break loop
|
||||||
break loop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
// waiting for the probe polling interval
|
// waiting for the probe polling interval
|
||||||
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
||||||
}
|
}
|
||||||
|
@ -307,7 +306,7 @@ func postChaosHTTPProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Res
|
||||||
}
|
}
|
||||||
case "Continuous", "OnChaos":
|
case "Continuous", "OnChaos":
|
||||||
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
||||||
err = checkForErrorInContinuousProbe(resultDetails, probe.Name)
|
err = checkForErrorInContinuousProbe(resultDetails, chaosDetails.Timeout, chaosDetails.Delay, probe.Name)
|
||||||
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
||||||
if err = markedVerdictInEnd(err, resultDetails, probe, "PostChaos"); err != nil {
|
if err = markedVerdictInEnd(err, resultDetails, probe, "PostChaos"); err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -320,6 +319,7 @@ func postChaosHTTPProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Res
|
||||||
func triggerOnChaosHTTPProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
func triggerOnChaosHTTPProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
||||||
|
|
||||||
var isExperimentFailed bool
|
var isExperimentFailed bool
|
||||||
|
probeDetails := getProbeByName(probe.Name, chaosresult.ProbeDetails)
|
||||||
duration := chaosDetails.ChaosDuration
|
duration := chaosDetails.ChaosDuration
|
||||||
// waiting for initial delay
|
// waiting for initial delay
|
||||||
if probe.RunProperties.InitialDelaySeconds != 0 {
|
if probe.RunProperties.InitialDelaySeconds != 0 {
|
||||||
|
@ -343,15 +343,13 @@ loop:
|
||||||
err = triggerHTTPProbe(probe, chaosresult)
|
err = triggerHTTPProbe(probe, chaosresult)
|
||||||
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
||||||
if err != nil {
|
if err != nil {
|
||||||
for index := range chaosresult.ProbeDetails {
|
probeDetails.IsProbeFailedWithError = err
|
||||||
if chaosresult.ProbeDetails[index].Name == probe.Name {
|
log.Errorf("%v http probe has Failed, err: %v", probe.Name, err)
|
||||||
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
|
isExperimentFailed = true
|
||||||
isExperimentFailed = true
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
break loop
|
break loop
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
// waiting for the probe polling interval
|
// waiting for the probe polling interval
|
||||||
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
||||||
}
|
}
|
||||||
|
|
|
@ -115,6 +115,7 @@ func triggerK8sProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets,
|
||||||
// triggerContinuousK8sProbe trigger the continuous k8s probes
|
// triggerContinuousK8sProbe trigger the continuous k8s probes
|
||||||
func triggerContinuousK8sProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
func triggerContinuousK8sProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
||||||
var isExperimentFailed bool
|
var isExperimentFailed bool
|
||||||
|
probeDetails := getProbeByName(probe.Name, chaosresult.ProbeDetails)
|
||||||
// waiting for initial delay
|
// waiting for initial delay
|
||||||
if probe.RunProperties.InitialDelaySeconds != 0 {
|
if probe.RunProperties.InitialDelaySeconds != 0 {
|
||||||
log.Infof("[Wait]: Waiting for %vs before probe execution", probe.RunProperties.InitialDelaySeconds)
|
log.Infof("[Wait]: Waiting for %vs before probe execution", probe.RunProperties.InitialDelaySeconds)
|
||||||
|
@ -128,15 +129,13 @@ loop:
|
||||||
err = triggerK8sProbe(probe, clients, chaosresult)
|
err = triggerK8sProbe(probe, clients, chaosresult)
|
||||||
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
||||||
if err != nil {
|
if err != nil {
|
||||||
for index := range chaosresult.ProbeDetails {
|
probeDetails.IsProbeFailedWithError = err
|
||||||
if chaosresult.ProbeDetails[index].Name == probe.Name {
|
log.Errorf("%v http probe has Failed, err: %v", probe.Name, err)
|
||||||
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
|
isExperimentFailed = true
|
||||||
log.Errorf("the %v k8s probe has been Failed, err: %v", probe.Name, err)
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
isExperimentFailed = true
|
break loop
|
||||||
break loop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
// waiting for the probe polling interval
|
// waiting for the probe polling interval
|
||||||
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
||||||
}
|
}
|
||||||
|
@ -255,7 +254,7 @@ func postChaosK8sProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Resu
|
||||||
}
|
}
|
||||||
case "continuous", "onchaos":
|
case "continuous", "onchaos":
|
||||||
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
||||||
err = checkForErrorInContinuousProbe(resultDetails, probe.Name)
|
err = checkForErrorInContinuousProbe(resultDetails, chaosDetails.Timeout, chaosDetails.Delay, probe.Name)
|
||||||
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
||||||
if err = markedVerdictInEnd(err, resultDetails, probe, "PostChaos"); err != nil {
|
if err = markedVerdictInEnd(err, resultDetails, probe, "PostChaos"); err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -287,6 +286,7 @@ func onChaosK8sProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Result
|
||||||
func triggerOnChaosK8sProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
func triggerOnChaosK8sProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
||||||
|
|
||||||
var isExperimentFailed bool
|
var isExperimentFailed bool
|
||||||
|
probeDetails := getProbeByName(probe.Name, chaosresult.ProbeDetails)
|
||||||
duration := chaosDetails.ChaosDuration
|
duration := chaosDetails.ChaosDuration
|
||||||
// waiting for initial delay
|
// waiting for initial delay
|
||||||
if probe.RunProperties.InitialDelaySeconds != 0 {
|
if probe.RunProperties.InitialDelaySeconds != 0 {
|
||||||
|
@ -310,15 +310,13 @@ loop:
|
||||||
err = triggerK8sProbe(probe, clients, chaosresult)
|
err = triggerK8sProbe(probe, clients, chaosresult)
|
||||||
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
||||||
if err != nil {
|
if err != nil {
|
||||||
for index := range chaosresult.ProbeDetails {
|
probeDetails.IsProbeFailedWithError = err
|
||||||
if chaosresult.ProbeDetails[index].Name == probe.Name {
|
log.Errorf("%v http probe has Failed, err: %v", probe.Name, err)
|
||||||
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
|
isExperimentFailed = true
|
||||||
log.Errorf("The %v k8s probe has been Failed, err: %v", probe.Name, err)
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
isExperimentFailed = true
|
break loop
|
||||||
break loop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
// waiting for the probe polling interval
|
// waiting for the probe polling interval
|
||||||
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
||||||
}
|
}
|
||||||
|
|
|
@ -139,7 +139,7 @@ func getProbesFromEngine(chaosDetails *types.ChaosDetails, clients clients.Clien
|
||||||
// it fetch the probe details from the chaosengine and set into the chaosresult
|
// it fetch the probe details from the chaosengine and set into the chaosresult
|
||||||
func InitializeProbesInChaosResultDetails(chaosDetails *types.ChaosDetails, clients clients.ClientSets, chaosresult *types.ResultDetails) error {
|
func InitializeProbesInChaosResultDetails(chaosDetails *types.ChaosDetails, clients clients.ClientSets, chaosresult *types.ResultDetails) error {
|
||||||
|
|
||||||
var probeDetails []types.ProbeDetails
|
var probeDetails []*types.ProbeDetails
|
||||||
// get the probes from the chaosengine
|
// get the probes from the chaosengine
|
||||||
probes, err := getProbesFromEngine(chaosDetails, clients)
|
probes, err := getProbesFromEngine(chaosDetails, clients)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -148,7 +148,7 @@ func InitializeProbesInChaosResultDetails(chaosDetails *types.ChaosDetails, clie
|
||||||
|
|
||||||
// set the probe details for k8s probe
|
// set the probe details for k8s probe
|
||||||
for _, probe := range probes {
|
for _, probe := range probes {
|
||||||
tempProbe := types.ProbeDetails{}
|
tempProbe := &types.ProbeDetails{}
|
||||||
tempProbe.Name = probe.Name
|
tempProbe.Name = probe.Name
|
||||||
tempProbe.Type = probe.Type
|
tempProbe.Type = probe.Type
|
||||||
tempProbe.Mode = probe.Mode
|
tempProbe.Mode = probe.Mode
|
||||||
|
@ -252,14 +252,25 @@ func getDescription(mode, phase string) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
//CheckForErrorInContinuousProbe check for the error in the continuous probes
|
//CheckForErrorInContinuousProbe check for the error in the continuous probes
|
||||||
func checkForErrorInContinuousProbe(resultDetails *types.ResultDetails, probeName string) error {
|
func checkForErrorInContinuousProbe(resultDetails *types.ResultDetails, timeout, delay int, probeName string) error {
|
||||||
|
probe := getProbeByName(probeName, resultDetails.ProbeDetails)
|
||||||
|
timeoutChan := time.After(time.Duration(timeout) * time.Second)
|
||||||
|
|
||||||
for index, probe := range resultDetails.ProbeDetails {
|
loop:
|
||||||
if probe.Name == probeName {
|
for {
|
||||||
return resultDetails.ProbeDetails[index].IsProbeFailedWithError
|
select {
|
||||||
|
case <-timeoutChan:
|
||||||
|
return errors.Errorf("%v probe execution timed out", probeName)
|
||||||
|
default:
|
||||||
|
if probe.HasProbeExecutedOnce {
|
||||||
|
break loop
|
||||||
|
}
|
||||||
|
log.Infof("[Probe]: Waiting for %s probe to finish or timeout", probeName)
|
||||||
|
time.Sleep(time.Duration(delay) * time.Second)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
|
return probe.IsProbeFailedWithError
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseCommand parse the templated command and replace the templated value by actual value
|
// ParseCommand parse the templated command and replace the templated value by actual value
|
||||||
|
@ -282,7 +293,7 @@ func parseCommand(templatedCommand string, resultDetails *types.ResultDetails) (
|
||||||
// stopChaosEngine update the probe status and patch the chaosengine to stop state
|
// stopChaosEngine update the probe status and patch the chaosengine to stop state
|
||||||
func stopChaosEngine(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) error {
|
func stopChaosEngine(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) error {
|
||||||
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
||||||
err = checkForErrorInContinuousProbe(chaosresult, probe.Name)
|
err = checkForErrorInContinuousProbe(chaosresult, chaosDetails.Timeout, chaosDetails.Delay, probe.Name)
|
||||||
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
||||||
markedVerdictInEnd(err, chaosresult, probe, "PostChaos")
|
markedVerdictInEnd(err, chaosresult, probe, "PostChaos")
|
||||||
//patch chaosengine's state to stop
|
//patch chaosengine's state to stop
|
||||||
|
@ -332,3 +343,13 @@ func getProbeVerdict(resultDetails *types.ResultDetails, name, probeType string)
|
||||||
}
|
}
|
||||||
return v1alpha1.ProbeVerdictNA
|
return v1alpha1.ProbeVerdictNA
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getProbeByName returns the probe details of a probe given its name
|
||||||
|
func getProbeByName(name string, probeDetails []*types.ProbeDetails) *types.ProbeDetails {
|
||||||
|
for _, p := range probeDetails {
|
||||||
|
if p.Name == name {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
@ -128,7 +128,7 @@ func postChaosPromProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Res
|
||||||
case "continuous", "onchaos":
|
case "continuous", "onchaos":
|
||||||
|
|
||||||
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
// it will check for the error, It will detect the error if any error encountered in probe during chaos
|
||||||
err = checkForErrorInContinuousProbe(resultDetails, probe.Name)
|
err = checkForErrorInContinuousProbe(resultDetails, chaosDetails.Timeout, chaosDetails.Delay, probe.Name)
|
||||||
|
|
||||||
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
// failing the probe, if the success condition doesn't met after the retry & timeout combinations
|
||||||
if err = markedVerdictInEnd(err, resultDetails, probe, "PostChaos"); err != nil {
|
if err = markedVerdictInEnd(err, resultDetails, probe, "PostChaos"); err != nil {
|
||||||
|
@ -217,6 +217,7 @@ func triggerPromProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Resul
|
||||||
func triggerContinuousPromProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
func triggerContinuousPromProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
||||||
|
|
||||||
var isExperimentFailed bool
|
var isExperimentFailed bool
|
||||||
|
probeDetails := getProbeByName(probe.Name, chaosresult.ProbeDetails)
|
||||||
// waiting for initial delay
|
// waiting for initial delay
|
||||||
if probe.RunProperties.InitialDelaySeconds != 0 {
|
if probe.RunProperties.InitialDelaySeconds != 0 {
|
||||||
log.Infof("[Wait]: Waiting for %vs before probe execution", probe.RunProperties.InitialDelaySeconds)
|
log.Infof("[Wait]: Waiting for %vs before probe execution", probe.RunProperties.InitialDelaySeconds)
|
||||||
|
@ -230,15 +231,13 @@ loop:
|
||||||
err = triggerPromProbe(probe, chaosresult)
|
err = triggerPromProbe(probe, chaosresult)
|
||||||
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
||||||
if err != nil {
|
if err != nil {
|
||||||
for index := range chaosresult.ProbeDetails {
|
probeDetails.IsProbeFailedWithError = err
|
||||||
if chaosresult.ProbeDetails[index].Name == probe.Name {
|
log.Errorf("%v http probe has Failed, err: %v", probe.Name, err)
|
||||||
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
|
isExperimentFailed = true
|
||||||
log.Errorf("The %v prom probe has been Failed, err: %v", probe.Name, err)
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
isExperimentFailed = true
|
break loop
|
||||||
break loop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
// waiting for the probe polling interval
|
// waiting for the probe polling interval
|
||||||
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
||||||
}
|
}
|
||||||
|
@ -256,6 +255,7 @@ loop:
|
||||||
func triggerOnChaosPromProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
func triggerOnChaosPromProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) {
|
||||||
|
|
||||||
var isExperimentFailed bool
|
var isExperimentFailed bool
|
||||||
|
probeDetails := getProbeByName(probe.Name, chaosresult.ProbeDetails)
|
||||||
duration := chaosDetails.ChaosDuration
|
duration := chaosDetails.ChaosDuration
|
||||||
// waiting for initial delay
|
// waiting for initial delay
|
||||||
if probe.RunProperties.InitialDelaySeconds != 0 {
|
if probe.RunProperties.InitialDelaySeconds != 0 {
|
||||||
|
@ -277,16 +277,15 @@ loop:
|
||||||
break loop
|
break loop
|
||||||
default:
|
default:
|
||||||
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
|
||||||
if err = triggerPromProbe(probe, chaosresult); err != nil {
|
err = triggerPromProbe(probe, chaosresult)
|
||||||
for index := range chaosresult.ProbeDetails {
|
if err != nil {
|
||||||
if chaosresult.ProbeDetails[index].Name == probe.Name {
|
probeDetails.IsProbeFailedWithError = err
|
||||||
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
|
log.Errorf("%v http probe has Failed, err: %v", probe.Name, err)
|
||||||
log.Errorf("The %v prom probe has been Failed, err: %v", probe.Name, err)
|
isExperimentFailed = true
|
||||||
isExperimentFailed = true
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
break loop
|
break loop
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
probeDetails.HasProbeExecutedOnce = true
|
||||||
// waiting for the probe polling interval
|
// waiting for the probe polling interval
|
||||||
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
time.Sleep(time.Duration(probe.RunProperties.ProbePollingInterval) * time.Second)
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,7 +35,7 @@ type ResultDetails struct {
|
||||||
FailStep string
|
FailStep string
|
||||||
Phase v1alpha1.ResultPhase
|
Phase v1alpha1.ResultPhase
|
||||||
ResultUID clientTypes.UID
|
ResultUID clientTypes.UID
|
||||||
ProbeDetails []ProbeDetails
|
ProbeDetails []*ProbeDetails
|
||||||
PassedProbeCount int
|
PassedProbeCount int
|
||||||
ProbeArtifacts map[string]ProbeArtifact
|
ProbeArtifacts map[string]ProbeArtifact
|
||||||
}
|
}
|
||||||
|
@ -57,6 +57,7 @@ type ProbeDetails struct {
|
||||||
Mode string
|
Mode string
|
||||||
Status v1alpha1.ProbeStatus
|
Status v1alpha1.ProbeStatus
|
||||||
IsProbeFailedWithError error
|
IsProbeFailedWithError error
|
||||||
|
HasProbeExecutedOnce bool
|
||||||
RunID string
|
RunID string
|
||||||
RunCount int
|
RunCount int
|
||||||
}
|
}
|
||||||
|
|
|
@ -93,8 +93,11 @@ func SetHelperData(chaosDetails *types.ChaosDetails, setHelperData string, clien
|
||||||
|
|
||||||
// Get Labels
|
// Get Labels
|
||||||
labels := pod.ObjectMeta.Labels
|
labels := pod.ObjectMeta.Labels
|
||||||
delete(labels, "controller-uid")
|
for label := range labels {
|
||||||
delete(labels, "job-name")
|
if strings.HasSuffix(label, "job-name") || strings.HasSuffix(label, "controller-uid") {
|
||||||
|
delete(labels, label)
|
||||||
|
}
|
||||||
|
}
|
||||||
chaosDetails.Labels = labels
|
chaosDetails.Labels = labels
|
||||||
|
|
||||||
switch setHelperData {
|
switch setHelperData {
|
||||||
|
|
Loading…
Reference in New Issue