feat: implement otel telemetry sdk for distributed tracing (#221)

* feat: implement otel sdk

Signed-off-by: namkyu1999 <lak9348@konkuk.ac.kr>

* fix: update endpoint

Signed-off-by: namkyu1999 <lak9348@konkuk.ac.kr>

* fix: fix context logic

Signed-off-by: namkyu1999 <lak9348@konkuk.ac.kr>

* fix: make otel optional

Signed-off-by: namkyu1999 <lak9348@konkuk.ac.kr>

* (chore): Fix the release pipeline (#224)

Signed-off-by: Shubham Chaudhary <shubham.chaudhary@harness.io>

* fix: add logs

Signed-off-by: namkyu1999 <lak9348@gmail.com>

* chore

Signed-off-by: namkyu1999 <lak9348@gmail.com>

* feat: go version from 1.20 to 1.22

Signed-off-by: namkyu1999 <lak9348@gmail.com>

---------

Signed-off-by: namkyu1999 <lak9348@konkuk.ac.kr>
Signed-off-by: Shubham Chaudhary <shubham.chaudhary@harness.io>
Signed-off-by: namkyu1999 <lak9348@gmail.com>
Co-authored-by: Shubham Chaudhary <shubham.chaudhary@harness.io>

.github/workflows/build.yml

@@ -10,7 +10,7 @@ jobs:
       # Install golang
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.17.5
+          go-version: 1.20.0
 
       # Checkout to the latest commit
       # On specific directory/path
@@ -26,21 +26,17 @@ jobs:
       - name: unused-package check
         run: make unused-package-check
 
-  security:
-    container:
-      image: litmuschaos/snyk:1.0
-      volumes:
-      - /home/runner/work/_actions/:/home/runner/work/_actions/
+  gitleaks-scan:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: snyk/actions/setup@master
-      - run: snyk auth ${SNYK_TOKEN}
-      - uses: actions/setup-go@v1
+      - uses: actions/checkout@v3
         with:
-          go-version: '1.17'
-      - name: Snyk monitor
-        run: snyk test
+          fetch-depth: 0
+      - name: Run GitLeaks
+        run: |
+          wget https://github.com/gitleaks/gitleaks/releases/download/v8.18.2/gitleaks_8.18.2_linux_x64.tar.gz && \
+          tar -zxvf gitleaks_8.18.2_linux_x64.tar.gz && \
+          sudo mv gitleaks /usr/local/bin && gitleaks detect --source . -v
         
   trivy:
     needs: pre-checks
@@ -88,7 +84,7 @@ jobs:
       # Install golang
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.17.5
+          go-version: 1.22.0
 
       # Checkout to the latest commit
       # On specific directory/path

.github/workflows/push.yml

@@ -13,7 +13,7 @@ jobs:
       # Install golang
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.17.5
+          go-version: 1.22.0
 
       # Checkout to the latest commit
       # On specific directory/path
@@ -31,7 +31,6 @@ jobs:
 
   image-build:
     runs-on: ubuntu-latest
-    needs: tests
     steps:
       # Checkout to the latest commit
       # On specific directory/path
@@ -60,43 +59,3 @@ jobs:
           DNAME: ${{ secrets.DNAME }}
           DPASS: ${{ secrets.DPASS }}
         run: make push-chaos-runner
-
-  tests:
-    needs: pre-checks
-    runs-on: ubuntu-latest
-    steps:
-      # Checkout to the latest commit
-      # On specific directory/path
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      # Install golang
-      - uses: actions/setup-go@v2
-        with:
-          go-version: 1.17.5
-
-      - name: Build chaos-runner image
-        run: make build-amd64
-
-      #Install and configure a kind cluster
-      - name: Installing Prerequisites (K3S Cluster)
-        env: 
-          KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-        run: |
-          curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.21.11+k3s1 sh -s - --docker --write-kubeconfig-mode 664
-          kubectl wait node --all --for condition=ready --timeout=90s
-          mkdir -p $HOME/.kube
-          cp /etc/rancher/k3s/k3s.yaml $HOME/.kube/config
-          kubectl get nodes
-
-      - name: Dependency checks
-        env: 
-          KUBECONFIG: /etc/rancher/k3s/k3s.yaml      
-        run: |
-          make deps
-
-      - name: Running Go BDD Test
-        env: 
-          KUBECONFIG: /etc/rancher/k3s/k3s.yaml      
-        run: |
-          make test

.github/workflows/release.yml

@@ -11,25 +11,15 @@ jobs:
       # Install golang
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.17.5
+          go-version: 1.22.0
 
       # Checkout to the latest commit
       # On specific directory/path
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: gofmt check
-        run: make gofmt-check
-
-      - name: golangci-lint
-        uses: reviewdog/action-golangci-lint@v1
-
-      - name: unused-package check
-        run: make unused-package-check
-
   image-build:
     runs-on: ubuntu-latest
-    needs: tests
     steps:
       # Checkout to the latest commit
       # On specific directory/path
@@ -73,45 +63,3 @@ jobs:
           DNAME: ${{ secrets.DNAME }}
           DPASS: ${{ secrets.DPASS }}
         run: make push-chaos-runner
-
-  tests:
-    needs: pre-checks
-    runs-on: ubuntu-latest
-    steps:
-      # Checkout to the latest commit
-      # On specific directory/path
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      # Install golang
-      - uses: actions/setup-go@v2
-        with:
-          go-version: 1.17.5
-
-      #Install and configure a kind cluster
-      - name: Installing Prerequisites (K3S Cluster)
-        env: 
-          KUBECONFIG: /etc/rancher/k3s/k3s.yaml
-        run: |
-          curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.21.11+k3s1 sh -s - --docker --write-kubeconfig-mode 664
-          kubectl wait node --all --for condition=ready --timeout=90s
-          mkdir -p $HOME/.kube
-          cp /etc/rancher/k3s/k3s.yaml $HOME/.kube/config
-          kubectl get nodes
-
-      - name: Dependency checks
-        run: |
-          make deps
-
-      - name: Build Docker Image
-        env:
-          DOCKER_REPO: litmuschaos
-          DOCKER_IMAGE: chaos-runner
-          DOCKER_TAG: ci
-        run: |
-          make build-amd64
-
-      - name: Running Go BDD Test
-        run: |
-          make test
-          

.github/workflows/security-scan.yml

@@ -0,0 +1,24 @@
+---
+name: Security Scan
+on:
+  workflow_dispatch:
+
+jobs:
+
+  trivy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: Build an image from Dockerfile
+        run: |
+          docker build -f build/Dockerfile -t docker.io/litmuschaos/chaos-runner:${{ github.sha }} . --build-arg TARGETARCH=amd64
+      
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'docker.io/litmuschaos/chaos-runner:${{ github.sha }}'
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'

Makefile

@@ -52,14 +52,14 @@ build-chaos-runner:
 	@echo "-------------------------"
 	@echo "--> Build chaos-runner image" 
 	@echo "-------------------------"
-	@docker buildx build --file build/Dockerfile --progress plane  --no-cache --platform linux/arm64,linux/amd64 --tag $(DOCKER_REGISTRY)/$(DOCKER_REPO)/$(DOCKER_IMAGE):$(DOCKER_TAG) .
+	@docker buildx build --file build/Dockerfile --progress plain  --no-cache --platform linux/arm64,linux/amd64 --tag $(DOCKER_REGISTRY)/$(DOCKER_REPO)/$(DOCKER_IMAGE):$(DOCKER_TAG) .
 
 .PHONY: push-chaos-runner
 push-chaos-runner:
 	@echo "------------------------------"
 	@echo "--> Pushing image" 
 	@echo "------------------------------"
-	@docker buildx build --file build/Dockerfile --progress plane --no-cache --push --platform linux/arm64,linux/amd64 --tag $(DOCKER_REGISTRY)/$(DOCKER_REPO)/$(DOCKER_IMAGE):$(DOCKER_TAG) .
+	@docker buildx build --file build/Dockerfile --progress plain --no-cache --push --platform linux/arm64,linux/amd64 --tag $(DOCKER_REGISTRY)/$(DOCKER_REPO)/$(DOCKER_IMAGE):$(DOCKER_TAG) .
 
 .PHONY: build-amd64
 build-amd64:

README.md

@@ -7,7 +7,6 @@
 [![Twitter Follow](https://img.shields.io/twitter/follow/litmuschaos?style=social)](https://twitter.com/LitmusChaos)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5291/badge)](https://bestpractices.coreinfrastructure.org/projects/5291)
 [![Go Report Card](https://goreportcard.com/badge/github.com/litmuschaos/chaos-runner)](https://goreportcard.com/report/github.com/litmuschaos/chaos-runner)
-[![BCH compliance](https://bettercodehub.com/edge/badge/litmuschaos/chaos-runner?branch=master)](https://bettercodehub.com/)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Flitmuschaos%2Fchaos-runner.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Flitmuschaos%2Fchaos-runner?ref=badge_shield)
 [![YouTube Channel](https://img.shields.io/badge/YouTube-Subscribe-red)](https://www.youtube.com/channel/UCa57PMqmz_j0wnteRa9nCaw)
 <br><br>

bin/runner.go

@@ -1,10 +1,16 @@
 package main
 
 import (
+	"context"
+	"errors"
+	"os"
+
 	"github.com/litmuschaos/chaos-runner/pkg/log"
+	"github.com/litmuschaos/chaos-runner/pkg/telemetry"
 	"github.com/litmuschaos/chaos-runner/pkg/utils"
 	"github.com/litmuschaos/chaos-runner/pkg/utils/analytics"
 	"github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel"
 )
 
 func init() {
@@ -17,9 +23,26 @@ func init() {
 }
 
 func main() {
+	ctx := context.Background()
+	// Set up Observability.
+	if otelExporterEndpoint := os.Getenv(telemetry.OTELExporterOTLPEndpoint); otelExporterEndpoint != "" {
+		shutdown, err := telemetry.InitOTelSDK(ctx, otelExporterEndpoint)
+		if err != nil {
+			log.Errorf("Failed to initialize OTel SDK: %v", err)
+			return
+		}
+		defer func() {
+			err = errors.Join(err, shutdown(ctx))
+		}()
+		ctx = telemetry.GetTraceParentContext()
+	}
 
 	engineDetails := utils.EngineDetails{}
 	clients := utils.ClientSets{}
+
+	ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "ExecuteChaosRunner")
+	defer span.End()
+
 	// Getting kubeConfig and Generate ClientSets
 	if err := clients.GenerateClientSetFromKubeConfig(); err != nil {
 		log.Errorf("unable to create ClientSets, error: %v", err)
@@ -35,9 +58,7 @@ func main() {
 	log.InfoWithValues("Experiments details are as follows", logrus.Fields{
 		"Experiments List":     engineDetails.Experiments,
 		"Engine Name":          engineDetails.Name,
-		"appLabels":            engineDetails.AppLabel,
-		"appNs":                engineDetails.AppNs,
-		"appKind":              engineDetails.AppKind,
+		"Targets":              engineDetails.Targets,
 		"Service Account Name": engineDetails.SvcAccount,
 		"Engine Namespace":     engineDetails.EngineNamespace,
 	})
@@ -68,12 +89,19 @@ func main() {
 			continue
 		}
 		// derive the envs from the chaos experiment and override their values from chaosengine if any
-		if err := experiment.SetENV(engineDetails, clients); err != nil {
+		if err := experiment.SetENV(ctx, engineDetails, clients); err != nil {
 			log.Errorf("unable to patch ENV, error: %v", err)
 			experiment.ExperimentSkipped(utils.ExperimentEnvParseErrorReason, engineDetails, clients)
 			engineDetails.ExperimentSkippedPatchEngine(&experiment, clients)
 			continue
 		}
+		// derive the sidecar details from chaosengine
+		if err := experiment.SetSideCarDetails(engineDetails.Name, clients); err != nil {
+			log.Errorf("unable to get sidecar details, error: %v", err)
+			experiment.ExperimentSkipped(utils.ExperimentSideCarPatchErrorReason, engineDetails, clients)
+			engineDetails.ExperimentSkippedPatchEngine(&experiment, clients)
+			continue
+		}
 
 		log.Infof("Preparing to run Chaos Experiment: %v", experiment.Name)
 
@@ -87,7 +115,7 @@ func main() {
 		experiment.ExperimentDependencyCheck(engineDetails, clients)
 
 		// Creation of PodTemplateSpec, and Final Job
-		if err := utils.BuildingAndLaunchJob(&experiment, clients); err != nil {
+		if err := utils.BuildingAndLaunchJob(ctx, &experiment, clients); err != nil {
 			log.Errorf("unable to construct chaos experiment job, error: %v", err)
 			experiment.ExperimentSkipped(utils.ExperimentDependencyCheckReason, engineDetails, clients)
 			engineDetails.ExperimentSkippedPatchEngine(&experiment, clients)

build/Dockerfile

@@ -17,14 +17,15 @@ RUN go env
 RUN CGO_ENABLED=0 go build -buildvcs=false -o /output/chaos-runner -v ./bin
 
 # Packaging stage
-# Image source: https://github.com/litmuschaos/test-tools/blob/master/custom/hardend-alpine/control-plane/Dockerfile
-# The base image is non-root (have litmus user) with default litmus directory.
-FROM litmuschaos/infra-alpine
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.4
 
 LABEL maintainer="LitmusChaos"
 
 ENV RUNNER=/usr/local/bin/chaos-runner 
 
 COPY --from=builder /output/chaos-runner ${RUNNER}
+RUN chown 65534:0 ${RUNNER} && chmod 755 ${RUNNER}
+
+USER 65534
 
 ENTRYPOINT ["/usr/local/bin/chaos-runner"]

go.mod

@@ -1,71 +1,81 @@
 module github.com/litmuschaos/chaos-runner
 
-go 1.17
+go 1.22
 
 require (
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24
 	github.com/jpillora/go-ogle-analytics v0.0.0-20161213085824-14b04e0594ef
-	github.com/litmuschaos/chaos-operator v0.0.0-20221006080214-d78985569b82
-	github.com/litmuschaos/elves v0.0.0-20210325101625-5620f93aed51
-	github.com/litmuschaos/litmus-go v0.0.0-20220927112726-25d81a302a70
+	github.com/litmuschaos/chaos-operator v0.0.0-20240601063404-e96a7ee7f1f7
+	github.com/litmuschaos/elves v0.0.0-20230607095010-c7119636b529
+	github.com/litmuschaos/litmus-go v0.0.0-20230605073551-d73728198577
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.15.0
 	github.com/pkg/errors v0.9.1
-	github.com/sirupsen/logrus v1.7.0
-	github.com/stretchr/testify v1.7.0
-	k8s.io/api v0.22.2
-	k8s.io/apimachinery v0.22.2
+	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.9.0
+	go.opentelemetry.io/otel v1.27.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0
+	go.opentelemetry.io/otel/sdk v1.27.0
+	k8s.io/api v0.26.0
+	k8s.io/apimachinery v0.26.0
 	k8s.io/client-go v12.0.0+incompatible
 )
 
 require (
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kr/pretty v0.2.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
-	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect
-	golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e // indirect
-	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
+	go.opentelemetry.io/otel/metric v1.27.0 // indirect
+	go.opentelemetry.io/otel/trace v1.27.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.2.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect
+	google.golang.org/grpc v1.64.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/klog/v2 v2.9.0 // indirect
+	k8s.io/klog/v2 v2.80.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect
-	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect
+	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
 	sigs.k8s.io/controller-runtime v0.10.0 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )
 
 // Pinned to kubernetes-1.21.2
 replace (
 	k8s.io/api => k8s.io/api v0.21.2
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.2
 	k8s.io/apimachinery => k8s.io/apimachinery v0.21.2
-	k8s.io/apiserver => k8s.io/apiserver v0.21.2
 	k8s.io/cli-runtime => k8s.io/cli-runtime v0.21.2
 	k8s.io/client-go => k8s.io/client-go v0.21.2
 	k8s.io/cloud-provider => k8s.io/cloud-provider v0.21.2
 	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.21.2
-	k8s.io/code-generator => k8s.io/code-generator v0.21.2
 	k8s.io/component-base => k8s.io/component-base v0.21.2
 	k8s.io/cri-api => k8s.io/cri-api v0.21.2
 	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.21.2
@@ -81,3 +91,5 @@ replace (
 )
 
 replace github.com/docker/docker => github.com/moby/moby v0.7.3-0.20190826074503-38ab9da00309 // Required by Helm
+
+replace golang.org/x/net => golang.org/x/net v0.17.0

pkg/log/log.go

@@ -4,24 +4,24 @@ import (
 	logrus "github.com/sirupsen/logrus"
 )
 
-//Fatalf Logs first and then calls `logger.Exit(1)`
+// Fatalf Logs first and then calls `logger.Exit(1)`
 // logging level is set to Panic.
 func Fatalf(msg string, err error) {
 	logrus.WithFields(logrus.Fields{}).Fatalf(msg, err)
 }
 
-//Fatal Logs first and then calls `logger.Exit(1)`
+// Fatal Logs first and then calls `logger.Exit(1)`
 // logging level is set to Panic.
 func Fatal(msg string) {
 	logrus.WithFields(logrus.Fields{}).Fatal(msg)
 }
 
-//Infof log the General operational entries about what's going on inside the application
+// Infof log the General operational entries about what's going on inside the application
 func Infof(msg string, val ...interface{}) {
 	logrus.WithFields(logrus.Fields{}).Infof(msg, val...)
 }
 
-//Info log the General operational entries about what's going on inside the application
+// Info log the General operational entries about what's going on inside the application
 func Info(msg string) {
 	logrus.WithFields(logrus.Fields{}).Infof(msg)
 }
@@ -38,23 +38,23 @@ func ErrorWithValues(msg string, val map[string]interface{}) {
 	logrus.WithFields(val).Error(msg)
 }
 
-//Warn log the Non-critical entries that deserve eyes.
+// Warn log the Non-critical entries that deserve eyes.
 func Warn(msg string) {
 	logrus.WithFields(logrus.Fields{}).Warn(msg)
 }
 
-//Warnf log the Non-critical entries that deserve eyes.
+// Warnf log the Non-critical entries that deserve eyes.
 func Warnf(msg string, val ...interface{}) {
 	logrus.WithFields(logrus.Fields{}).Warnf(msg, val...)
 }
 
-//Errorf used for errors that should definitely be noted.
+// Errorf used for errors that should definitely be noted.
 // Commonly used for hooks to send errors to an error tracking service.
 func Errorf(msg string, err ...interface{}) {
 	logrus.WithFields(logrus.Fields{}).Errorf(msg, err...)
 }
 
-//Error used for errors that should definitely be noted.
+// Error used for errors that should definitely be noted.
 // Commonly used for hooks to send errors to an error tracking service
 func Error(msg string) {
 	logrus.WithFields(logrus.Fields{}).Error(msg)

pkg/telemetry/otel.go

@@ -0,0 +1,89 @@
+package telemetry
+
+import (
+	"context"
+	"errors"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/sdk/trace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
+)
+
+// TODO: change endpoint to be configurable
+const OTELExporterOTLPEndpoint = "OTEL_EXPORTER_OTLP_ENDPOINT"
+const OTELServiceName = "chaos_runner"
+
+func InitOTelSDK(ctx context.Context, endpoint string) (shutdown func(context.Context) error, err error) {
+	var shutdownFuncs []func(context.Context) error
+
+	shutdown = func(ctx context.Context) error {
+		var err error
+		for _, fn := range shutdownFuncs {
+			err = errors.Join(err, fn(ctx))
+		}
+		shutdownFuncs = nil
+		return err
+	}
+
+	handleErr := func(inErr error) {
+		err = errors.Join(inErr, shutdown(ctx))
+	}
+
+	tracerProvider, err := newTracerProvider(ctx, endpoint)
+	if err != nil {
+		handleErr(err)
+		return
+	}
+
+	prop := newPropagator()
+	otel.SetTextMapPropagator(prop)
+
+	shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown)
+	otel.SetTracerProvider(tracerProvider)
+
+	log.Info("OTel SDK initialized")
+
+	// TODO: need to add metrics & logging provider
+	return
+}
+
+func newPropagator() propagation.TextMapPropagator {
+	return propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{},
+		propagation.Baggage{},
+	)
+}
+
+func newTracerProvider(ctx context.Context, endpoint string) (*trace.TracerProvider, error) {
+	res, err := resource.New(ctx,
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(OTELServiceName),
+		),
+	)
+	traceExporter, err := otlptrace.New(
+		ctx,
+		otlptracegrpc.NewClient(
+			// TODO: add secure option
+			otlptracegrpc.WithInsecure(),
+			otlptracegrpc.WithEndpoint(endpoint),
+		),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	batchSpanProcessor := sdktrace.NewBatchSpanProcessor(traceExporter)
+	tracerProvider := sdktrace.NewTracerProvider(
+		sdktrace.WithSampler(sdktrace.AlwaysSample()),
+		sdktrace.WithResource(res),
+		sdktrace.WithSpanProcessor(batchSpanProcessor),
+	)
+
+	return tracerProvider, nil
+}

pkg/telemetry/tracing.go

@@ -0,0 +1,52 @@
+package telemetry
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+
+	"github.com/litmuschaos/chaos-runner/pkg/log"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+)
+
+const (
+	TracerName  = "litmuschaos.io/chaos-runner"
+	TraceParent = "TRACE_PARENT"
+)
+
+func GetTraceParentContext() context.Context {
+	traceParent := os.Getenv(TraceParent)
+
+	pro := otel.GetTextMapPropagator()
+	carrier := make(map[string]string)
+	if err := json.Unmarshal([]byte(traceParent), &carrier); err != nil {
+		log.Fatal(err.Error())
+	}
+
+	return pro.Extract(context.Background(), propagation.MapCarrier(carrier))
+}
+
+// GetMarshalledSpanFromContext Extract spanContext from the context and return it as json encoded string
+func GetMarshalledSpanFromContext(ctx context.Context) string {
+	carrier := make(map[string]string)
+	pro := otel.GetTextMapPropagator()
+
+	pro.Inject(ctx, propagation.MapCarrier(carrier))
+
+	if len(carrier) == 0 {
+		log.Error("spanContext not present in the context, unable to marshall")
+		return ""
+	}
+
+	marshalled, err := json.Marshal(carrier)
+	if err != nil {
+		log.Error(err.Error())
+		return ""
+	}
+	if len(marshalled) >= 1024 {
+		log.Error("marshalled span context is too large, unable to marshall")
+		return ""
+	}
+	return string(marshalled)
+}

pkg/utils/builders.go

@@ -2,17 +2,19 @@ package utils
 
 import (
 	"context"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"reflect"
 
-	"github.com/pkg/errors"
-	batchv1 "k8s.io/api/batch/v1"
-	corev1 "k8s.io/api/core/v1"
-
+	"github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1"
+	"github.com/litmuschaos/chaos-runner/pkg/telemetry"
 	"github.com/litmuschaos/elves/kubernetes/container"
 	"github.com/litmuschaos/elves/kubernetes/job"
 	"github.com/litmuschaos/elves/kubernetes/jobspec"
 	"github.com/litmuschaos/elves/kubernetes/podtemplatespec"
+	"github.com/pkg/errors"
+	"go.opentelemetry.io/otel"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // PodTemplateSpec is struct for creating the *core1.PodTemplateSpec
@@ -31,15 +33,11 @@ func buildContainerSpec(experiment *ExperimentDetails, envVars []corev1.EnvVar)
 		WithEnvsNew(envVars)
 
 	if !reflect.DeepEqual(experiment.SecurityContext.ContainerSecurityContext, corev1.SecurityContext{}) {
-
 		containerSpec.WithSecurityContext(experiment.SecurityContext.ContainerSecurityContext)
-
 	}
 
 	if !reflect.DeepEqual(experiment.ResourceRequirements, corev1.ResourceRequirements{}) {
-
 		containerSpec.WithResourceRequirements(experiment.ResourceRequirements)
-
 	}
 
 	if experiment.VolumeOpts.VolumeMounts != nil {
@@ -56,6 +54,45 @@ func buildContainerSpec(experiment *ExperimentDetails, envVars []corev1.EnvVar)
 
 }
 
+// buildSideCarSpec builds a Container with following properties
+func buildSideCarSpec(experiment *ExperimentDetails) ([]*container.Builder, error) {
+	var sidecarContainers []*container.Builder
+
+	for _, sidecar := range experiment.SideCars {
+		var volumeOpts VolumeOpts
+
+		if len(sidecar.Secrets) != 0 {
+			volumeOpts.NewVolumeMounts().BuildVolumeMountsForSecrets(sidecar.Secrets)
+		}
+
+		containerSpec := container.NewBuilder().
+			WithName(experiment.JobName + "-sidecar-" + RandomString(6)).
+			WithImage(sidecar.Image).
+			WithImagePullPolicy(sidecar.ImagePullPolicy).
+			WithEnvsNew(sidecar.ENV)
+
+		if !reflect.DeepEqual(experiment.ResourceRequirements, corev1.ResourceRequirements{}) {
+			containerSpec.WithResourceRequirements(experiment.ResourceRequirements)
+		}
+
+		if volumeOpts.VolumeMounts != nil {
+			containerSpec.WithVolumeMountsNew(volumeOpts.VolumeMounts)
+		}
+
+		if len(sidecar.EnvFrom) != 0 {
+			containerSpec.WithEnvsFrom(sidecar.EnvFrom)
+		}
+
+		if _, err := containerSpec.Build(); err != nil {
+			return nil, err
+		}
+
+		sidecarContainers = append(sidecarContainers, containerSpec)
+	}
+
+	return sidecarContainers, err
+}
+
 func getEnvFromMap(m map[string]corev1.EnvVar) []corev1.EnvVar {
 	var envVars []corev1.EnvVar
 	for _, v := range m {
@@ -77,7 +114,10 @@ func getEnvFromMap(m map[string]corev1.EnvVar) []corev1.EnvVar {
 }
 
 // BuildingAndLaunchJob builds Job, and then launch it.
-func BuildingAndLaunchJob(experiment *ExperimentDetails, clients ClientSets) error {
+func BuildingAndLaunchJob(ctx context.Context, experiment *ExperimentDetails, clients ClientSets) error {
+	ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "BuildingAndLaunchJob")
+	defer span.End()
+
 	experiment.VolumeOpts.VolumeOperations(experiment)
 
 	envVars := getEnvFromMap(experiment.envMap)
@@ -86,8 +126,19 @@ func BuildingAndLaunchJob(experiment *ExperimentDetails, clients ClientSets) err
 	if err != nil {
 		return errors.Errorf("unable to build Container for Chaos Experiment, error: %v", err)
 	}
+
+	containers := []*container.Builder{containerForPod}
+
+	if len(experiment.SideCars) != 0 {
+		sidecars, err := buildSideCarSpec(experiment)
+		if err != nil {
+			return errors.Errorf("unable to build sidecar Container for Chaos Experiment, error: %v", err)
+		}
+		containers = append(containers, sidecars...)
+	}
+
 	// Will build a PodSpecTemplate
-	pod, err := buildPodTemplateSpec(experiment, containerForPod)
+	pod, err := buildPodTemplateSpec(experiment, containers...)
 	if err != nil {
 
 		return errors.Errorf("unable to build PodTemplateSpec for Chaos Experiment, error: %v", err)
@@ -115,8 +166,8 @@ func (expDetails *ExperimentDetails) launchJob(job *batchv1.Job, clients ClientS
 	return err
 }
 
-// BuildPodTemplateSpec return a PodTempplateSpec
-func buildPodTemplateSpec(experiment *ExperimentDetails, containerForPod *container.Builder) (*podtemplatespec.Builder, error) {
+// BuildPodTemplateSpec return a PodTemplateSpec
+func buildPodTemplateSpec(experiment *ExperimentDetails, containers ...*container.Builder) (*podtemplatespec.Builder, error) {
 	podtemplate := podtemplatespec.NewBuilder().
 		WithName(experiment.JobName).
 		WithNamespace(experiment.Namespace).
@@ -125,7 +176,7 @@ func buildPodTemplateSpec(experiment *ExperimentDetails, containerForPod *contai
 		WithRestartPolicy(corev1.RestartPolicyNever).
 		WithVolumeBuilders(experiment.VolumeOpts.VolumeBuilders).
 		WithAnnotations(experiment.Annotations).
-		WithContainerBuildersNew(containerForPod)
+		WithContainerBuildersNew(containers...)
 
 	if experiment.TerminationGracePeriodSeconds != 0 {
 		podtemplate.WithTerminationGracePeriodSeconds(experiment.TerminationGracePeriodSeconds)
@@ -137,6 +188,15 @@ func buildPodTemplateSpec(experiment *ExperimentDetails, containerForPod *contai
 
 	}
 
+	if len(experiment.SideCars) != 0 {
+		secrets := setSidecarSecrets(experiment)
+		if len(secrets) != 0 {
+			var volumeOpts VolumeOpts
+			volumeOpts.NewVolumeBuilder().BuildVolumeBuilderForSecrets(secrets)
+			podtemplate.WithVolumeBuilders(volumeOpts.VolumeBuilders)
+		}
+	}
+
 	if experiment.HostPID {
 		podtemplate.WithHostPID(experiment.HostPID)
 	}
@@ -159,6 +219,20 @@ func buildPodTemplateSpec(experiment *ExperimentDetails, containerForPod *contai
 	return podtemplate, nil
 }
 
+func setSidecarSecrets(experiment *ExperimentDetails) []v1alpha1.Secret {
+	var secrets []v1alpha1.Secret
+	secretMap := make(map[string]bool)
+	for _, sidecar := range experiment.SideCars {
+		for _, secret := range sidecar.Secrets {
+			if _, ok := secretMap[secret.Name]; !ok {
+				secretMap[secret.Name] = true
+				secrets = append(secrets, secret)
+			}
+		}
+	}
+	return secrets
+}
+
 // BuildJobSpec returns a JobSpec
 func buildJobSpec(pod *podtemplatespec.Builder) (*jobspec.Builder, error) {
 	jobSpecObj := jobspec.NewBuilder().

pkg/utils/builders_fuzz_test.go

@@ -0,0 +1,81 @@
+package utils
+
+import (
+	fuzz "github.com/AdaLogics/go-fuzz-headers"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	"testing"
+)
+
+func FuzzBuildContainerSpec(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		fuzzConsumer := fuzz.NewConsumer(data)
+		targetStruct := &struct {
+			ExpDetails *ExperimentDetails
+			EnvVars    []corev1.EnvVar
+		}{}
+
+		err := fuzzConsumer.GenerateStruct(targetStruct)
+		if err != nil {
+			return
+		}
+		containerSpec, err := buildContainerSpec(targetStruct.ExpDetails, targetStruct.EnvVars)
+		if err != nil {
+			return
+		}
+
+		container, err := containerSpec.Build()
+		if err != nil {
+			return
+		}
+
+		require.Equal(t, targetStruct.ExpDetails.JobName, container.Name)
+		require.Equal(t, targetStruct.ExpDetails.ExpImage, container.Image)
+		require.Equal(t, targetStruct.ExpDetails.ExpCommand, container.Command)
+		require.Equal(t, targetStruct.ExpDetails.ExpArgs, container.Args)
+		require.Equal(t, targetStruct.ExpDetails.ExpImagePullPolicy, container.ImagePullPolicy)
+		require.Equal(t, targetStruct.EnvVars, container.Env)
+	})
+}
+
+func FuzzGetEnvFromMap(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		fuzzConsumer := fuzz.NewConsumer(data)
+		targetStruct := &struct {
+			m map[string]corev1.EnvVar
+		}{}
+		err := fuzzConsumer.GenerateStruct(targetStruct)
+		if err != nil {
+			return
+		}
+		envs := getEnvFromMap(targetStruct.m)
+		var envCount = len(envs)
+		require.Equal(t, envCount, len(targetStruct.m)+1)
+	})
+}
+
+func FuzzSetSidecarSecrets(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		fuzzConsumer := fuzz.NewConsumer(data)
+		targetStruct := &struct {
+			experiment *ExperimentDetails
+		}{}
+
+		err := fuzzConsumer.GenerateStruct(targetStruct)
+		if err != nil {
+			return
+		}
+		if targetStruct.experiment != nil {
+			secrets := setSidecarSecrets(targetStruct.experiment)
+			require.GreaterOrEqual(t, len(secrets), 1)
+
+			for _, sidecar := range targetStruct.experiment.SideCars {
+				for _, secret := range sidecar.Secrets {
+					for _, s := range secrets {
+						require.Equal(t, s.Name, secret.Name)
+					}
+				}
+			}
+		}
+	})
+}

pkg/utils/common.go

@@ -7,10 +7,10 @@ import (
 )
 
 // RandomString will generate a random string of length 6
-func RandomString() string {
+func RandomString(length int) string {
 	rand.Seed(time.Now().UnixNano())
 	chars := []rune("abcdefghijklmnopqrstuvwxyz" + "0123456789")
-	length := 6
+
 	var b strings.Builder
 	for i := 0; i < length; i++ {
 		b.WriteRune(chars[rand.Intn(len(chars))])

pkg/utils/common_fuzz_test.go

@@ -0,0 +1,38 @@
+package utils
+
+import (
+	"strings"
+	"testing"
+)
+
+func FuzzRandomString(f *testing.F) {
+	f.Add(6)
+	f.Fuzz(func(t *testing.T, n int) {
+		randomString := RandomString(n)
+		// Perform checks on the generated string
+		// Check if the length matches the expected length
+		if n >= 0 && len(randomString) != n {
+			t.Errorf("Generated string length doesn't match expected length")
+		}
+
+		// Check if the string contains only valid characters
+		if !isValidString(randomString) {
+			t.Errorf("Generated string contains invalid characters")
+		}
+	})
+
+}
+
+func isValidString(s string) bool {
+	// Define the set of valid characters
+	validChars := "abcdefghijklmnopqrstuvwxyz0123456789"
+
+	// Iterate over each character in the string
+	for _, char := range s {
+		// Check if the character is not in the set of valid characters
+		if !strings.ContainsRune(validChars, char) {
+			return false
+		}
+	}
+	return true
+}

pkg/utils/configMapUtils.go

@@ -8,7 +8,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-//PatchConfigMaps patches configmaps in experimentDetails struct.
+// PatchConfigMaps patches configmaps in experimentDetails struct.
 func (expDetails *ExperimentDetails) PatchConfigMaps(clients ClientSets, engineDetails EngineDetails) error {
 	if err := expDetails.SetConfigMaps(clients, engineDetails); err != nil {
 		return err

pkg/utils/engineHelper.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"context"
+	"fmt"
 	"reflect"
 	"strconv"
 
@@ -12,6 +13,11 @@ import (
 	litmuschaosv1alpha1 "github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1"
 )
 
+const (
+	SideCarEnabled = "sidecar/enabled"
+	SideCarPrefix  = "SIDECAR"
+)
+
 // SetInstanceAttributeValuesFromChaosEngine set the value from the chaosengine
 func (expDetails *ExperimentDetails) SetInstanceAttributeValuesFromChaosEngine(engine *EngineDetails, clients ClientSets) error {
 	chaosEngine, err := engine.GetChaosEngine(clients)
@@ -20,6 +26,7 @@ func (expDetails *ExperimentDetails) SetInstanceAttributeValuesFromChaosEngine(e
 	}
 	// fetch all the values from chaosengine and set into expDetails struct
 	expDetails.SetExpAnnotationFromEngine(chaosEngine).
+		SetEngineLabels(chaosEngine).
 		SetExpNodeSelectorFromEngine(chaosEngine).
 		SetResourceRequirementsFromEngine(chaosEngine).
 		SetImagePullSecretsFromEngine(chaosEngine).
@@ -110,7 +117,7 @@ func (expDetails *ExperimentDetails) SetTolerationsFromEngine(engine *litmuschao
 
 // SetDefaultHealthCheck sets th default health checks provided inside the chaosEngine
 func (expDetails *ExperimentDetails) SetDefaultHealthCheck(engine *litmuschaosv1alpha1.ChaosEngine) *ExperimentDetails {
-	expDetails.DefaultHealthCheck = engine.Spec.DefaultHealthCheck
+	expDetails.DefaultHealthCheck = strconv.FormatBool(engine.Spec.DefaultHealthCheck)
 	return expDetails
 }
 
@@ -134,11 +141,7 @@ func (expDetails *ExperimentDetails) SetOverrideEnvFromChaosEngine(engineName st
 				}
 			}
 
-			delay, timeout := 2, 180
-			if sc := exp.Spec.Components.StatusCheckTimeouts; !reflect.DeepEqual(sc, litmuschaosv1alpha1.StatusCheckTimeout{}) {
-				delay = sc.Delay
-				timeout = sc.Timeout
-			}
+			delay, timeout := getStatusCheckDelayAndTimeout(exp)
 
 			expDetails.envMap["STATUS_CHECK_DELAY"] = v1.EnvVar{
 				Name:  "STATUS_CHECK_DELAY",
@@ -160,3 +163,97 @@ func (expDetails *ExperimentDetails) SetOverrideEnvFromChaosEngine(engineName st
 
 	return nil
 }
+
+func (expDetails *ExperimentDetails) SetSideCarDetails(engineName string, clients ClientSets) error {
+
+	engineSpec, err := clients.LitmusClient.LitmuschaosV1alpha1().ChaosEngines(expDetails.Namespace).Get(context.Background(), engineName, metav1.GetOptions{})
+	if err != nil {
+		return errors.Errorf("unable to get ChaosEngine Resource in namespace: %v", expDetails.Namespace)
+	}
+
+	if engineSpec.Annotations == nil {
+		return nil
+	}
+
+	if sidecarEnabled, ok := engineSpec.Annotations[SideCarEnabled]; !ok || (sidecarEnabled == "false") {
+		return nil
+	}
+
+	if len(engineSpec.Spec.Components.Sidecar) == 0 {
+		return fmt.Errorf("sidecar image is not set inside chaosengine")
+	}
+
+	expDetails.SideCars = expDetails.getSidecarDetails(engineSpec)
+	return nil
+}
+
+func (expDetails *ExperimentDetails) getSidecarDetails(engineSpec *litmuschaosv1alpha1.ChaosEngine) []SideCar {
+	var sidecars []SideCar
+	for _, v := range engineSpec.Spec.Components.Sidecar {
+		sidecar := SideCar{
+			Image:           v.Image,
+			ImagePullPolicy: v.ImagePullPolicy,
+			Secrets:         v.Secrets,
+			ENV:             append(v.ENV, getDefaultEnvs(expDetails.JobName)...),
+			EnvFrom:         v.EnvFrom,
+		}
+
+		if sidecar.ImagePullPolicy == "" {
+			sidecar.ImagePullPolicy = v1.PullIfNotPresent
+		}
+
+		sidecars = append(sidecars, sidecar)
+	}
+	return sidecars
+}
+
+func getDefaultEnvs(cName string) []v1.EnvVar {
+	return []v1.EnvVar{
+		{
+			Name:      "POD_NAME",
+			ValueFrom: getEnvSource("v1", "metadata.name"),
+		},
+		{
+			Name:      "POD_NAMESPACE",
+			ValueFrom: getEnvSource("v1", "metadata.namespace"),
+		},
+		{
+			Name:  "MAIN_CONTAINER",
+			Value: cName,
+		},
+	}
+}
+
+// getEnvSource return the env source for the given apiVersion & fieldPath
+func getEnvSource(apiVersion string, fieldPath string) *v1.EnvVarSource {
+	downwardENV := v1.EnvVarSource{
+		FieldRef: &v1.ObjectFieldSelector{
+			APIVersion: apiVersion,
+			FieldPath:  fieldPath,
+		},
+	}
+	return &downwardENV
+}
+
+// SetEngineLabels sets the engine labels
+func (expDetails *ExperimentDetails) SetEngineLabels(engine *litmuschaosv1alpha1.ChaosEngine) *ExperimentDetails {
+	for k, v := range engine.Labels {
+		expDetails.ExpLabels[k] = v
+	}
+	return expDetails
+}
+
+func getStatusCheckDelayAndTimeout(exp litmuschaosv1alpha1.ExperimentList) (int, int) {
+	delay, timeout := 2, 180
+
+	if sc := exp.Spec.Components.StatusCheckTimeouts; !reflect.DeepEqual(sc, litmuschaosv1alpha1.StatusCheckTimeout{}) {
+		if sc.Timeout != 0 {
+			timeout = sc.Timeout
+		}
+
+		if sc.Delay != 0 {
+			delay = sc.Delay
+		}
+	}
+	return delay, timeout
+}

pkg/utils/environment.go

@@ -1,28 +1,25 @@
 package utils
 
 import (
+	"context"
 	"os"
 	"strconv"
 	"strings"
 
-	v1 "k8s.io/api/core/v1"
-
 	"github.com/litmuschaos/chaos-runner/pkg/log"
+	"github.com/litmuschaos/chaos-runner/pkg/telemetry"
+	v1 "k8s.io/api/core/v1"
 )
 
 // SetEngineDetails adds the ENV's to EngineDetails
 func (engineDetails *EngineDetails) SetEngineDetails() *EngineDetails {
 	engineDetails.Experiments = strings.Split(os.Getenv("EXPERIMENT_LIST"), ",")
 	engineDetails.Name = os.Getenv("CHAOSENGINE")
-	engineDetails.AppLabel = os.Getenv("APP_LABEL")
-	engineDetails.AppNs = os.Getenv("APP_NAMESPACE")
 	engineDetails.EngineNamespace = os.Getenv("CHAOS_NAMESPACE")
-	engineDetails.AppKind = os.Getenv("APP_KIND")
 	engineDetails.SvcAccount = os.Getenv("CHAOS_SVC_ACC")
 	engineDetails.ClientUUID = os.Getenv("CLIENT_UUID")
 	engineDetails.AuxiliaryAppInfo = os.Getenv("AUXILIARY_APPINFO")
-	engineDetails.AnnotationKey = os.Getenv("ANNOTATION_KEY")
-	engineDetails.AnnotationCheck = os.Getenv("ANNOTATION_CHECK")
+	engineDetails.Targets = os.Getenv("TARGETS")
 	return engineDetails
 }
 
@@ -36,23 +33,22 @@ func (engineDetails *EngineDetails) SetEngineUID(clients ClientSets) error {
 	return nil
 }
 
-//SetENV sets ENV values in experimentDetails struct.
-func (expDetails *ExperimentDetails) SetENV(engineDetails EngineDetails, clients ClientSets) error {
+// SetENV sets ENV values in experimentDetails struct.
+func (expDetails *ExperimentDetails) SetENV(ctx context.Context, engineDetails EngineDetails, clients ClientSets) error {
+
 	// Setting envs from engine fields other than env
 	expDetails.setEnv("CHAOSENGINE", engineDetails.Name).
-		setEnv("APP_LABEL", engineDetails.AppLabel).
+		setEnv("TARGETS", engineDetails.Targets).
 		setEnv("CHAOS_NAMESPACE", engineDetails.EngineNamespace).
-		setEnv("APP_NAMESPACE", engineDetails.AppNs).
-		setEnv("APP_KIND", engineDetails.AppKind).
 		setEnv("AUXILIARY_APPINFO", engineDetails.AuxiliaryAppInfo).
 		setEnv("CHAOS_UID", engineDetails.UID).
 		setEnv("EXPERIMENT_NAME", expDetails.Name).
-		setEnv("ANNOTATION_KEY", engineDetails.AnnotationKey).
-		setEnv("ANNOTATION_CHECK", engineDetails.AnnotationCheck).
 		setEnv("LIB_IMAGE_PULL_POLICY", string(expDetails.ExpImagePullPolicy)).
 		setEnv("TERMINATION_GRACE_PERIOD_SECONDS", strconv.Itoa(int(expDetails.TerminationGracePeriodSeconds))).
 		setEnv("DEFAULT_HEALTH_CHECK", expDetails.DefaultHealthCheck).
-		setEnv("CHAOS_SERVICE_ACCOUNT", expDetails.SvcAccount)
+		setEnv("CHAOS_SERVICE_ACCOUNT", expDetails.SvcAccount).
+		setEnv("OTEL_EXPORTER_OTLP_ENDPOINT", os.Getenv(telemetry.OTELExporterOTLPEndpoint)).
+		setEnv("TRACE_PARENT", telemetry.GetMarshalledSpanFromContext(ctx))
 
 	// Get the Default ENV's from ChaosExperiment
 	log.Info("Getting the ENV Variables")

pkg/utils/event.go

@@ -11,7 +11,7 @@ import (
 	clientTypes "k8s.io/apimachinery/pkg/types"
 )
 
-//CreateEvents create the events in the desired resource
+// CreateEvents create the events in the desired resource
 func (engineDetails EngineDetails) CreateEvents(eventAttributes *EventAttributes, clients ClientSets) error {
 
 	events := &apiv1.Event{
@@ -42,7 +42,7 @@ func (engineDetails EngineDetails) CreateEvents(eventAttributes *EventAttributes
 
 }
 
-//GenerateEvents update the events and increase the count by 1, if already present
+// GenerateEvents update the events and increase the count by 1, if already present
 // else it will create a new event
 func (engineDetails EngineDetails) GenerateEvents(eventAttributes *EventAttributes, clients ClientSets) error {
 

pkg/utils/experimentHelper.go

@@ -28,7 +28,7 @@ func (engineDetails *EngineDetails) NewExperimentDetails(i int) ExperimentDetail
 	experimentDetails.SvcAccount = engineDetails.SvcAccount
 	experimentDetails.Namespace = engineDetails.EngineNamespace
 	// Setting the JobName in Experiment related struct
-	experimentDetails.JobName = experimentDetails.Name + "-" + RandomString()
+	experimentDetails.JobName = experimentDetails.Name + "-" + RandomString(6)
 	return experimentDetails
 }
 
@@ -73,7 +73,7 @@ func (expDetails *ExperimentDetails) HandleChaosExperimentExistence(engineDetail
 	return nil
 }
 
-//SetDefaultAttributeValuesFromChaosExperiment sets value in experimentDetails struct from chaosExperiment
+// SetDefaultAttributeValuesFromChaosExperiment sets value in experimentDetails struct from chaosExperiment
 func (expDetails *ExperimentDetails) SetDefaultAttributeValuesFromChaosExperiment(clients ClientSets, engine *EngineDetails) error {
 
 	experimentSpec, err := clients.LitmusClient.LitmuschaosV1alpha1().ChaosExperiments(expDetails.Namespace).Get(context.Background(), expDetails.Name, metav1.GetOptions{})

pkg/utils/hostFileVolumeUtils.go

@@ -15,7 +15,7 @@ import (
 //socket file mounts etc., and are often have fixed paths, i.e., similar to securityContext/hostPID
 //and other such mandatory attributes
 
-//PatchHostFileVolumes patches hostFileVolume in experimentDetails struct.
+// PatchHostFileVolumes patches hostFileVolume in experimentDetails struct.
 func (expDetails *ExperimentDetails) PatchHostFileVolumes(clients ClientSets, engineDetails EngineDetails) error {
 	err := expDetails.SetHostFileVolumes(clients, engineDetails)
 	if err != nil {

pkg/utils/testdata/fuzz/FuzzSetSidecarSecrets/582528ddfad69eb5

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0")

pkg/utils/testdata/fuzz/FuzzSetWatchChaosContainerForCompletion/582528ddfad69eb5

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0")

pkg/utils/types.go

@@ -19,16 +19,12 @@ import (
 type EngineDetails struct {
 	Name             string
 	Experiments      []string
-	AppLabel         string
-	AppNs            string
+	Targets          string
 	SvcAccount       string
-	AppKind          string
 	ClientUUID       string
 	AuxiliaryAppInfo string
 	UID              string
 	EngineNamespace  string
-	AnnotationKey    string
-	AnnotationCheck  string
 }
 
 // ExperimentDetails is for collecting all the experiment-related details
@@ -60,9 +56,18 @@ type ExperimentDetails struct {
 	StatusCheckTimeout            int
 	TerminationGracePeriodSeconds int64
 	DefaultHealthCheck            string
+	SideCars                      []SideCar
 }
 
-//VolumeOpts is a strcuture for all volume related operations
+type SideCar struct {
+	ENV             []v1.EnvVar
+	Image           string
+	ImagePullPolicy v1.PullPolicy
+	Secrets         []v1alpha1.Secret
+	EnvFrom         []v1.EnvFromSource
+}
+
+// VolumeOpts is a strcuture for all volume related operations
 type VolumeOpts struct {
 	VolumeMounts   []v1.VolumeMount
 	VolumeBuilders []*volume.Builder
@@ -106,6 +111,8 @@ const (
 	ExperimentChaosContainerWatchErrorReason string = "ChaosContainerWatchNotPermitted"
 	// ChaosResourceNotFoundReason contains the reason for the chaos-resources-not-found event
 	ChaosResourceNotFoundReason string = "ChaosResourceNotFound"
+	// ExperimentSideCarPatchErrorReason contains the reason for the side-car-patch-error event
+	ExperimentSideCarPatchErrorReason string = "SideCarPatchError"
 )
 
 // GenerateClientSetFromKubeConfig will generation both ClientSets (k8s, and Litmus)

pkg/utils/watchChaosContainer.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	"github.com/litmuschaos/litmus-go/pkg/utils/retry"
@@ -49,20 +50,21 @@ func GetChaosContainerStatus(experimentDetails *ExperimentDetails, clients Clien
 	if err != nil {
 		return false, errors.Errorf("unable to get the chaos pod, error: %v", err)
 	}
-	if pod.Status.Phase == corev1.PodRunning || pod.Status.Phase == corev1.PodSucceeded {
+	if pod.Status.Phase == corev1.PodSucceeded {
+		return true, nil
+	}
+
+	if pod.Status.Phase == corev1.PodRunning {
 		for _, container := range pod.Status.ContainerStatuses {
 
 			//NOTE: The name of container inside chaos-pod is same as the chaos job name
 			// we only have one container inside chaos pod to inject the chaos
 			// looking the chaos container is completed or not
-			if container.Name == experimentDetails.JobName && container.State.Terminated != nil {
-				if container.State.Terminated.Reason == "Completed" {
-					isCompleted = !container.Ready
-
-				}
+			if strings.Contains(container.Name, experimentDetails.JobName) && container.State.Terminated == nil {
+				return false, nil
 			}
 		}
-
+		return true, nil
 	} else if pod.Status.Phase == corev1.PodPending {
 		delay := 2
 		err := retry.
@@ -101,6 +103,10 @@ func (engineDetails EngineDetails) WatchChaosContainerForCompletion(experiment *
 			return err
 		}
 
+		if isChaosCompleted {
+			return nil
+		}
+
 		var expStatus ExperimentStatus
 		chaosPod, err := GetChaosPod(experiment, clients)
 		if err != nil {

pkg/utils/watchJob_test.go

@@ -19,7 +19,6 @@ func TestPatchChaosEngineStatus(t *testing.T) {
 	fakeServiceAcc := "Fake Service Account"
 	fakeAppLabel := "Fake Label"
 	fakeAppKind := "Fake Kind"
-	fakeAnnotationCheck := "Fake Annotation Check"
 	expStatus := ExperimentStatus{
 		Name:   "Fake exp Name",
 		Status: v1alpha1.ExperimentStatusRunning,
@@ -41,10 +40,9 @@ func TestPatchChaosEngineStatus(t *testing.T) {
 				},
 				Spec: v1alpha1.ChaosEngineSpec{
 					ChaosServiceAccount: fakeServiceAcc,
-					AnnotationCheck:     fakeAnnotationCheck,
 					Appinfo: v1alpha1.ApplicationParams{
-						Applabel: fakeAppLabel,
 						Appns:    engineDetails.EngineNamespace,
+						Applabel: fakeAppLabel,
 						AppKind:  fakeAppKind,
 					},
 				},
@@ -67,10 +65,9 @@ func TestPatchChaosEngineStatus(t *testing.T) {
 				},
 				Spec: v1alpha1.ChaosEngineSpec{
 					ChaosServiceAccount: fakeServiceAcc,
-					AnnotationCheck:     fakeAnnotationCheck,
 					Appinfo: v1alpha1.ApplicationParams{
-						Applabel: fakeAppLabel,
 						Appns:    engineDetails.EngineNamespace,
+						Applabel: fakeAppLabel,
 						AppKind:  fakeAppKind,
 					},
 				},
@@ -123,7 +120,6 @@ func TestUpdateEngineWithResult(t *testing.T) {
 	fakeServiceAcc := "Fake Service Account"
 	fakeAppLabel := "Fake Label"
 	fakeAppKind := "Fake Kind"
-	fakeAnnotationCheck := "Fake Annotation Check"
 	expStatus := ExperimentStatus{
 		Name:   "Fake-Exp-Name",
 		Status: v1alpha1.ExperimentStatusRunning,
@@ -153,10 +149,9 @@ func TestUpdateEngineWithResult(t *testing.T) {
 				},
 				Spec: v1alpha1.ChaosEngineSpec{
 					ChaosServiceAccount: fakeServiceAcc,
-					AnnotationCheck:     fakeAnnotationCheck,
 					Appinfo: v1alpha1.ApplicationParams{
-						Applabel: fakeAppLabel,
 						Appns:    engineDetails.EngineNamespace,
+						Applabel: fakeAppLabel,
 						AppKind:  fakeAppKind,
 					},
 					JobCleanUpPolicy: "retain",
@@ -206,10 +201,9 @@ func TestUpdateEngineWithResult(t *testing.T) {
 				},
 				Spec: v1alpha1.ChaosEngineSpec{
 					ChaosServiceAccount: fakeServiceAcc,
-					AnnotationCheck:     fakeAnnotationCheck,
 					Appinfo: v1alpha1.ApplicationParams{
-						Applabel: fakeAppLabel,
 						Appns:    engineDetails.EngineNamespace,
+						Applabel: fakeAppLabel,
 						AppKind:  fakeAppKind,
 					},
 					JobCleanUpPolicy: "retain",

tests/runner_test.go

@@ -113,7 +113,7 @@ var _ = BeforeSuite(func() {
 	log.Info("Chaos-Operator is in running state")
 
 	By("Installing Pod Delete Experiment")
-	err = exec.Command("kubectl", "apply", "-f", "https://hub.litmuschaos.io/api/chaos/master?file=charts/generic/pod-delete/experiment.yaml", "-n", "litmus").Run()
+	err = exec.Command("kubectl", "apply", "-f", "https://hub.litmuschaos.io/api/chaos/master?file=faults/kubernetes/pod-delete/fault.yaml", "-n", "litmus").Run()
 	Expect(err).To(BeNil(), "unable to create Pod-Delete Experiment")
 	log.Info("pod-delete ChaosExperiment created")
 
@@ -122,7 +122,7 @@ var _ = BeforeSuite(func() {
 	log.Info("pod-delete-sa created")
 })
 
-//BDD Tests to check secondary resources
+// BDD Tests to check secondary resources
 var _ = Describe("BDD on chaos-runner", func() {
 
 	// BDD TEST CASE 1
@@ -206,9 +206,7 @@ var _ = Describe("BDD on chaos-runner", func() {
 					Experiments: []v1alpha1.ExperimentList{
 						{
 							Name: "pod-delete",
-							Spec: v1alpha1.ExperimentAttributes{
-								Rank: uint32(1),
-							},
+							Spec: v1alpha1.ExperimentAttributes{},
 						},
 					},
 				},
@@ -281,7 +279,7 @@ var _ = Describe("BDD on chaos-runner", func() {
 
 })
 
-//Deleting all unused resources
+// Deleting all unused resources
 var _ = AfterSuite(func() {
 
 	By("Deleting chaosengine CRD")