Add a retry functionality to workflows (#2739)

* retry functionality for lifestyle e2e test

Signed-off-by: David Fridrich <fridrich.david19@gmail.com>

* attempt multiple cluster allocs

Signed-off-by: David Fridrich <fridrich.david19@gmail.com>

* delete cluster before retry

Signed-off-by: David Fridrich <fridrich.david19@gmail.com>

* fix

Signed-off-by: David Fridrich <fridrich.david19@gmail.com>

* better output

Signed-off-by: David Fridrich <fridrich.david19@gmail.com>

* retry allocate in all workflows

Signed-off-by: David Fridrich <fridrich.david19@gmail.com>

* add a timer

Signed-off-by: David Fridrich <fridrich.david19@gmail.com>

* add timers to ci

Signed-off-by: David Fridrich <fridrich.david19@gmail.com>

---------

Signed-off-by: David Fridrich <fridrich.david19@gmail.com>
This commit is contained in:
David Fridrich 2025-03-11 13:32:37 +01:00 committed by GitHub
parent e0e79957f1
commit f66b67aa8f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 178 additions and 11 deletions

View File

@ -58,7 +58,24 @@ jobs:
- name: Install Binaries
run: ./hack/install-binaries.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
run: |
attempt=0
max_attempts=5
until [ $attempt -ge $max_attempts ]
do
attempt=$((attempt+1))
echo "------------------ Attempt $attempt ------------------"
./hack/allocate.sh && break
echo "------------------ failed, retrying... ------------------"
if [ $attempt -ge $max_attempts ]; then
echo "------------------ max # of retries reached, exiting ------------------"
exit 1
fi
./hack/delete.sh
echo "------------------ sleep for 5 minutes ------------------"
sleep 300
done
echo "------------------ finished! attempt $attempt ------------------"
- name: Local Registry
run: ./hack/registry.sh
- name: Setup testing images
@ -88,7 +105,24 @@ jobs:
- name: Install Binaries
run: ./hack/install-binaries.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
run: |
attempt=0
max_attempts=5
until [ $attempt -ge $max_attempts ]
do
attempt=$((attempt+1))
echo "------------------ Attempt $attempt ------------------"
./hack/allocate.sh && break
echo "------------------ failed, retrying... ------------------"
if [ $attempt -ge $max_attempts ]; then
echo "------------------ max # of retries reached, exiting ------------------"
exit 1
fi
./hack/delete.sh
echo "------------------ sleep for 5 minutes ------------------"
sleep 300
done
echo "------------------ finished! attempt $attempt ------------------"
- name: Local Registry
run: ./hack/registry.sh
- name: E2E Test
@ -117,7 +151,24 @@ jobs:
- name: Install Binaries
run: ./hack/install-binaries.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
run: |
attempt=0
max_attempts=5
until [ $attempt -ge $max_attempts ]
do
attempt=$((attempt+1))
echo "------------------ Attempt $attempt ------------------"
./hack/allocate.sh && break
echo "------------------ failed, retrying... ------------------"
if [ $attempt -ge $max_attempts ]; then
echo "------------------ max # of retries reached, exiting ------------------"
exit 1
fi
./hack/delete.sh
echo "------------------ sleep for 5 minutes ------------------"
sleep 300
done
echo "------------------ finished! attempt $attempt ------------------"
- name: Setup testing images
run: ./hack/setup-testing-images.sh
- name: Deploy Tekton

View File

@ -22,7 +22,24 @@ jobs:
- name: Install Binaries
run: ./hack/install-binaries.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
run: |
attempt=0
max_attempts=5
until [ $attempt -ge $max_attempts ]
do
attempt=$((attempt+1))
echo "------------------ Attempt $attempt ------------------"
./hack/allocate.sh && break
echo "------------------ failed, retrying... ------------------"
if [ $attempt -ge $max_attempts ]; then
echo "------------------ max # of retries reached, exiting ------------------"
exit 1
fi
./hack/delete.sh
echo "------------------ sleep for 5 minutes ------------------"
sleep 300
done
echo "------------------ finished! attempt $attempt ------------------"
- name: Setup testing images
run: ./hack/setup-testing-images.sh
- name: Deploy Tekton

View File

@ -20,7 +20,24 @@ jobs:
- name: Install Binaries
run: ./hack/install-binaries.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
run: |
attempt=0
max_attempts=5
until [ $attempt -ge $max_attempts ]
do
attempt=$((attempt+1))
echo "------------------ Attempt $attempt ------------------"
./hack/allocate.sh && break
echo "------------------ failed, retrying... ------------------"
if [ $attempt -ge $max_attempts ]; then
echo "------------------ max # of retries reached, exiting ------------------"
exit 1
fi
./hack/delete.sh
echo "------------------ sleep for 5 minutes ------------------"
sleep 300
done
echo "------------------ finished! attempt $attempt ------------------"
- name: Setup testing images
run: ./hack/setup-testing-images.sh
- name: Deploy Tekton

View File

@ -24,13 +24,44 @@ jobs:
- name: Install Binaries
run: ./hack/install-binaries.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
run: |
attempt=0
max_attempts=5
until [ $attempt -ge $max_attempts ]
do
attempt=$((attempt+1))
echo "------------------ Attempt $attempt for ${{matrix.runtime}} ------------------"
./hack/allocate.sh && break
echo "------------------ failed, retrying... ------------------"
if [ $attempt -ge $max_attempts ]; then
echo "------------------ max # of retries reached, exiting ------------------"
exit 1
fi
./hack/delete.sh
echo "------------------ sleep for 5 minutes ------------------"
sleep 300
done
echo "------------------ finished! attempt $attempt ------------------"
- name: Local Registry
run: ./hack/registry.sh
- name: Build
run: make
- name: E2E runtime for ${{ matrix.runtime }}
run: make test-e2e-runtime runtime=${{ matrix.runtime }}
run: |
attempt=0
max_attempts=5
until [ $attempt -ge $max_attempts ]
do
attempt=$((attempt+1))
echo "------------------ Attempt $attempt for ${{matrix.runtime}} ------------------"
make test-e2e-runtime runtime=${{ matrix.runtime }} && break
echo "------------------ failed, retrying... ------------------"
if [ $attempt -ge $max_attempts ]; then
echo "------------------ max # of retries reached, exiting ------------------"
exit 1
fi
done
echo "------------------ finished! attempt $attempt ------------------"
- uses: codecov/codecov-action@v5
with:
files: ./coverage.txt

View File

@ -19,7 +19,24 @@ jobs:
- name: Install Binaries
run: ./hack/install-binaries.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
run: |
attempt=0
max_attempts=5
until [ $attempt -ge $max_attempts ]
do
attempt=$((attempt+1))
echo "------------------ Attempt $attempt ------------------"
./hack/allocate.sh && break
echo "------------------ failed, retrying... ------------------"
if [ $attempt -ge $max_attempts ]; then
echo "------------------ max # of retries reached, exiting ------------------"
exit 1
fi
./hack/delete.sh
echo "------------------ sleep for 5 minutes ------------------"
sleep 300
done
echo "------------------ finished! attempt $attempt ------------------"
- name: Local Registry
run: ./hack/registry.sh
- name: E2E Test

View File

@ -29,10 +29,27 @@ jobs:
- uses: imjasonh/setup-ko@v0.6
- name: Install Binaries
run: ./hack/install-binaries.sh
- name: Allocate Cluster
run: |
attempt=0
max_attempts=5
until [ $attempt -ge $max_attempts ]
do
attempt=$((attempt+1))
echo "------------------ Attempt $attempt ------------------"
./hack/allocate.sh && break
echo "------------------ failed, retrying... ------------------"
if [ $attempt -ge $max_attempts ]; then
echo "------------------ max # of retries reached, exiting ------------------"
exit 1
fi
./hack/delete.sh
echo "------------------ sleep for 5 minutes ------------------"
sleep 300
done
echo "------------------ finished! attempt $attempt ------------------"
- name: Local Registry
run: ./hack/registry.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
- name: Setup testing images
run: ./hack/setup-testing-images.sh
- name: Install Tekton

View File

@ -24,7 +24,24 @@ jobs:
- name: Install Binaries
run: ./hack/install-binaries.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
run: |
attempt=0
max_attempts=5
until [ $attempt -ge $max_attempts ]
do
attempt=$((attempt+1))
echo "------------------ Attempt $attempt ------------------"
./hack/allocate.sh && break
echo "------------------ failed, retrying... ------------------"
if [ $attempt -ge $max_attempts ]; then
echo "------------------ max # of retries reached, exiting ------------------"
exit 1
fi
./hack/delete.sh
echo "------------------ sleep for 5 minutes ------------------"
sleep 300
done
echo "------------------ finished! attempt $attempt ------------------"
- name: Local Registry
run: ./hack/registry.sh
- name: Setup testing images