kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test(components): Reduce sagemaker component test flakiness (#10225) 

Signed-off-by: ananth102 <abashyam@amazon.com>
This commit is contained in:
ananth102 2024-02-14 11:29:10 -08:00 committed by GitHub
parent 3f0fc06295
commit 664deaf933
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
components/aws/sagemaker/tests/integration_tests/scripts/run_integration_tests
View File

@ -38,11 +38,12 @@ ROBOMAKER_EXECUTION_ROLE_ARN=${ROBOMAKER_EXECUTION_ROLE_ARN:-""}
SKIP_FSX_TESTS=${SKIP_FSX_TESTS:-"false"}
ACK_RELEASE_VERSION=${ACK_RELEASE_VERSION:-"v1.2.1"}
ACK_RELEASE_VERSION=${ACK_RELEASE_VERSION:-"1.2.4"}
HELM_EXPERIMENTAL_OCI=1
SERVICE=sagemaker
CHART_EXPORT_PATH=/tmp/chart
CHART_REF=sagemaker-chart
TEST_EXIT_STATUS=1
while getopts ":n:r:s:" opt; do
case $opt in
@ -88,10 +89,6 @@ fi
function cleanup() {
set +e
#push to metrics to cloudwatch
echo "Pushing Codebuild stats to Cloudwatch."
python ../../codebuild/scripts/push_stats_to_cloudwatch.py
cleanup_kfp
# If installation fails before ack installation resources should be freed.
if [[ -v ACK_K8S_NAMESPACE ]]; then
@ -101,6 +98,10 @@ function cleanup() {
[ "${SKIP_KFP_OIDC_SETUP}" == "false" ] && delete_oidc_role
#push to metrics to cloudwatch
echo "Pushing Codebuild stats to Cloudwatch."
python ../../codebuild/scripts/push_stats_to_cloudwatch.py
if [[ "${SKIP_FSX_TESTS}" == "false" ]]; then
delete_fsx_instance
# Sleep in order for the security group to detach before attempting to delete it
@ -280,6 +281,10 @@ function cleanup_kfp() {
# If this fails, deleting the nodegroup later will clean it up anyway
kill -9 $MINIO_PID || true
fi
if [[ $TEST_EXIT_STATUS -gt 0 ]]; then
kubectl delete -k "github.com/kubeflow/pipelines/manifests/kustomize/env/cert-manager/dev?ref=$KFP_VERSION&timeout=90s"
kubectl delete -k "github.com/kubeflow/pipelines/manifests/kustomize/env/cert-manager/cluster-scoped-resources?ref=$KFP_VERSION&timeout=90s"
fi
}
if [[ -z "${EKS_EXISTING_CLUSTER}" ]]; then
@ -332,4 +337,6 @@ fi
DIR_THIS_FILE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd $DIR_THIS_FILE/../ && python -m pytest "${pytest_args[@]}" --junitxml ./integration_tests.xml -n 9
cd $DIR_THIS_FILE/../
python -m pytest "${pytest_args[@]}" --junitxml ./integration_tests.xml -n 9
TEST_EXIT_STATUS=$?