From dcc4a73e498df7455ff292ffad46b2781cb81a7a Mon Sep 17 00:00:00 2001
From: Catherine Luse <catherine.luse@gmail.com>
Date: Fri, 13 Jan 2023 17:48:36 -0700
Subject: [PATCH] Allow more retries to finish project creation

---
 shell/models/management.cattle.io.project.js | 38 ++++++++++++++------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/shell/models/management.cattle.io.project.js b/shell/models/management.cattle.io.project.js
index 5727372817..1540a172b9 100644
--- a/shell/models/management.cattle.io.project.js
+++ b/shell/models/management.cattle.io.project.js
@@ -99,18 +99,36 @@ export default class Project extends HybridModel {
     // and the PUT request should have a query param _replace=true.
     const newValue = await norman.save({ replace: forceReplaceOnReq });
 
-    try {
-      await newValue.doAction('setpodsecuritypolicytemplate', { podSecurityPolicyTemplateId: this.spec.podSecurityPolicyTemplateId || null });
-    } catch (err) {
-      if ( err.status === 409 || err.status === 403 ) {
-        // The backend updates each new project soon after it is created,
-        // so there is a chance of a resource conflict or forbidden error. If that happens,
-        // retry the action.
+    let retryCount = 0;
+
+    const finishProjectCreation = async() => {
+      try {
         await newValue.doAction('setpodsecuritypolicytemplate', { podSecurityPolicyTemplateId: this.spec.podSecurityPolicyTemplateId || null });
-      } else {
-        throw err;
+      } catch (err) {
+        if ( err.status === 409 || err.status === 403 ) {
+          // The backend updates each new project soon after it is created,
+          // so there is a chance of a 409 resource conflict or forbidden error. If that happens,
+          // retry the action.
+
+          // The 403 permission error can happen due to the user requesting
+          // the project before the project is fully created and the project
+          // permissions are assigned to the user so we allow some
+          // time for that process to complete.
+
+          if (retryCount < 3) {
+            retryCount++;
+            await setTimeout(() => {
+              // Delay for three seconds to avoid another failure due to latency.
+              finishProjectCreation();
+            }, '3000');
+          } else {
+            throw err;
+          }
+        }
       }
-    }
+    };
+
+    await finishProjectCreation();
 
     return newValue;
   }