generator: recover from kustomize build panics

Signed-off-by: Hidde Beydals <hello@hidde.co>

controllers/kustomization_generator.go

@@ -247,7 +247,7 @@ var kustomizeBuildMutex sync.Mutex
 //  - load files from outside the kustomization dir path
 //    (but not outside root)
 //  - disable plugins except for the builtin ones
-func secureBuildKustomization(root, dirPath string) (resmap.ResMap, error) {
+func secureBuildKustomization(root, dirPath string) (_ resmap.ResMap, err error) {
 	// Create secure FS for root
 	fs, err := securefs.MakeFsOnDiskSecureBuild(root)
 	if err != nil {
@@ -259,6 +259,15 @@ func secureBuildKustomization(root, dirPath string) (resmap.ResMap, error) {
 	kustomizeBuildMutex.Lock()
 	defer kustomizeBuildMutex.Unlock()
 
+	// Kustomize tends to panic in unpredicted ways due to (accidental)
+	// invalid object data; recover when this happens to ensure continuity of
+	// operations
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("recovered from kustomize build panic: %v", r)
+		}
+	}()
+
 	buildOptions := &krusty.Options{
 		LoadRestrictions: kustypes.LoadRestrictionsNone,
 		PluginConfig:     kustypes.DisabledPluginConfig(),

controllers/kustomization_generator_test.go

@@ -36,6 +36,10 @@ func Test_secureBuildKustomization_panic(t *testing.T) {
 		g := NewWithT(t)
 
 		_, err := secureBuildKustomization("testdata/panic", "testdata/panic")
-		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(err).To(HaveOccurred())
+		g.Expect(err.Error()).To(ContainSubstring("recovered from kustomize build panic"))
+		// Run again to ensure the lock is released
+		_, err = secureBuildKustomization("testdata/panic", "testdata/panic")
+		g.Expect(err).To(HaveOccurred())
 	})
 }