diff --git a/upup/pkg/fi/BUILD.bazel b/upup/pkg/fi/BUILD.bazel
index cf87f577b8..021c74f44a 100644
--- a/upup/pkg/fi/BUILD.bazel
+++ b/upup/pkg/fi/BUILD.bazel
@@ -73,6 +73,7 @@ go_test(
     ],
     embed = [":go_default_library"],
     deps = [
+        "//pkg/apis/kops:go_default_library",
         "//pkg/pki:go_default_library",
         "//util/pkg/vfs:go_default_library",
     ],
diff --git a/upup/pkg/fi/vfs_castore_test.go b/upup/pkg/fi/vfs_castore_test.go
index b14cf6b488..71e95afb1e 100644
--- a/upup/pkg/fi/vfs_castore_test.go
+++ b/upup/pkg/fi/vfs_castore_test.go
@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/pki"
 	"k8s.io/kops/util/pkg/vfs"
 )
@@ -214,4 +215,20 @@ spec:
 		}
 	}
 
+	// Check that keyset gets deleted
+	{
+		keyset := &kops.Keyset{}
+		keyset.Name = "ca"
+		keyset.Spec.Type = kops.SecretTypeKeypair
+
+		s.DeleteKeysetItem(keyset, "237054359138908419352140518924933177492")
+
+		_, err := pathMap["memfs://tests/private/ca/237054359138908419352140518924933177492.key"].ReadFile()
+		pathMap["memfs://tests/private/ca/237054359138908419352140518924933177492.key"].ReadFile()
+		if err == nil {
+			t.Fatalf("File memfs://tests/private/ca/237054359138908419352140518924933177492.key still exists")
+		}
+
+	}
+
 }