diff --git a/pkg/client/simple/vfsclientset/clientset.go b/pkg/client/simple/vfsclientset/clientset.go
index 46b72b381b..4ab2cc4d64 100644
--- a/pkg/client/simple/vfsclientset/clientset.go
+++ b/pkg/client/simple/vfsclientset/clientset.go
@@ -201,6 +201,25 @@ func deleteAllPaths(basePath vfs.Path) error {
 	return nil
 }
 func (c *VFSClientset) DeleteCluster(ctx context.Context, cluster *kops.Cluster) error {
+	if cluster.Spec.ServiceAccountIssuerDiscovery != nil {
+		discoveryStore := cluster.Spec.ServiceAccountIssuerDiscovery.DiscoveryStore
+		if discoveryStore != "" {
+			path, err := vfs.Context.BuildVfsPath(discoveryStore)
+			if err != nil {
+				return err
+			}
+
+			err = path.Join("openid/v1/jwks").Remove()
+			if err != nil {
+				return err
+			}
+			err = path.Join(".well-known/openid-configuration").Remove()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	secretStore := cluster.Spec.SecretStore
 	if secretStore != "" {
 		path, err := vfs.Context.BuildVfsPath(secretStore)