diff --git a/pkg/endpoints/handlers/fieldmanager/scalehandler.go b/pkg/endpoints/handlers/fieldmanager/scalehandler.go
index 114cdb184..4f7296b88 100644
--- a/pkg/endpoints/handlers/fieldmanager/scalehandler.go
+++ b/pkg/endpoints/handlers/fieldmanager/scalehandler.go
@@ -135,6 +135,9 @@ func (h *ScaleHandler) ToParent(scaleEntries []metav1.ManagedFieldsEntry) ([]met
 	}
 
 	for manager, versionedSet := range scaleFields {
+		if !versionedSet.Set().Has(replicasPathInScale) {
+			continue
+		}
 		newVersionedSet := fieldpath.NewVersionedSet(
 			fieldpath.NewSet(h.mappings[h.groupVersion.String()]),
 			fieldpath.APIVersion(h.groupVersion.String()),
diff --git a/pkg/endpoints/handlers/fieldmanager/scalehandler_test.go b/pkg/endpoints/handlers/fieldmanager/scalehandler_test.go
index 6ea8a8c76..291f32fb3 100644
--- a/pkg/endpoints/handlers/fieldmanager/scalehandler_test.go
+++ b/pkg/endpoints/handlers/fieldmanager/scalehandler_test.go
@@ -463,7 +463,7 @@ func TestTransformingManagedFieldsToParent(t *testing.T) {
 					Operation:  metav1.ManagedFieldsOperationApply,
 					APIVersion: "apps/v1",
 					FieldsType: "FieldsV1",
-					FieldsV1:   &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:replicas":{},"f:selector":{}}}`)},
+					FieldsV1:   &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:selector":{}}}`)},
 				},
 			},
 			subresource: []metav1.ManagedFieldsEntry{
@@ -472,7 +472,7 @@ func TestTransformingManagedFieldsToParent(t *testing.T) {
 					Operation:   metav1.ManagedFieldsOperationUpdate,
 					APIVersion:  "autoscaling/v1",
 					FieldsType:  "FieldsV1",
-					FieldsV1:    &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:another":{}}}`)},
+					FieldsV1:    &metav1.FieldsV1{Raw: []byte(`{"f:status":{"f:replicas":{}}}`)},
 					Subresource: "scale",
 				},
 			},
@@ -484,14 +484,6 @@ func TestTransformingManagedFieldsToParent(t *testing.T) {
 					FieldsType: "FieldsV1",
 					FieldsV1:   &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:selector":{}}}`)},
 				},
-				{
-					Manager:     "scale",
-					Operation:   metav1.ManagedFieldsOperationUpdate,
-					APIVersion:  "apps/v1",
-					FieldsType:  "FieldsV1",
-					FieldsV1:    &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:replicas":{}}}`)},
-					Subresource: "scale",
-				},
 			},
 		},
 		{