Merge pull request #65027 from liggitt/etcd-health-check

Automatic merge from submit-queue (batch tested with PRs 64140, 64898, 65022, 65037, 65027). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Use actual etcd client for /healthz/etcd checks

* avoids redialing etcd on every health check (which makes slow DNS a false-positive healthz failure)
* ensures etcd TLS setup is correct (errors verifying the etcd API or sending client credentials manifest as healthz failures)
* ensures the etcd cluster is actually responsive

fixes #64909

```release-note
Etcd health checks by the apiserver now ensure the apiserver can connect to and exercise the etcd API
```

Kubernetes-commit: 9d97913e75059a2ab7ed5488c9dd42bff2a23836

Godeps/Godeps.json

@@ -1,6 +1,6 @@
 {
 	"ImportPath": "k8s.io/apiserver",
-	"GoVersion": "go1.10",
+	"GoVersion": "go1.9",
 	"GodepVersion": "v80",
 	"Packages": [
 		"./..."
@@ -832,331 +832,331 @@
 		},
 		{
 			"ImportPath": "k8s.io/api/admission/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/admissionregistration/v1alpha1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/admissionregistration/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/apps/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/apps/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/apps/v1beta2",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/authentication/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/authentication/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/authorization/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/authorization/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/autoscaling/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/autoscaling/v2beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/batch/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/batch/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/batch/v2alpha1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/certificates/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/core/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/events/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/extensions/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/networking/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/policy/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/rbac/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/rbac/v1alpha1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/rbac/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/scheduling/v1alpha1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/scheduling/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/settings/v1alpha1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/storage/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/storage/v1alpha1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/api/storage/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "0629ca67c7adeabf4d1bc63cb816ce9e0d658406"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/equality",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/errors",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/meta",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/resource",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/testing",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/testing/fuzzer",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/testing/roundtrip",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/validation",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/validation/path",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/fuzzer",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/internalversion",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1/validation",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/conversion",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/conversion/queryparams",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/fields",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/labels",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/schema",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/json",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/protobuf",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/recognizer",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/streaming",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/versioning",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/selection",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/types",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/cache",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/clock",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/diff",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/errors",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/framer",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/intstr",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/json",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/mergepatch",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/net",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/rand",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/runtime",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/sets",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/strategicpatch",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/uuid",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/validation",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/validation/field",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/wait",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/waitgroup",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/yaml",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/version",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/watch",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/json",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+			"Rev": "954ae1c14eaec8ab19f63260d42de50e9ecc9961"
 		},
 		{
 			"ImportPath": "k8s.io/client-go/discovery",
@@ -1166,6 +1166,10 @@
 			"ImportPath": "k8s.io/client-go/discovery/fake",
 			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
 		},
+		{
+			"ImportPath": "k8s.io/client-go/informers",
+			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+		},
 		{
 			"ImportPath": "k8s.io/client-go/informers/admissionregistration",
 			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
@@ -1330,6 +1334,10 @@
 			"ImportPath": "k8s.io/client-go/kubernetes",
 			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
 		},
+		{
+			"ImportPath": "k8s.io/client-go/kubernetes/fake",
+			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+		},
 		{
 			"ImportPath": "k8s.io/client-go/kubernetes/scheme",
 			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
@@ -1706,6 +1714,10 @@
 			"ImportPath": "k8s.io/client-go/tools/cache",
 			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
 		},
+		{
+			"ImportPath": "k8s.io/client-go/tools/clientcmd",
+			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
+		},
 		{
 			"ImportPath": "k8s.io/client-go/tools/clientcmd/api",
 			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
@@ -1781,78 +1793,6 @@
 		{
 			"ImportPath": "k8s.io/kube-openapi/pkg/util/proto",
 			"Rev": "91cfa479c814065e420cee7ed227db0f63a5854e"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/discovery",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/informers",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/kubernetes",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/kubernetes/fake",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/kubernetes/scheme",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/kubernetes/typed/authentication/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/kubernetes/typed/authorization/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/kubernetes/typed/core/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/listers/admissionregistration/v1beta1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/listers/core/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/rest",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/testing",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/tools/cache",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/tools/clientcmd",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/tools/clientcmd/api",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/tools/clientcmd/api/v1",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/util/cert",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
-		},
-		{
-			"ImportPath": "k8s.io/client-go/util/flowcontrol",
-			"Rev": "2a94c229c06da7d798878d0bbef74e58c08d44be"
 		}
 	]
 }

pkg/server/options/etcd.go

@@ -32,8 +32,8 @@ import (
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/healthz"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
-	"k8s.io/apiserver/pkg/storage/etcd3/preflight"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
+	storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
 )
 
 type EtcdOptions struct {
@@ -181,29 +181,30 @@ func (s *EtcdOptions) ApplyTo(c *server.Config) error {
 	if s == nil {
 		return nil
 	}
-
-	s.addEtcdHealthEndpoint(c)
+	if err := s.addEtcdHealthEndpoint(c); err != nil {
+		return err
+	}
 	c.RESTOptionsGetter = &SimpleRestOptionsFactory{Options: *s}
 	return nil
 }
 
 func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error {
-	s.addEtcdHealthEndpoint(c)
+	if err := s.addEtcdHealthEndpoint(c); err != nil {
+		return err
+	}
 	c.RESTOptionsGetter = &storageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
 	return nil
 }
 
-func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) {
-	c.HealthzChecks = append(c.HealthzChecks, healthz.NamedCheck("etcd", func(r *http.Request) error {
-		done, err := preflight.EtcdConnection{ServerList: s.StorageConfig.ServerList}.CheckEtcdServers()
-		if !done {
-			return fmt.Errorf("etcd failed")
-		}
+func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
+	healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig)
 	if err != nil {
 		return err
 	}
-		return nil
+	c.HealthzChecks = append(c.HealthzChecks, healthz.NamedCheck("etcd", func(r *http.Request) error {
+		return healthCheck()
 	}))
+	return nil
 }
 
 type SimpleRestOptionsFactory struct {

pkg/storage/storagebackend/factory/etcd2.go

@@ -17,6 +17,8 @@ limitations under the License.
 package factory
 
 import (
+	"context"
+	"fmt"
 	"net"
 	"net/http"
 	"time"
@@ -30,6 +32,29 @@ import (
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 )
 
+func newETCD2HealthCheck(c storagebackend.Config) (func() error, error) {
+	tr, err := newTransportForETCD2(c.CertFile, c.KeyFile, c.CAFile)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := newETCD2Client(tr, c.ServerList)
+	if err != nil {
+		return nil, err
+	}
+
+	members := etcd2client.NewMembersAPI(client)
+
+	return func() error {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		if _, err := members.List(ctx); err != nil {
+			return fmt.Errorf("error listing etcd members: %v", err)
+		}
+		return nil
+	}, nil
+}
+
 func newETCD2Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
 	tr, err := newTransportForETCD2(c.CertFile, c.KeyFile, c.CAFile)
 	if err != nil {

pkg/storage/storagebackend/factory/etcd3.go

@@ -18,11 +18,14 @@ package factory
 
 import (
 	"context"
+	"fmt"
+	"sync/atomic"
 	"time"
 
 	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/pkg/transport"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/storage"
 	"k8s.io/apiserver/pkg/storage/etcd3"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
@@ -38,7 +41,41 @@ var (
 	dialTimeout = 10 * time.Second
 )
 
-func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
+func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
+	// constructing the etcd v3 client blocks and times out if etcd is not available.
+	// retry in a loop in the background until we successfully create the client, storing the client or error encountered
+
+	clientValue := &atomic.Value{}
+
+	clientErrMsg := &atomic.Value{}
+	clientErrMsg.Store("etcd client connection not yet established")
+
+	go wait.PollUntil(time.Second, func() (bool, error) {
+		client, err := newETCD3Client(c)
+		if err != nil {
+			clientErrMsg.Store(err.Error())
+			return false, nil
+		}
+		clientValue.Store(client)
+		clientErrMsg.Store("")
+		return true, nil
+	}, wait.NeverStop)
+
+	return func() error {
+		if errMsg := clientErrMsg.Load().(string); len(errMsg) > 0 {
+			return fmt.Errorf(errMsg)
+		}
+		client := clientValue.Load().(*clientv3.Client)
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		if _, err := client.Cluster.MemberList(ctx); err != nil {
+			return fmt.Errorf("error listing etcd members: %v", err)
+		}
+		return nil
+	}, nil
+}
+
+func newETCD3Client(c storagebackend.Config) (*clientv3.Client, error) {
 	tlsInfo := transport.TLSInfo{
 		CertFile: c.CertFile,
 		KeyFile:  c.KeyFile,
@@ -46,7 +83,7 @@ func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, e
 	}
 	tlsConfig, err := tlsInfo.ClientConfig()
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	// NOTE: Client relies on nil tlsConfig
 	// for non-secure connections, update the implicit variable
@@ -61,6 +98,11 @@ func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, e
 		TLS:                  tlsConfig,
 	}
 	client, err := clientv3.New(cfg)
+	return client, err
+}
+
+func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
+	client, err := newETCD3Client(c)
 	if err != nil {
 		return nil, nil, err
 	}

pkg/storage/storagebackend/factory/factory.go

@@ -41,3 +41,15 @@ func Create(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
 		return nil, nil, fmt.Errorf("unknown storage type: %s", c.Type)
 	}
 }
+
+// CreateHealthCheck creates a healthcheck function based on given config.
+func CreateHealthCheck(c storagebackend.Config) (func() error, error) {
+	switch c.Type {
+	case storagebackend.StorageTypeETCD2:
+		return newETCD2HealthCheck(c)
+	case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3:
+		return newETCD3HealthCheck(c)
+	default:
+		return nil, fmt.Errorf("unknown storage type: %s", c.Type)
+	}
+}