kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9069 from justinsb/healthcheck 

kube-apiserver: healthcheck via sidecar container
This commit is contained in:
Kubernetes Prow Robot 2020-05-07 08:39:08 -07:00 committed by GitHub
parent 3df14f2bea 75fd939a62
commit c73659561e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1112 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Makefile
View File

@ -56,6 +56,8 @@ unexport SKIP_REGION_CHECK S3_ACCESS_KEY_ID S3_ENDPOINT S3_REGION S3_SECRET_ACCE
DNS_CONTROLLER_TAG=1.18.0-alpha.3
# Keep in sync with upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/
KOPS_CONTROLLER_TAG=1.18.0-alpha.3
# Keep in sync with pkg/model/components/kubeapiserver/model.go
KUBE_APISERVER_HEALTHCHECK_TAG=1.18.0-alpha.3
# Keep in sync with logic in get_workspace_status
# TODO: just invoke tools/get_workspace_status.sh?
@ -732,6 +734,13 @@ bazel-protokube-export:
cp -fp bazel-bin/images/protokube.tar.gz.sha1 ${BAZELIMAGES}/protokube.tar.gz.sha1
cp -fp bazel-bin/images/protokube.tar.gz.sha256 ${BAZELIMAGES}/protokube.tar.gz.sha256
.PHONY: bazel-kube-apiserver-healthcheck-export
bazel-kube-apiserver-healthcheck-export:
mkdir -p ${BAZELIMAGES}
DOCKER_REGISTRY="" DOCKER_IMAGE_PREFIX="kope/" KUBE_APISERVER_HEALTHCHECK_TAG=${KUBE_APISERVER_HEALTHCHECK_TAG} bazel build ${BAZEL_CONFIG} --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //cmd/kube-apiserver-healthcheck:image-bundle.tar.gz //cmd/kube-apiserver-healthcheck:image-bundle.tar.gz.sha256
cp -fp bazel-bin/cmd/kube-apiserver-healthcheck/image-bundle.tar.gz ${BAZELIMAGES}/kube-apiserver-healthcheck.tar.gz
cp -fp bazel-bin/cmd/kube-apiserver-healthcheck/image-bundle.tar.gz.sha256 ${BAZELIMAGES}/kube-apiserver-healthcheck.tar.gz.sha256
.PHONY: bazel-kops-controller-export
bazel-kops-controller-export:
mkdir -p ${BAZELIMAGES}
@ -749,7 +758,7 @@ bazel-dns-controller-export:
cp -fp bazel-bin/dns-controller/cmd/dns-controller/image-bundle.tar.gz.sha256 ${BAZELIMAGES}/dns-controller.tar.gz.sha256
.PHONY: bazel-version-dist
bazel-version-dist: bazel-crossbuild-nodeup bazel-crossbuild-kops bazel-kops-controller-export bazel-dns-controller-export bazel-protokube-export bazel-utils-dist
bazel-version-dist: bazel-crossbuild-nodeup bazel-crossbuild-kops bazel-kops-controller-export bazel-kube-apiserver-healthcheck-export bazel-dns-controller-export bazel-protokube-export bazel-utils-dist
rm -rf ${BAZELUPLOAD}
mkdir -p ${BAZELUPLOAD}/kops/${VERSION}/linux/amd64/
mkdir -p ${BAZELUPLOAD}/kops/${VERSION}/darwin/amd64/
@ -765,6 +774,8 @@ bazel-version-dist: bazel-crossbuild-nodeup bazel-crossbuild-kops bazel-kops-con
cp ${BAZELIMAGES}/kops-controller.tar.gz ${BAZELUPLOAD}/kops/${VERSION}/images/kops-controller.tar.gz
cp ${BAZELIMAGES}/kops-controller.tar.gz.sha1 ${BAZELUPLOAD}/kops/${VERSION}/images/kops-controller.tar.gz.sha1
cp ${BAZELIMAGES}/kops-controller.tar.gz.sha256 ${BAZELUPLOAD}/kops/${VERSION}/images/kops-controller.tar.gz.sha256
cp ${BAZELIMAGES}/kube-apiserver-healthcheck.tar.gz ${BAZELUPLOAD}/kops/${VERSION}/images/kube-apiserver-healthcheck.tar.gz
cp ${BAZELIMAGES}/kube-apiserver-healthcheck.tar.gz.sha256 ${BAZELUPLOAD}/kops/${VERSION}/images/kube-apiserver-healthcheck.tar.gz.sha256
cp ${BAZELIMAGES}/dns-controller.tar.gz ${BAZELUPLOAD}/kops/${VERSION}/images/dns-controller.tar.gz
cp ${BAZELIMAGES}/dns-controller.tar.gz.sha1 ${BAZELUPLOAD}/kops/${VERSION}/images/dns-controller.tar.gz.sha1
cp ${BAZELIMAGES}/dns-controller.tar.gz.sha256 ${BAZELUPLOAD}/kops/${VERSION}/images/dns-controller.tar.gz.sha256
@ -856,6 +867,14 @@ dev-upload-dns-controller: bazel-dns-controller-export # Upload kops to GCS
cp -fp ${BAZELIMAGES}/dns-controller.tar.gz.sha256 ${BAZELUPLOAD}/kops/${VERSION}/images/dns-controller.tar.gz.sha256
${UPLOAD_CMD} ${BAZELUPLOAD}/ ${UPLOAD_DEST}
# dev-upload-kube-apiserver-healthcheck uploads kube-apiserver-healthcheck to GCS
.PHONY: dev-upload-kube-apiserver-healthcheck
dev-upload-kube-apiserver-healthcheck: bazel-kube-apiserver-healthcheck-export # Upload kops to GCS
mkdir -p ${BAZELUPLOAD}/kops/${VERSION}/images/
cp -fp ${BAZELIMAGES}/kube-apiserver-healthcheck.tar.gz ${BAZELUPLOAD}/kops/${VERSION}/images/kube-apiserver-healthcheck.tar.gz
cp -fp ${BAZELIMAGES}/kube-apiserver-healthcheck.tar.gz.sha256 ${BAZELUPLOAD}/kops/${VERSION}/images/kube-apiserver-healthcheck.tar.gz.sha256
${UPLOAD_CMD} ${BAZELUPLOAD}/ ${UPLOAD_DEST}
# dev-copy-utils copies utils from a recent release
# We don't currently have a bazel build for them, and the build is pretty slow, but they change rarely.
.PHONE: dev-copy-utils
@ -869,7 +888,7 @@ dev-copy-utils:
# dev-upload does a faster build and uploads to GCS / S3
# It copies utils instead of building it
.PHONY: dev-upload
dev-upload: dev-upload-nodeup dev-upload-protokube dev-upload-dns-controller dev-upload-kops-controller dev-copy-utils
dev-upload: dev-upload-nodeup dev-upload-protokube dev-upload-dns-controller dev-upload-kops-controller dev-copy-utils dev-upload-kube-apiserver-healthcheck
echo "Done"
.PHONY: crds
@ -882,3 +901,10 @@ crds:
.PHONY: kops-controller-push
kops-controller-push:
DOCKER_REGISTRY=${DOCKER_REGISTRY} DOCKER_IMAGE_PREFIX=${DOCKER_IMAGE_PREFIX} KOPS_CONTROLLER_TAG=${KOPS_CONTROLLER_TAG} bazel run --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //cmd/kops-controller:push-image
#------------------------------------------------------
# kube-apiserver-healthcheck
.PHONY: kube-apiserver-healthcheck-push
kube-apiserver-healthcheck-push:
DOCKER_REGISTRY=${DOCKER_REGISTRY} DOCKER_IMAGE_PREFIX=${DOCKER_IMAGE_PREFIX} KUBE_APISERVER_HEALTHCHECK_TAG=${KUBE_APISERVER_HEALTHCHECK_TAG} bazel run --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //cmd/kube-apiserver-healthcheck:push-image

70
cmd/kube-apiserver-healthcheck/BUILD.bazel Normal file
View File

@ -0,0 +1,70 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = ["main.go"],
importpath = "k8s.io/kops/cmd/kube-apiserver-healthcheck",
visibility = ["//visibility:private"],
deps = ["//vendor/k8s.io/klog:go_default_library"],
)
go_binary(
name = "kube-apiserver-healthcheck",
embed = [":go_default_library"],
visibility = ["//visibility:public"],
)
load(
"@io_bazel_rules_docker//container:container.bzl",
"container_image",
"container_push",
"container_bundle",
)
container_image(
name = "image",
base = "@distroless_base//image",
cmd = ["/usr/bin/kube-apiserver-healthcheck"],
user = "10012",
directory = "/usr/bin/",
files = [
"//cmd/kube-apiserver-healthcheck",
],
stamp = True,
)
container_push(
name = "push-image",
format = "Docker",
image = ":image",
registry = "{STABLE_DOCKER_REGISTRY}",
repository = "{STABLE_DOCKER_IMAGE_PREFIX}kube-apiserver-healthcheck",
tag = "{STABLE_KUBE_APISERVER_HEALTHCHECK_TAG}",
)
container_bundle(
name = "image-bundle",
images = {
"{STABLE_DOCKER_IMAGE_PREFIX}kube-apiserver-healthcheck:{STABLE_KUBE_APISERVER_HEALTHCHECK_TAG}": "image",
},
)
load("//tools:gzip.bzl", "gzip")
gzip(
name = "image-bundle.tar.gz",
src = "image-bundle.tar",
)
load("//tools:hashes.bzl", "hashes")
hashes(
name = "image-bundle.tar.gz.hashes",
src = "image-bundle.tar.gz",
)
go_test(
name = "go_default_test",
srcs = ["proxy_test.go"],
embed = [":go_default_library"],
)

18
cmd/kube-apiserver-healthcheck/README.md Normal file
View File

@ -0,0 +1,18 @@
## kube-apiserver-healthcheck
This is a small sidecar container that allows for health-checking the
kube-apiserver without enabling anonymous authentication and without
enabling the unauthenticated port.
It listens on port 8080 (http), and proxies a few known-safe requests
to the real apiserver listening on 443. It uses a client certificate
to authenticate itself to the apiserver.
This lets us turn off the unauthenticated kube-apiserver endpoint, but
it also lets us have better load-balancer health-checks.
Because it runs as a sidecar next to kube-apiserver, it is in the same
network namespace, and thus it can reach apiserver on
https://127.0.0.1 . The kube-apiserver-healthcheck process listens on
8080, but the health checks for the apiserver container are configured
for :8080 and actually go via the sidecar.

191
cmd/kube-apiserver-healthcheck/main.go Normal file
View File

@ -0,0 +1,191 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"crypto/tls"
"crypto/x509"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"k8s.io/klog"
)
// healthCheckServer is the http server
type healthCheckServer struct {
transport *http.Transport
}
// handler processes a single http request
func (s *healthCheckServer) handler(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" && r.URL.Path == "/.kube-apiserver-healthcheck/healthz" {
// This is a check for our own health
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
return
}
if proxyRequest := mapToProxyRequest(r); proxyRequest != nil {
s.proxyRequest(w, proxyRequest)
return
}
klog.Infof("unknown request: %s %s", r.Method, r.URL.Path)
http.Error(w, "not found", http.StatusNotFound)
}
// httpClient builds an isolated http.Client
func (s *healthCheckServer) httpClient() *http.Client {
return &http.Client{Transport: s.transport}
}
// mapToProxyRequest returns the request we should make to the apiserver,
// or nil if the query is not on the safelist
func mapToProxyRequest(r *http.Request) *http.Request {
if r.Method == "GET" {
switch r.URL.Path {
case "/livez", "/healthz", "/readyz":
// This is a health-check we will proxy
return sanitizeRequest(r, []string{"exclude"})
}
}
return nil
}
// sanitizeRequest builds the request we should pass to the target apiserver,
// passing through only allowedQueryParameters
func sanitizeRequest(r *http.Request, allowedQueryParameters []string) *http.Request {
u := &url.URL{
Scheme: "https",
Host: "127.0.0.1",
Path: r.URL.Path,
}
// Pass-through (only) the parameters in allowedQueryParameters
{
in := r.URL.Query()
out := make(url.Values)
for _, k := range allowedQueryParameters {
for _, v := range in[k] {
out.Add(k, v)
}
}
u.RawQuery = out.Encode()
}
req := &http.Request{
Method: r.Method,
URL: u,
}
return req
}
// proxyRequest forwards a request, that has been sanitized by mapToProxyRequest/buildProxyRequest
func (s *healthCheckServer) proxyRequest(w http.ResponseWriter, forwardRequest *http.Request) {
httpClient := s.httpClient()
resp, err := httpClient.Do(forwardRequest)
if err != nil {
klog.Infof("error from %s: %v", forwardRequest.URL, err)
http.Error(w, "internal error", http.StatusBadGateway)
return
}
defer resp.Body.Close()
w.WriteHeader(resp.StatusCode)
if _, err := io.Copy(w, resp.Body); err != nil {
klog.Warningf("error writing response body: %v", err)
return
}
switch resp.StatusCode {
case 200:
klog.V(2).Infof("proxied to %s %s: %s", forwardRequest.Method, forwardRequest.URL, resp.Status)
default:
klog.Infof("proxied to %s %s: %s", forwardRequest.Method, forwardRequest.URL, resp.Status)
}
}
func run() error {
listen := ":8080"
clientCert := ""
clientKey := ""
caCert := ""
flag.StringVar(&clientCert, "client-cert", clientCert, "path to client certificate")
flag.StringVar(&clientKey, "client-key", clientKey, "path to client key")
flag.StringVar(&caCert, "ca-cert", caCert, "path to ca certificate")
klog.InitFlags(nil)
flag.Parse()
tlsConfig := &tls.Config{}
if caCert != "" {
b, err := ioutil.ReadFile(caCert)
if err != nil {
return fmt.Errorf("error reading certificate %q: %v", caCert, err)
}
rootCAs := x509.NewCertPool()
rootCAs.AppendCertsFromPEM(b)
tlsConfig.RootCAs = rootCAs
}
if clientKey != "" {
keypair, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
return fmt.Errorf("error reading client keypair: %v", err)
}
tlsConfig.Certificates = []tls.Certificate{keypair}
}
transport := &http.Transport{
TLSClientConfig: tlsConfig,
}
s := &healthCheckServer{
transport: transport,
}
http.HandleFunc("/", s.handler)
klog.Infof("listening on %s", listen)
if err := http.ListenAndServe(listen, nil); err != nil {
return fmt.Errorf("error listening on %q: %v", listen, err)
}
return fmt.Errorf("unexpected return from ListenAndServe")
}
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
}

79
cmd/kube-apiserver-healthcheck/proxy_test.go Normal file
View File

@ -0,0 +1,79 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"net/http"
"net/url"
"testing"
)
func TestBuildProxyRequest(t *testing.T) {
grid := []struct {
In string
Out string
}{
{In: "http://127.0.0.1:8080/readyz", Out: "https://127.0.0.1/readyz"},
{In: "http://127.0.0.1:8080/livez", Out: "https://127.0.0.1/livez"},
{In: "http://127.0.0.1:8080/healthz", Out: "https://127.0.0.1/healthz"},
{In: "http://127.0.0.1:8080/ready", Out: ""},
{In: "http://127.0.0.1:8080/", Out: ""},
{In: "http://127.0.0.1:8080/readyz/foo", Out: ""},
{In: "http://127.0.0.1:8080/readyzfoo", Out: ""},
{In: "http://127.0.0.1:8080/readyz?", Out: "https://127.0.0.1/readyz"},
{In: "http://127.0.0.1:8080/readyz?foo=bar", Out: "https://127.0.0.1/readyz"},
{In: "http://127.0.0.1:8080/readyz?exclude=1", Out: "https://127.0.0.1/readyz?exclude=1"},
{In: "http://127.0.0.1:8080/readyz?exclude=1&exclude=2", Out: "https://127.0.0.1/readyz?exclude=1&exclude=2"},
{In: "http://127.0.0.1:8080/readyz?exclude=1&verbose", Out: "https://127.0.0.1/readyz?exclude=1"},
{In: "http://127.0.0.1:8080/readyz?exclude", Out: "https://127.0.0.1/readyz?exclude="},
}
for _, g := range grid {
g := g
t.Run(g.In, func(t *testing.T) {
u, err := url.Parse(g.In)
if err != nil {
t.Fatalf("failed to parse %q: %v", g.In, err)
}
req := &http.Request{
Method: "GET",
URL: u,
}
out := mapToProxyRequest(req)
actual := ""
if out != nil {
if out.Method != "GET" {
t.Fatalf("unexpected method %q", out.Method)
}
if out.URL == nil {
t.Fatalf("expected URL to be set")
}
actual = out.URL.String()
if actual == "" {
t.Fatalf("unexpected empty URL")
}
}
if actual != g.Out {
t.Fatalf("unexpected mapToProxyRequest result %q => %q, expected %q",
g.In,
actual,
g.Out)
}
})
}
}

2
hack/.packages
View File

@ -14,6 +14,7 @@ k8s.io/kops/cmd/kops/util
k8s.io/kops/cmd/kops-controller
k8s.io/kops/cmd/kops-controller/controllers
k8s.io/kops/cmd/kops-controller/pkg/config
k8s.io/kops/cmd/kube-apiserver-healthcheck
k8s.io/kops/cmd/nodeup
k8s.io/kops/dns-controller/cmd/dns-controller
k8s.io/kops/dns-controller/pkg/dns
@ -96,6 +97,7 @@ k8s.io/kops/pkg/model/alimodel
k8s.io/kops/pkg/model/awsmodel
k8s.io/kops/pkg/model/components
k8s.io/kops/pkg/model/components/etcdmanager
k8s.io/kops/pkg/model/components/kubeapiserver
k8s.io/kops/pkg/model/components/node-authorizer
k8s.io/kops/pkg/model/defaults
k8s.io/kops/pkg/model/domodel

2
hack/set-version
View File

@ -57,6 +57,8 @@ git grep -l "version..v${KOPS_RELEASE_VERSION}" upup/models/cloudup/resources/ad
git grep -l kope/kops-controller | xargs -I {} sed -i -e "s@kops-controller:${KOPS_RELEASE_VERSION}@kops-controller:${NEW_RELEASE_VERSION}@g" {}
git grep -l "version..v${KOPS_RELEASE_VERSION}" upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/ | xargs -I {} sed -i -e "s@version: v${KOPS_RELEASE_VERSION}@version: v${NEW_RELEASE_VERSION}@g" {}
git grep -l kope/kube-apiserver-healthcheck | xargs -I {} sed -i -e "s@kube-apiserver-healthcheck:${KOPS_RELEASE_VERSION}@kube-apiserver-healthcheck:${NEW_RELEASE_VERSION}@g" {}
git grep -l "version..${KOPS_RELEASE_VERSION}" upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/ | xargs -I {} sed -i -e "s@version: ${KOPS_RELEASE_VERSION}@version: ${NEW_RELEASE_VERSION}@g" {}
sed -i -e "s@${KOPS_CI_VERSION}@${NEW_CI_VERSION}@g" version.go

1
hack/update-expected.sh
View File

@ -29,6 +29,7 @@ make kops-gobindata
export KOPS_BASE_URL=
export DNSCONTROLLER_IMAGE=
export KOPSCONTROLLER_IMAGE=
export KUBE_APISERVER_HEALTHCHECK_IMAGE=
# Run the tests in "autofix mode"
HACK_UPDATE_EXPECTED_IN_PLACE=1 go test ./... -count=1

2
nodeup/pkg/model/BUILD.bazel
View File

@ -18,6 +18,7 @@ go_library(
"firewall.go",
"hooks.go",
"kube_apiserver.go",
"kube_apiserver_healthcheck.go",
"kube_controller_manager.go",
"kube_proxy.go",
"kube_router.go",
@ -80,6 +81,7 @@ go_library(
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/mount:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library",
],
)

19
nodeup/pkg/model/context.go
View File

@ -45,6 +45,7 @@ type NodeupModelContext struct {
Architecture Architecture
Assets *fi.AssetStore
Cluster *kops.Cluster
ConfigBase vfs.Path
Distribution distros.Distribution
InstanceGroup *kops.InstanceGroup
KeyStore fi.CAStore
@ -206,17 +207,17 @@ func (c *NodeupModelContext) CNIConfDir() string {
// BuildPKIKubeconfig generates a kubeconfig
func (c *NodeupModelContext) BuildPKIKubeconfig(name string) (string, error) {
ca, err := c.FindCert(fi.CertificateId_CA)
ca, err := c.GetCert(fi.CertificateId_CA)
if err != nil {
return "", err
}
cert, err := c.FindCert(name)
cert, err := c.GetCert(name)
if err != nil {
return "", err
}
key, err := c.FindPrivateKey(name)
key, err := c.GetPrivateKey(name)
if err != nil {
return "", err
}
@ -572,27 +573,27 @@ func EvaluateHostnameOverride(hostnameOverride string) (string, error) {
return *(result.Reservations[0].Instances[0].PrivateDnsName), nil
}
// FindCert is a helper method to retrieving a certificate from the store
func (c *NodeupModelContext) FindCert(name string) ([]byte, error) {
// GetCert is a helper method to retrieve a certificate from the store
func (c *NodeupModelContext) GetCert(name string) ([]byte, error) {
cert, err := c.KeyStore.FindCert(name)
if err != nil {
return []byte{}, fmt.Errorf("error fetching certificate: %v from keystore: %v", name, err)
}
if cert == nil {
return []byte{}, fmt.Errorf("unable to found certificate: %s", name)
return []byte{}, fmt.Errorf("unable to find certificate: %s", name)
}
return cert.AsBytes()
}
// FindPrivateKey is a helper method to retrieving a private key from the store
func (c *NodeupModelContext) FindPrivateKey(name string) ([]byte, error) {
// GetPrivateKey is a helper method to retrieve a private key from the store
func (c *NodeupModelContext) GetPrivateKey(name string) ([]byte, error) {
key, err := c.KeyStore.FindPrivateKey(name)
if err != nil {
return []byte{}, fmt.Errorf("error fetching private key: %v from keystore: %v", name, err)
}
if key == nil {
return []byte{}, fmt.Errorf("unable to found private key: %s", name)
return []byte{}, fmt.Errorf("unable to find private key: %s", name)
}
return key.AsBytes()

32
nodeup/pkg/model/kube_apiserver.go
View File

@ -100,6 +100,13 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
})
}
// If we're using kube-apiserver-healthcheck, we need to set up the client cert etc
if b.findHealthcheckManifest() != nil {
if err := b.addHealthcheckSidecarTasks(c); err != nil {
return err
}
}
// @check if we are using secure client certificates for kubelet and grab the certificates
if b.UseSecureKubelet() {
name := "kubelet-api"
@ -397,12 +404,16 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
},
}
useHealthcheckProxy := b.findHealthcheckManifest() != nil
probeAction := &v1.HTTPGetAction{
Host: "127.0.0.1",
Path: "/healthz",
Port: intstr.FromInt(8080),
}
if kubeAPIServer.InsecurePort != 0 {
if useHealthcheckProxy {
// kube-apiserver-healthcheck sidecar container runs on port 8080
} else if kubeAPIServer.InsecurePort != 0 {
probeAction.Port = intstr.FromInt(int(kubeAPIServer.InsecurePort))
} else if kubeAPIServer.SecurePort != 0 {
probeAction.Port = intstr.FromInt(int(kubeAPIServer.SecurePort))
@ -431,11 +442,6 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
ContainerPort: b.Cluster.Spec.KubeAPIServer.SecurePort,
HostPort: b.Cluster.Spec.KubeAPIServer.SecurePort,
},
{
Name: "local",
ContainerPort: 8080,
HostPort: 8080,
},
},
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
@ -444,6 +450,14 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
},
}
if kubeAPIServer.InsecurePort != 0 {
container.Ports = append(container.Ports, v1.ContainerPort{
Name: "local",
ContainerPort: kubeAPIServer.InsecurePort,
HostPort: kubeAPIServer.InsecurePort,
})
}
// Log both to docker and to the logfile
addHostPathMapping(pod, container, "logfile", "/var/log/kube-apiserver.log").ReadOnly = false
if b.IsKubernetesGTE("1.15") {
@ -517,6 +531,12 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
kubemanifest.MarkPodAsCritical(pod)
kubemanifest.MarkPodAsClusterCritical(pod)
if useHealthcheckProxy {
if err := b.addHealthcheckSidecar(pod); err != nil {
return nil, err
}
}
return pod, nil
}

203
nodeup/pkg/model/kube_apiserver_healthcheck.go Normal file
View File

@ -0,0 +1,203 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package model
import (
"crypto/x509"
"crypto/x509/pkix"
"fmt"
"path/filepath"
corev1 "k8s.io/api/core/v1"
"k8s.io/klog"
"k8s.io/kops/pkg/apis/nodeup"
"k8s.io/kops/pkg/pki"
"k8s.io/kops/pkg/wellknownusers"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
"sigs.k8s.io/yaml"
)
func (b *KubeAPIServerBuilder) findHealthcheckManifest() *nodeup.StaticManifest {
if b.NodeupConfig == nil {
return nil
}
for _, manifest := range b.NodeupConfig.StaticManifests {
if manifest.Key == "kube-apiserver-healthcheck" {
return manifest
}
}
return nil
}
func (b *KubeAPIServerBuilder) addHealthcheckSidecar(pod *corev1.Pod) error {
manifest := b.findHealthcheckManifest()
if manifest == nil {
return nil
}
p := b.ConfigBase.Join(manifest.Path)
data, err := p.ReadFile()
if err != nil {
return fmt.Errorf("error reading kube-apiserver-healthcheck manifest %s: %v", manifest.Path, err)
}
sidecar := &corev1.Pod{}
if err := yaml.Unmarshal(data, sidecar); err != nil {
return fmt.Errorf("error parsing kube-apiserver-healthcheck manifest %s: %v", manifest.Path, err)
}
// Quick-and-dirty merge of the fields we care about
pod.Spec.Containers = append(pod.Spec.Containers, sidecar.Spec.Containers...)
pod.Spec.Volumes = append(pod.Spec.Volumes, sidecar.Spec.Volumes...)
return nil
}
func (b *KubeAPIServerBuilder) addHealthcheckSidecarTasks(c *fi.ModelBuilderContext) error {
id := "kube-apiserver-healthcheck"
secretsDir := "/etc/kubernetes/" + id + "/secrets"
userID := wellknownusers.KubeApiserverHealthcheckID
userName := wellknownusers.KubeApiserverHealthcheckName
// We create user a user and hardcode its UID to 10012 as
// that is the ID used inside the container.
{
c.AddTask(&nodetasks.UserTask{
Name: userName,
UID: userID,
Shell: "/sbin/nologin",
Home: secretsDir,
})
}
clientKey, clientCert, err := b.buildClientKeypair(id)
if err != nil {
return err
}
c.AddTask(&nodetasks.File{
Path: filepath.Join(secretsDir),
Type: nodetasks.FileType_Directory,
Mode: s("0755"),
})
clientCertBytes, err := clientCert.AsBytes()
if err != nil {
return err
}
c.AddTask(&nodetasks.File{
Path: filepath.Join(secretsDir, "client.crt"),
Contents: fi.NewBytesResource(clientCertBytes),
Type: nodetasks.FileType_File,
Mode: s("0644"),
Owner: s(userName),
})
clientKeyBytes, err := clientKey.AsBytes()
if err != nil {
return err
}
c.AddTask(&nodetasks.File{
Path: filepath.Join(secretsDir, "client.key"),
Contents: fi.NewBytesResource(clientKeyBytes),
Type: nodetasks.FileType_File,
Mode: s("0600"),
Owner: s(userName),
})
cert, err := b.GetCert(fi.CertificateId_CA)
if err != nil {
return err
}
c.AddTask(&nodetasks.File{
Path: filepath.Join(secretsDir, "ca.crt"),
Contents: fi.NewBytesResource(cert),
Type: nodetasks.FileType_File,
Mode: s("0644"),
Owner: s(userName),
})
return nil
}
func (b *KubeAPIServerBuilder) buildClientKeypair(commonName string) (*pki.PrivateKey, *pki.Certificate, error) {
signerID := fi.CertificateId_CA
var signerKey *pki.PrivateKey
{
k, err := b.KeyStore.FindPrivateKey(signerID)
if err != nil {
return nil, nil, err
}
if k == nil {
return nil, nil, fmt.Errorf("private key %q not found", signerID)
}
signerKey = k
}
var signerCertificate *pki.Certificate
{
cert, err := b.KeyStore.FindCert(signerID)
if err != nil {
return nil, nil, err
}
if cert == nil {
return nil, nil, fmt.Errorf("certificate %q not found", signerID)
}
signerCertificate = cert
}
privateKey, err := pki.GeneratePrivateKey()
if err != nil {
return nil, nil, err
}
template := &x509.Certificate{
BasicConstraintsValid: true,
IsCA: false,
Subject: pkix.Name{
CommonName: commonName,
},
}
// https://tools.ietf.org/html/rfc5280#section-4.2.1.3
//
// Digital signature allows the certificate to be used to verify
// digital signatures used during TLS negotiation.
template.KeyUsage = template.KeyUsage | x509.KeyUsageDigitalSignature
// KeyEncipherment allows the cert/key pair to be used to encrypt
// keys, including the symmetric keys negotiated during TLS setup
// and used for data transfer.
template.KeyUsage = template.KeyUsage | x509.KeyUsageKeyEncipherment
// ClientAuth allows the cert to be used by a TLS client to
// authenticate itself to the TLS server.
template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
klog.Infof("signing certificate for %q", commonName)
cert, err := pki.SignNewCertificate(privateKey, template, signerCertificate.Certificate, signerKey)
if err != nil {
return nil, nil, fmt.Errorf("error signing certificate for %q: %v", commonName, err)
}
return privateKey, cert, nil
}

12
pkg/apis/nodeup/config.go
View File

@ -40,6 +40,10 @@ type Config struct {
// Manifests for running etcd
EtcdManifests []string `json:"etcdManifests,omitempty"`
// StaticManifests describes generic static manifests
// Using this allows us to keep complex logic out of nodeup
StaticManifests []*StaticManifest `json:"staticManifests,omitempty"`
}
// Image is a docker image we should pre-load
@ -51,3 +55,11 @@ type Image struct {
// Hash is the hash of the file, to verify image integrity (even over http)
Hash string `json:"hash,omitempty"`
}
// StaticManifest is a generic static manifest
type StaticManifest struct {
// Key identifies the static manifest
Key string `json:"key,omitempty"`
// Path is the path to the manifest
Path string `json:"path,omitempty"`
}

21
pkg/assets/builder.go
View File

@ -52,6 +52,20 @@ type AssetBuilder struct {
// KubernetesVersion is the version of kubernetes we are installing
KubernetesVersion semver.Version
// StaticManifests records static manifests
StaticManifests []*StaticManifest
}
type StaticManifest struct {
// Key is the unique identifier of the manifest
Key string
// Path is the path to the manifest
Path string
// The static manifest will only be applied to instances matching the specified role
Roles []kops.InstanceGroupRole
}
// ContainerAsset models a container's location.
@ -164,6 +178,13 @@ func (a *AssetBuilder) RemapImage(image string) (string, error) {
}
}
if strings.HasPrefix(image, "kope/kube-apiserver-healthcheck:") {
override := os.Getenv("KUBE_APISERVER_HEALTHCHECK_IMAGE")
if override != "" {
image = override
}
}
if a.AssetsLocation != nil && a.AssetsLocation.ContainerProxy != nil {
containerProxy := strings.TrimSuffix(*a.AssetsLocation.ContainerProxy, "/")
normalized := image

4
pkg/model/BUILD.bazel
View File

@ -11,6 +11,7 @@ go_library(
"external_access.go",
"firewall.go",
"iam.go",
"manifests.go",
"master_volumes.go",
"names.go",
"network.go",
@ -49,8 +50,11 @@ go_library(
"//util/pkg/vfs:go_default_library",
"//vendor/github.com/blang/semver:go_default_library",
"//vendor/github.com/ghodss/yaml:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/legacy-cloud-providers/aws:go_default_library",
],

7
pkg/model/components/apiserver.go
View File

@ -233,8 +233,13 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error {
// We make sure to disable AnonymousAuth
c.AnonymousAuth = fi.Bool(false)
// FIXME : Disable the insecure port when kubernetes issue #43784 is fixed
if b.IsKubernetesGTE("1.18") {
// We query via the kube-apiserver-healthcheck proxy, which listens on port 8080
c.InsecurePort = 0
} else {
// Older versions of kubernetes continue to rely on the insecure port: kubernetes issue #43784
c.InsecurePort = 8080
}
return nil
}

3
pkg/model/components/etcdmanager/BUILD.bazel
View File

@ -32,9 +32,6 @@ go_library(
"//util/pkg/exec:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)

38
pkg/model/components/etcdmanager/model.go
View File

@ -17,18 +17,12 @@ limitations under the License.
package etcdmanager
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"strings"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/yaml"
scheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/klog"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/assets"
@ -161,35 +155,6 @@ func (b *EtcdManagerBuilder) buildManifest(etcdCluster *kops.EtcdClusterSpec) (*
return b.buildPod(etcdCluster)
}
// parseManifest parses a set of objects from a []byte
func parseManifest(data []byte) ([]runtime.Object, error) {
decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096)
deser := scheme.Codecs.UniversalDeserializer()
var objects []runtime.Object
for {
ext := runtime.RawExtension{}
if err := decoder.Decode(&ext); err != nil {
if err == io.EOF {
break
}
fmt.Fprintf(os.Stderr, "%s", string(data))
klog.Infof("manifest: %s", string(data))
return nil, fmt.Errorf("error parsing manifest: %v", err)
}
obj, _, err := deser.Decode([]byte(ext.Raw), nil, nil)
if err != nil {
return nil, fmt.Errorf("error parsing object in manifest: %v", err)
}
objects = append(objects, obj)
}
return objects, nil
}
// Until we introduce the bundle, we hard-code the manifest
var defaultManifest = `
apiVersion: v1
@ -239,7 +204,7 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po
manifest = []byte(defaultManifest)
{
objects, err := parseManifest(manifest)
objects, err := model.ParseManifest(manifest)
if err != nil {
return nil, err
}
@ -261,7 +226,6 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po
klog.Warningf("overloading image in manifest %s with images %s", bundle, etcdCluster.Manager.Image)
container.Image = etcdCluster.Manager.Image
}
}
// With etcd-manager the hosts changes are self-contained, so

30
pkg/model/components/kubeapiserver/BUILD.bazel Normal file
View File

@ -0,0 +1,30 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = ["model.go"],
importpath = "k8s.io/kops/pkg/model/components/kubeapiserver",
visibility = ["//visibility:public"],
deps = [
"//pkg/apis/kops:go_default_library",
"//pkg/assets:go_default_library",
"//pkg/k8scodecs:go_default_library",
"//pkg/model:go_default_library",
"//upup/pkg/fi:go_default_library",
"//upup/pkg/fi/fitasks:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["model_test.go"],
data = glob(["tests/**"]),
embed = [":go_default_library"],
deps = [
"//pkg/assets:go_default_library",
"//pkg/model:go_default_library",
"//pkg/testutils:go_default_library",
"//upup/pkg/fi:go_default_library",
],
)

163
pkg/model/components/kubeapiserver/model.go Normal file
View File

@ -0,0 +1,163 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubeapiserver
import (
"fmt"
corev1 "k8s.io/api/core/v1"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/pkg/k8scodecs"
"k8s.io/kops/pkg/model"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/fitasks"
)
// KubeApiserverBuilder builds the static manifest for kube-apiserver-healthcheck sidecar
type KubeApiserverBuilder struct {
*model.KopsModelContext
Lifecycle *fi.Lifecycle
AssetBuilder *assets.AssetBuilder
}
var _ fi.ModelBuilder = &KubeApiserverBuilder{}
func (b *KubeApiserverBuilder) useHealthCheckSidecar(c *fi.ModelBuilderContext) bool {
// Should we use our health-check proxy, which allows us to
// query the secure port without enabling anonymous auth?
useHealthCheckSidecar := true
// We only turn on the proxy in k8s 1.18 and above
if b.IsKubernetesLT("1.18") {
useHealthCheckSidecar = false
}
return useHealthCheckSidecar
}
// Build creates the tasks relating to kube-apiserver
// Currently we only build the kube-apiserver-healthcheck sidecar
func (b *KubeApiserverBuilder) Build(c *fi.ModelBuilderContext) error {
if !b.useHealthCheckSidecar(c) {
return nil
}
manifest, err := b.buildManifest()
if err != nil {
return err
}
manifestYAML, err := k8scodecs.ToVersionedYaml(manifest)
if err != nil {
return fmt.Errorf("error marshaling manifest to yaml: %v", err)
}
key := "kube-apiserver-healthcheck"
location := "manifests/static/" + key + ".yaml"
c.AddTask(&fitasks.ManagedFile{
Contents: fi.WrapResource(fi.NewBytesResource(manifestYAML)),
Lifecycle: b.Lifecycle,
Location: fi.String(location),
Name: fi.String("manifests-static-" + key),
})
b.AssetBuilder.StaticManifests = append(b.AssetBuilder.StaticManifests, &assets.StaticManifest{
Key: key,
Path: location,
Roles: []kops.InstanceGroupRole{kops.InstanceGroupRoleMaster},
})
return nil
}
func (b *KubeApiserverBuilder) buildManifest() (*corev1.Pod, error) {
return b.buildHealthcheckSidecar()
}
const defaultManifest = `
apiVersion: v1
kind: Pod
spec:
containers:
- name: healthcheck
image: kope/kube-apiserver-healthcheck:1.18.0-alpha.3
livenessProbe:
httpGet:
# The sidecar serves a healthcheck on the same port,
# but with a .kube-apiserver-healthcheck prefix
path: /.kube-apiserver-healthcheck/healthz
port: 8080
host: 127.0.0.1
initialDelaySeconds: 5
timeoutSeconds: 5
command:
- /usr/bin/kube-apiserver-healthcheck
args:
- --ca-cert=/secrets/ca.crt
- --client-cert=/secrets/client.crt
- --client-key=/secrets/client.key
volumeMounts:
- name: healthcheck-secrets
mountPath: /secrets
readOnly: true
volumes:
- name: healthcheck-secrets
hostPath:
path: /etc/kubernetes/kube-apiserver-healthcheck/secrets
type: Directory
`
// buildHealthcheckSidecar builds the partial pod for the healthcheck sidecar.
// nodeup will merge it into the kube-apiserver pod.
func (b *KubeApiserverBuilder) buildHealthcheckSidecar() (*corev1.Pod, error) {
// TODO: pull from bundle
bundle := "(embedded kube-apiserver-healthcheck manifest)"
manifest := []byte(defaultManifest)
var pod *corev1.Pod
var container *corev1.Container
{
objects, err := model.ParseManifest(manifest)
if err != nil {
return nil, err
}
if len(objects) != 1 {
return nil, fmt.Errorf("expected exactly one object in manifest %s, found %d", bundle, len(objects))
}
if podObject, ok := objects[0].(*corev1.Pod); !ok {
return nil, fmt.Errorf("expected Pod object in manifest %s, found %T", bundle, objects[0])
} else {
pod = podObject
}
if len(pod.Spec.Containers) != 1 {
return nil, fmt.Errorf("expected exactly one container in etcd-manager Pod, found %d", len(pod.Spec.Containers))
}
container = &pod.Spec.Containers[0]
}
// Remap image via AssetBuilder
{
remapped, err := b.AssetBuilder.RemapImage(container.Image)
if err != nil {
return nil, fmt.Errorf("unable to remap container image %q: %v", container.Image, err)
}
container.Image = remapped
}
return pod, nil
}

77
pkg/model/components/kubeapiserver/model_test.go Normal file
View File

@ -0,0 +1,77 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubeapiserver
import (
"fmt"
"testing"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/pkg/model"
"k8s.io/kops/pkg/testutils"
"k8s.io/kops/upup/pkg/fi"
)
func Test_RunKubeApiserverBuilder(t *testing.T) {
tests := []string{
"tests/minimal",
}
for _, basedir := range tests {
basedir := basedir
t.Run(fmt.Sprintf("basedir=%s", basedir), func(t *testing.T) {
context := &fi.ModelBuilderContext{
Tasks: make(map[string]fi.Task),
}
kopsModelContext, err := LoadKopsModelContext(basedir)
if err != nil {
t.Fatalf("error loading model %q: %v", basedir, err)
return
}
builder := KubeApiserverBuilder{
KopsModelContext: kopsModelContext,
AssetBuilder: assets.NewAssetBuilder(kopsModelContext.Cluster, ""),
}
if err := builder.Build(context); err != nil {
t.Fatalf("error from Build: %v", err)
return
}
testutils.ValidateTasks(t, basedir, context)
})
}
}
func LoadKopsModelContext(basedir string) (*model.KopsModelContext, error) {
spec, err := testutils.LoadModel(basedir)
if err != nil {
return nil, err
}
if spec.Cluster == nil {
return nil, fmt.Errorf("no cluster found in %s", basedir)
}
kopsContext := &model.KopsModelContext{
Cluster: spec.Cluster,
InstanceGroups: spec.InstanceGroups,
}
return kopsContext, nil
}

28
pkg/model/components/kubeapiserver/tests/minimal/cluster.yaml Normal file
View File

@ -0,0 +1,28 @@
apiVersion: kops.k8s.io/v1alpha2
kind: Cluster
metadata:
creationTimestamp: "2016-12-10T22:42:27Z"
name: minimal.example.com
spec:
kubernetesApiAccess:
- 0.0.0.0/0
channel: stable
cloudProvider: aws
configBase: memfs://clusters.example.com/minimal.example.com
kubernetesVersion: v1.18.0
masterInternalName: api.internal.minimal.example.com
masterPublicName: api.minimal.example.com
networkCIDR: 172.20.0.0/16
networking:
kubenet: {}
nonMasqueradeCIDR: 100.64.0.0/10
sshAccess:
- 0.0.0.0/0
topology:
masters: public
nodes: public
subnets:
- cidr: 172.20.32.0/19
name: us-test-1a
type: Public
zone: us-test-1a

38
pkg/model/components/kubeapiserver/tests/minimal/tasks.yaml Normal file
View File

@ -0,0 +1,38 @@
Contents:
Name: ""
Resource: |
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
spec:
containers:
- args:
- --ca-cert=/secrets/ca.crt
- --client-cert=/secrets/client.crt
- --client-key=/secrets/client.key
command:
- /usr/bin/kube-apiserver-healthcheck
image: kope/kube-apiserver-healthcheck:1.18.0-alpha.3
livenessProbe:
httpGet:
host: 127.0.0.1
path: /.kube-apiserver-healthcheck/healthz
port: 8080
initialDelaySeconds: 5
timeoutSeconds: 5
name: healthcheck
resources: {}
volumeMounts:
- mountPath: /secrets
name: healthcheck-secrets
readOnly: true
volumes:
- hostPath:
path: /etc/kubernetes/kube-apiserver-healthcheck/secrets
type: Directory
name: healthcheck-secrets
status: {}
Lifecycle: null
Location: manifests/static/kube-apiserver-healthcheck.yaml
Name: manifests-static-kube-apiserver-healthcheck

58
pkg/model/manifests.go Normal file
View File

@ -0,0 +1,58 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package model
import (
"bytes"
"fmt"
"io"
"os"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/yaml"
scheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/klog"
)
// ParseManifest parses a set of objects from a []byte
func ParseManifest(data []byte) ([]runtime.Object, error) {
decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096)
deser := scheme.Codecs.UniversalDeserializer()
var objects []runtime.Object
for {
ext := runtime.RawExtension{}
if err := decoder.Decode(&ext); err != nil {
if err == io.EOF {
break
}
fmt.Fprintf(os.Stderr, "%s", string(data))
klog.Infof("manifest: %s", string(data))
return nil, fmt.Errorf("error parsing manifest: %v", err)
}
obj, _, err := deser.Decode([]byte(ext.Raw), nil, nil)
if err != nil {
return nil, fmt.Errorf("error parsing object in manifest: %v", err)
}
objects = append(objects, obj)
}
return objects, nil
}

8
pkg/wellknownusers/wellknownusers.go
View File

@ -26,4 +26,12 @@ const (
// AWSAuthenticator is the user-id for the aws-iam-authenticator (built externally)
AWSAuthenticator = 10000
// KubeApiserverHealthcheckID is the user id for kube-apiserver-healthcheck sidecar
// The user needs some extra permissions e.g. to read local secrets
// This should match the user in cmd/kube-apiserver-healthcheck/BUILD.bazel
KubeApiserverHealthcheckID = 10012
// KubeApiserverHealthcheckName is the username for the kube-apiserver-healthcheck user
KubeApiserverHealthcheckName = "kube-apiserver-healthcheck"
)

6
tools/get_workspace_status.sh
View File

@ -80,3 +80,9 @@ if [[ -z "${DNS_CONTROLLER_TAG}" ]]; then
DNS_CONTROLLER_TAG="${PROTOKUBE_TAG}"
fi
echo "STABLE_DNS_CONTROLLER_TAG ${DNS_CONTROLLER_TAG}"
if [[ -z "${KUBE_APISERVER_HEALTHCHECK_TAG}" ]]; then
KUBE_APISERVER_HEALTHCHECK_TAG="${PROTOKUBE_TAG}"
fi
echo "STABLE_KUBE_APISERVER_HEALTHCHECK_TAG ${KUBE_APISERVER_HEALTHCHECK_TAG}"

1
upup/pkg/fi/cloudup/BUILD.bazel
View File

@ -46,6 +46,7 @@ go_library(
"//pkg/model/awsmodel:go_default_library",
"//pkg/model/components:go_default_library",
"//pkg/model/components/etcdmanager:go_default_library",
"//pkg/model/components/kubeapiserver:go_default_library",
"//pkg/model/components/node-authorizer:go_default_library",
"//pkg/model/domodel:go_default_library",
"//pkg/model/gcemodel:go_default_library",

26
upup/pkg/fi/cloudup/apply_cluster.go
View File

@ -45,6 +45,7 @@ import (
"k8s.io/kops/pkg/model/awsmodel"
"k8s.io/kops/pkg/model/components"
"k8s.io/kops/pkg/model/components/etcdmanager"
"k8s.io/kops/pkg/model/components/kubeapiserver"
"k8s.io/kops/pkg/model/domodel"
"k8s.io/kops/pkg/model/gcemodel"
"k8s.io/kops/pkg/model/openstackmodel"
@ -614,6 +615,11 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
KopsModelContext: modelContext,
Lifecycle: &clusterLifecycle,
},
&kubeapiserver.KubeApiserverBuilder{
AssetBuilder: assetBuilder,
KopsModelContext: modelContext,
Lifecycle: &clusterLifecycle,
},
&etcdmanager.EtcdManagerBuilder{
AssetBuilder: assetBuilder,
KopsModelContext: modelContext,
@ -1369,7 +1375,7 @@ func (c *ApplyClusterCmd) BuildNodeUpConfig(assetBuilder *assets.AssetBuilder, i
// `docker load` our images when using a KOPS_BASE_URL, so we
// don't need to push/pull from a registry
if os.Getenv("KOPS_BASE_URL") != "" && isMaster {
for _, name := range []string{"kops-controller", "dns-controller"} {
for _, name := range []string{"kops-controller", "dns-controller", "kube-apiserver-healthcheck"} {
baseURL, err := url.Parse(os.Getenv("KOPS_BASE_URL"))
if err != nil {
return nil, err
@ -1414,6 +1420,24 @@ func (c *ApplyClusterCmd) BuildNodeUpConfig(assetBuilder *assets.AssetBuilder, i
}
}
for _, manifest := range assetBuilder.StaticManifests {
match := false
for _, r := range manifest.Roles {
if r == role {
match = true
}
}
if !match {
continue
}
config.StaticManifests = append(config.StaticManifests, &nodeup.StaticManifest{
Key: manifest.Key,
Path: manifest.Path,
})
}
config.Images = images
config.Channels = channels

1
upup/pkg/fi/nodeup/command.go
View File

@ -185,6 +185,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
Architecture: model.ArchitectureAmd64,
Assets: assetStore,
Cluster: c.cluster,
ConfigBase: configBase,
Distribution: distribution,
InstanceGroup: c.instanceGroup,
NodeupConfig: c.config,