kops/nodeup/pkg/model/kubelet_test.go

384 lines
15 KiB
Go

/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package model
import (
"fmt"
"path/filepath"
"testing"
"k8s.io/klog/v2"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/nodeup"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/pkg/client/simple/vfsclientset"
"k8s.io/kops/pkg/pki"
"k8s.io/kops/pkg/testutils"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup"
"k8s.io/kops/util/pkg/vfs"
)
func Test_InstanceGroupKubeletMerge(t *testing.T) {
var cluster = &kops.Cluster{}
cluster.Spec.Kubelet = &kops.KubeletConfigSpec{}
cluster.Spec.Kubelet.NvidiaGPUs = 0
cluster.Spec.KubernetesVersion = "1.6.0"
var instanceGroup = &kops.InstanceGroup{}
instanceGroup.Spec.Kubelet = &kops.KubeletConfigSpec{}
instanceGroup.Spec.Kubelet.NvidiaGPUs = 1
instanceGroup.Spec.Role = kops.InstanceGroupRoleNode
config, auxConfig := nodeup.NewConfig(cluster, instanceGroup)
b := &KubeletBuilder{
&NodeupModelContext{
Cluster: cluster,
NodeupConfig: config,
NodeupAuxConfig: auxConfig,
},
}
if err := b.Init(); err != nil {
t.Error(err)
}
var mergedKubeletSpec, err = b.buildKubeletConfigSpec()
if err != nil {
t.Error(err)
}
if mergedKubeletSpec == nil {
t.Error("Returned nil kubelet spec")
t.FailNow()
}
if mergedKubeletSpec.NvidiaGPUs != instanceGroup.Spec.Kubelet.NvidiaGPUs {
t.Errorf("InstanceGroup kubelet value (%d) should be reflected in merged output", instanceGroup.Spec.Kubelet.NvidiaGPUs)
}
}
func TestTaintsApplied(t *testing.T) {
tests := []struct {
version string
taints []string
expectError bool
expectSchedulable bool
expectTaints []string
}{
{
version: "1.9.0",
taints: []string{"foo", "bar", "baz"},
expectTaints: []string{"foo", "bar", "baz"},
expectSchedulable: true,
},
}
for _, g := range tests {
cluster := &kops.Cluster{Spec: kops.ClusterSpec{KubernetesVersion: g.version}}
ig := &kops.InstanceGroup{Spec: kops.InstanceGroupSpec{Role: kops.InstanceGroupRoleMaster, Taints: g.taints}}
config, auxConfig := nodeup.NewConfig(cluster, ig)
b := &KubeletBuilder{
&NodeupModelContext{
Cluster: cluster,
NodeupConfig: config,
NodeupAuxConfig: auxConfig,
},
}
if err := b.Init(); err != nil {
t.Error(err)
}
c, err := b.buildKubeletConfigSpec()
if g.expectError {
if err == nil {
t.Fatalf("Expected error but did not get one for version %q", g.version)
}
continue
} else {
if err != nil {
t.Fatalf("Unexpected error for version %q: %v", g.version, err)
}
}
if fi.BoolValue(c.RegisterSchedulable) != g.expectSchedulable {
t.Fatalf("Expected RegisterSchedulable == %v, got %v (for %v)", g.expectSchedulable, fi.BoolValue(c.RegisterSchedulable), g.version)
}
if !stringSlicesEqual(g.expectTaints, c.Taints) {
t.Fatalf("Expected taints %v, got %v", g.expectTaints, c.Taints)
}
}
}
func stringSlicesEqual(exp, other []string) bool {
if exp == nil && other != nil {
return false
}
if exp != nil && other == nil {
return false
}
if len(exp) != len(other) {
return false
}
for i, e := range exp {
if other[i] != e {
return false
}
}
return true
}
func Test_RunKubeletBuilder(t *testing.T) {
basedir := "tests/kubelet/featuregates"
context := &fi.ModelBuilderContext{
Tasks: make(map[string]fi.Task),
}
nodeUpModelContext, err := BuildNodeupModelContext(basedir)
if err != nil {
t.Fatalf("error loading model %q: %v", basedir, err)
return
}
runKubeletBuilder(t, context, nodeUpModelContext)
testutils.ValidateTasks(t, filepath.Join(basedir, "tasks.yaml"), context)
}
func Test_RunKubeletBuilderWarmPool(t *testing.T) {
basedir := "tests/kubelet/warmpool"
context := &fi.ModelBuilderContext{
Tasks: make(map[string]fi.Task),
}
nodeUpModelContext, err := BuildNodeupModelContext(basedir)
if err != nil {
t.Fatalf("error loading model %q: %v", basedir, err)
return
}
nodeUpModelContext.ConfigurationMode = "Warming"
runKubeletBuilder(t, context, nodeUpModelContext)
testutils.ValidateTasks(t, filepath.Join(basedir, "tasks.yaml"), context)
}
func runKubeletBuilder(t *testing.T, context *fi.ModelBuilderContext, nodeupModelContext *NodeupModelContext) {
builder := KubeletBuilder{NodeupModelContext: nodeupModelContext}
kubeletConfig, err := builder.buildKubeletConfig()
if err != nil {
t.Fatalf("error from KubeletBuilder buildKubeletConfig: %v", err)
return
}
fileTask, err := builder.buildSystemdEnvironmentFile(kubeletConfig)
if err != nil {
t.Fatalf("error from KubeletBuilder buildSystemdEnvironmentFile: %v", err)
return
}
context.AddTask(fileTask)
{
task, err := builder.buildManifestDirectory(kubeletConfig)
if err != nil {
t.Fatalf("error from KubeletBuilder buildManifestDirectory: %v", err)
return
}
context.AddTask(task)
}
{
task := builder.buildSystemdService()
if err != nil {
t.Fatalf("error from KubeletBuilder buildSystemdService: %v", err)
return
}
context.AddTask(task)
}
}
func BuildNodeupModelContext(basedir string) (*NodeupModelContext, error) {
model, err := testutils.LoadModel(basedir)
if err != nil {
return nil, err
}
if model.Cluster == nil {
return nil, fmt.Errorf("no cluster found in %s", basedir)
}
nodeUpModelContext := &NodeupModelContext{
Cluster: model.Cluster,
Architecture: "amd64",
NodeupConfig: &nodeup.Config{},
}
if len(model.InstanceGroups) == 0 {
// We tolerate this - not all tests need an instance group
} else if len(model.InstanceGroups) == 1 {
nodeUpModelContext.NodeupConfig, nodeUpModelContext.NodeupAuxConfig = nodeup.NewConfig(model.Cluster, model.InstanceGroups[0])
} else {
return nil, fmt.Errorf("unexpected number of instance groups in %s, found %d", basedir, len(model.InstanceGroups))
}
if err := nodeUpModelContext.Init(); err != nil {
return nil, err
}
return nodeUpModelContext, nil
}
func mockedPopulateClusterSpec(c *kops.Cluster, cloud fi.Cloud) (*kops.Cluster, error) {
vfs.Context.ResetMemfsContext(true)
assetBuilder := assets.NewAssetBuilder(c, false)
basePath, err := vfs.Context.BuildVfsPath("memfs://tests")
if err != nil {
return nil, fmt.Errorf("error building vfspath: %v", err)
}
clientset := vfsclientset.NewVFSClientset(basePath)
return cloudup.PopulateClusterSpec(clientset, c, cloud, assetBuilder)
}
// Fixed cert and key, borrowed from the create_kubecfg_test.go test
// Wouldn't actually work in a real environment, but good enough for (today's) tests
const dummyCertificate = "-----BEGIN CERTIFICATE-----\nMIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw\nFTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy\nMzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA\nA4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd\nXEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY\nOACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1\nYYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC\nAn8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA\nNPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB\n/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r\nhzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO\nHE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe\nrN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt\nO6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy\nVpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh\n8yGeRx9AbknHh4Ia\n-----END CERTIFICATE-----\n"
const dummyKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH\nAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA\ngvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF\nGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/\nP2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx\n9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI\nBezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9\n/Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM\n0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t\nvpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd\ncuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G\n8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj\n22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1\nAsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV\n99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs\nz2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg\nST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK\nsfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7\nKhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i\nQ4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T\nt9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q\ndGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z\nworz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu\nBAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq\nY/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw==\n-----END RSA PRIVATE KEY-----\n"
const previousCertificate = "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMDA2WhcNMzEwNTAyMjAzMDA2WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\n2JbeF8dNwqfEKKD65aGlVs58fWkA0qZdVLKw8qATzRBJTi1nqbj2kAR4gyy/C8Mx\nouxva/om9d7Sq8Ka55T7+wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUI5beFHueAGyT1pQ6UTOdbMfj3gQwDQYJKoZI\nhvcNAQELBQADQQBwPLO+Np8o6k3aNBGKE4JTCOs06X72OXNivkWWWP/9XGz6x4DI\nHPU65kbUn/pWXBUVVlpsKsdmWA2Bu8pd/vD+\n-----END CERTIFICATE-----\n"
const previousKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----"
const nextCertificate = "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBBDANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMjE3WhcNMzEwNTAyMjAzMjE3WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\no4Tridlsf4Yz3UAiup/scSTiG/OqxkUW3Fz7zGKvVcLeYj9GEIKuzoB1VFk1nboD\nq4cCuGLfdzaQdCQKPIsDuwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUhPbxEmUbwVOCa+fZgxreFhf67UEwDQYJKoZI\nhvcNAQELBQADQQALMsyK2Q7C/bk27eCvXyZKUfrLvor10hEjwGhv14zsKWDeTj/J\nA1LPYp7U9VtFfgFOkVbkLE9Rstc0ltNrPqxA\n-----END CERTIFICATE-----\n"
const nextKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----"
func simplePrivateKeyset(s string) *kops.Keyset {
return &kops.Keyset{
Spec: kops.KeysetSpec{
PrimaryId: "3",
Keys: []kops.KeysetItem{
{
Id: "3",
PrivateMaterial: []byte(s),
},
},
},
}
}
func rotatingPrivateKeyset() *kops.Keyset {
return &kops.Keyset{
Spec: kops.KeysetSpec{
PrimaryId: "3",
Keys: []kops.KeysetItem{
{
Id: "2",
PrivateMaterial: []byte(previousKey),
PublicMaterial: []byte(previousCertificate),
},
{
Id: "3",
PrivateMaterial: []byte(dummyKey),
PublicMaterial: []byte(dummyCertificate),
},
{
Id: "4",
PrivateMaterial: []byte(nextKey),
PublicMaterial: []byte(nextCertificate),
},
},
},
}
}
func mustParseCertificate(s string) *pki.Certificate {
k, err := pki.ParsePEMCertificate([]byte(s))
if err != nil {
klog.Fatalf("error parsing certificate %v", err)
}
return k
}
func RunGoldenTest(t *testing.T, basedir string, key string, builder func(*NodeupModelContext, *fi.ModelBuilderContext) error) {
h := testutils.NewIntegrationTestHarness(t)
defer h.Close()
h.MockKopsVersion("1.18.0")
h.SetupMockAWS()
context := &fi.ModelBuilderContext{
Tasks: make(map[string]fi.Task),
}
nodeupModelContext, err := BuildNodeupModelContext(basedir)
if err != nil {
t.Fatalf("error loading model %q: %v", basedir, err)
}
keystore := &fakeCAStore{}
keystore.T = t
keystore.privateKeysets = map[string]*kops.Keyset{
"ca": simplePrivateKeyset(dummyKey),
"apiserver-aggregator-ca": simplePrivateKeyset(dummyKey),
"kube-controller-manager": simplePrivateKeyset(dummyKey),
"kube-proxy": simplePrivateKeyset(dummyKey),
"kube-scheduler": simplePrivateKeyset(dummyKey),
"service-account": rotatingPrivateKeyset(),
}
keystore.certs = map[string]*pki.Certificate{
"ca": mustParseCertificate(dummyCertificate),
"apiserver-aggregator-ca": mustParseCertificate(dummyCertificate),
"kube-controller-manager": mustParseCertificate(dummyCertificate),
"kube-proxy": mustParseCertificate(dummyCertificate),
"kube-scheduler": mustParseCertificate(dummyCertificate),
}
nodeupModelContext.KeyStore = keystore
// Populate the cluster
cloud, err := cloudup.BuildCloud(nodeupModelContext.Cluster)
if err != nil {
t.Fatalf("error from BuildCloud: %v", err)
}
{
err := cloudup.PerformAssignments(nodeupModelContext.Cluster, cloud)
if err != nil {
t.Fatalf("error from PerformAssignments: %v", err)
}
full, err := mockedPopulateClusterSpec(nodeupModelContext.Cluster, cloud)
if err != nil {
t.Fatalf("unexpected error from mockedPopulateClusterSpec: %v", err)
}
nodeupModelContext.Cluster = full
}
if err := builder(nodeupModelContext, context); err != nil {
t.Fatalf("error from Build: %v", err)
}
testutils.ValidateTasks(t, filepath.Join(basedir, "tasks-"+key+".yaml"), context)
}