kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add file assets to node user data scripts, fingerprint sections of the 

hooks and fileAssets specs.
This commit is contained in:
Kashif Saadat 2017-08-31 19:15:13 +01:00
parent 2fd0ddb484
commit e4919d0c39
8 changed files with 237 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
pkg/model/bootstrapscript.go
View File

@ -18,6 +18,8 @@ package model
import (
"bytes"
"crypto/sha1"
"encoding/base64"
"fmt"
"os"
"strconv"
@ -105,11 +107,22 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste
spec["masterKubelet"] = cs.MasterKubelet
}
hooks := b.getRelevantHooks(cs.Hooks, ig.Spec.Role)
hooks, err := b.getRelevantHooks(cs.Hooks, ig.Spec.Role)
if err != nil {
return "", err
}
if len(hooks) > 0 {
spec["hooks"] = hooks
}
fileAssets, err := b.getRelevantFileAssets(cs.FileAssets, ig.Spec.Role)
if err != nil {
return "", err
}
if len(fileAssets) > 0 {
spec["fileAssets"] = fileAssets
}
content, err := yaml.Marshal(spec)
if err != nil {
return "", fmt.Errorf("error converting cluster spec to yaml for inclusion within bootstrap script: %v", err)
@ -122,11 +135,23 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste
spec["kubelet"] = ig.Spec.Kubelet
spec["nodeLabels"] = ig.Spec.NodeLabels
spec["taints"] = ig.Spec.Taints
hooks := b.getRelevantHooks(ig.Spec.Hooks, ig.Spec.Role)
hooks, err := b.getRelevantHooks(ig.Spec.Hooks, ig.Spec.Role)
if err != nil {
return "", err
}
if len(hooks) > 0 {
spec["hooks"] = hooks
}
fileAssets, err := b.getRelevantFileAssets(ig.Spec.FileAssets, ig.Spec.Role)
if err != nil {
return "", err
}
if len(fileAssets) > 0 {
spec["fileAssets"] = fileAssets
}
content, err := yaml.Marshal(spec)
if err != nil {
return "", fmt.Errorf("error converting instancegroup spec to yaml for inclusion within bootstrap script: %v", err)
@ -143,10 +168,11 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste
return fi.WrapResource(templateResource), nil
}
// getRelevantHooks returns a list of hooks to be applied to the instance group
func (b *BootstrapScript) getRelevantHooks(hooks []kops.HookSpec, role kops.InstanceGroupRole) []kops.HookSpec {
// getRelevantHooks returns a list of hooks to be applied to the instance group,
// with the Manifest and ExecContainer Commands fingerprinted to reduce size
func (b *BootstrapScript) getRelevantHooks(allHooks []kops.HookSpec, role kops.InstanceGroupRole) ([]kops.HookSpec, error) {
relevantHooks := []kops.HookSpec{}
for _, hook := range hooks {
for _, hook := range allHooks {
if len(hook.Roles) == 0 {
relevantHooks = append(relevantHooks, hook)
continue
@ -158,7 +184,84 @@ func (b *BootstrapScript) getRelevantHooks(hooks []kops.HookSpec, role kops.Inst
}
}
}
return relevantHooks
hooks := []kops.HookSpec{}
if len(relevantHooks) > 0 {
for _, hook := range relevantHooks {
if hook.Manifest != "" {
manifestFingerprint, err := b.computeFingerprint(hook.Manifest)
if err != nil {
return nil, err
}
hook.Manifest = manifestFingerprint + " (fingerprint)"
}
if hook.ExecContainer != nil && hook.ExecContainer.Command != nil {
execContainerCommandFingerprint, err := b.computeFingerprint(hook.ExecContainer.Command)
if err != nil {
return nil, err
}
hook.ExecContainer.Command = []string{execContainerCommandFingerprint + " (fingerprint)"}
}
hook.Roles = nil
hooks = append(hooks, hook)
}
}
return hooks, nil
}
// getRelevantFileAssets returns a list of file assets to be applied to the
// instance group, with the Content fingerprinted to reduce size
func (b *BootstrapScript) getRelevantFileAssets(allFileAssets []kops.FileAssetSpec, role kops.InstanceGroupRole) ([]kops.FileAssetSpec, error) {
relevantFileAssets := []kops.FileAssetSpec{}
for _, fileAsset := range allFileAssets {
if len(fileAsset.Roles) == 0 {
relevantFileAssets = append(relevantFileAssets, fileAsset)
continue
}
for _, fileAssetRole := range fileAsset.Roles {
if role == fileAssetRole {
relevantFileAssets = append(relevantFileAssets, fileAsset)
break
}
}
}
fileAssets := []kops.FileAssetSpec{}
if len(relevantFileAssets) > 0 {
for _, fileAsset := range relevantFileAssets {
if fileAsset.Content != "" {
contentFingerprint, err := b.computeFingerprint(fileAsset.Content)
if err != nil {
return nil, err
}
fileAsset.Content = contentFingerprint + " (fingerprint)"
}
fileAsset.Roles = nil
fileAssets = append(fileAssets, fileAsset)
}
}
return fileAssets, nil
}
// computeFingerprint takes an object and returns a base64 encoded fingerprint
func (b *BootstrapScript) computeFingerprint(obj interface{}) (string, error) {
hasher := sha1.New()
data, err := kops.ToRawYaml(obj)
if err != nil {
return "", err
}
if _, err := hasher.Write(data); err != nil {
return "", fmt.Errorf("error computing fingerprint hash: %v", err)
}
return base64.StdEncoding.EncodeToString(hasher.Sum(nil)), nil
}
func (b *BootstrapScript) createProxyEnv(ps *kops.EgressProxySpec) string {

94
pkg/model/bootstrapscript_test.go
View File

@ -55,55 +55,64 @@ func Test_ProxyFunc(t *testing.T) {
func TestBootstrapUserData(t *testing.T) {
cs := []struct {
Role kops.InstanceGroupRole
ExpectedFilePath string
HookSpecRoles []kops.InstanceGroupRole
Role kops.InstanceGroupRole
ExpectedFilePath string
HookSpecRoles []kops.InstanceGroupRole
FileAssetSpecRoles []kops.InstanceGroupRole
}{
{
Role: "Master",
ExpectedFilePath: "tests/data/bootstrapscript_0.txt",
HookSpecRoles: []kops.InstanceGroupRole{""},
Role: "Master",
ExpectedFilePath: "tests/data/bootstrapscript_0.txt",
HookSpecRoles: []kops.InstanceGroupRole{""},
FileAssetSpecRoles: []kops.InstanceGroupRole{""},
},
{
Role: "Master",
ExpectedFilePath: "tests/data/bootstrapscript_0.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Node"},
Role: "Master",
ExpectedFilePath: "tests/data/bootstrapscript_0.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Node"},
FileAssetSpecRoles: []kops.InstanceGroupRole{"Node"},
},
{
Role: "Master",
ExpectedFilePath: "tests/data/bootstrapscript_1.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Master"},
Role: "Master",
ExpectedFilePath: "tests/data/bootstrapscript_1.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Master"},
FileAssetSpecRoles: []kops.InstanceGroupRole{"Master"},
},
{
Role: "Master",
ExpectedFilePath: "tests/data/bootstrapscript_2.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Master", "Node"},
Role: "Master",
ExpectedFilePath: "tests/data/bootstrapscript_2.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Master", "Node"},
FileAssetSpecRoles: []kops.InstanceGroupRole{"Master", "Node"},
},
{
Role: "Node",
ExpectedFilePath: "tests/data/bootstrapscript_3.txt",
HookSpecRoles: []kops.InstanceGroupRole{""},
Role: "Node",
ExpectedFilePath: "tests/data/bootstrapscript_3.txt",
HookSpecRoles: []kops.InstanceGroupRole{""},
FileAssetSpecRoles: []kops.InstanceGroupRole{""},
},
{
Role: "Node",
ExpectedFilePath: "tests/data/bootstrapscript_4.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Node"},
Role: "Node",
ExpectedFilePath: "tests/data/bootstrapscript_4.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Node"},
FileAssetSpecRoles: []kops.InstanceGroupRole{"Node"},
},
{
Role: "Node",
ExpectedFilePath: "tests/data/bootstrapscript_3.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Master"},
Role: "Node",
ExpectedFilePath: "tests/data/bootstrapscript_3.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Master"},
FileAssetSpecRoles: []kops.InstanceGroupRole{"Master"},
},
{
Role: "Node",
ExpectedFilePath: "tests/data/bootstrapscript_5.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Master", "Node"},
Role: "Node",
ExpectedFilePath: "tests/data/bootstrapscript_5.txt",
HookSpecRoles: []kops.InstanceGroupRole{"Master", "Node"},
FileAssetSpecRoles: []kops.InstanceGroupRole{"Master", "Node"},
},
}
for i, x := range cs {
spec := makeTestCluster(x.HookSpecRoles).Spec
group := makeTestInstanceGroup(x.Role, x.HookSpecRoles)
spec := makeTestCluster(x.HookSpecRoles, x.FileAssetSpecRoles).Spec
group := makeTestInstanceGroup(x.Role, x.HookSpecRoles, x.FileAssetSpecRoles)
renderNodeUpConfig := func(ig *kops.InstanceGroup) (*nodeup.Config, error) {
return &nodeup.Config{}, nil
@ -134,13 +143,13 @@ func TestBootstrapUserData(t *testing.T) {
if actual != string(expectedBytes) {
diffString := diff.FormatDiff(string(expectedBytes), actual)
t.Errorf("case %d failed, actual output differed from expected.", i)
t.Errorf("case %d failed, actual output differed from expected (%s).", i, x.ExpectedFilePath)
t.Logf("diff:\n%s\n", diffString)
}
}
}
func makeTestCluster(hookSpecRoles []kops.InstanceGroupRole) *kops.Cluster {
func makeTestCluster(hookSpecRoles []kops.InstanceGroupRole, fileAssetSpecRoles []kops.InstanceGroupRole) *kops.Cluster {
return &kops.Cluster{
Spec: kops.ClusterSpec{
CloudProvider: "aws",
@ -207,11 +216,19 @@ func makeTestCluster(hookSpecRoles []kops.InstanceGroupRole) *kops.Cluster {
Roles: hookSpecRoles,
},
},
FileAssets: []kops.FileAssetSpec{
{
Name: "iptables-restore",
Path: "/var/lib/iptables/rules-save",
Content: "blah blah",
Roles: fileAssetSpecRoles,
},
},
},
}
}
func makeTestInstanceGroup(role kops.InstanceGroupRole, hookSpecRoles []kops.InstanceGroupRole) *kops.InstanceGroup {
func makeTestInstanceGroup(role kops.InstanceGroupRole, hookSpecRoles []kops.InstanceGroupRole, fileAssetSpecRoles []kops.InstanceGroupRole) *kops.InstanceGroup {
return &kops.InstanceGroup{
Spec: kops.InstanceGroupSpec{
Kubelet: &kops.KubeletConfigSpec{
@ -240,6 +257,19 @@ func makeTestInstanceGroup(role kops.InstanceGroupRole, hookSpecRoles []kops.Ins
Manifest: "Type=oneshot\nExecStart=/usr/bin/systemctl start apply-to-all.service",
},
},
FileAssets: []kops.FileAssetSpec{
{
Name: "iptables-restore",
Path: "/var/lib/iptables/rules-save",
Content: "blah blah",
Roles: fileAssetSpecRoles,
},
{
Name: "tokens",
Path: "/kube/tokens.csv",
Content: "user,token",
},
},
},
}
}

8
pkg/model/tests/data/bootstrapscript_0.txt
View File

@ -167,10 +167,12 @@ masterKubelet:
__EOF_CLUSTER_SPEC
cat > ig_spec.yaml << '__EOF_IG_SPEC'
fileAssets:
- content: tre8+iQw12cCsccJY3cQk4HQV3g= (fingerprint)
name: tokens
path: /kube/tokens.csv
hooks:
- manifest: |-
Type=oneshot
ExecStart=/usr/bin/systemctl start apply-to-all.service
- manifest: FdicqEXLciSI1yRjQxsrye3QivU= (fingerprint)
name: apply-to-all.service
kubelet:
kubeconfigPath: /etc/kubernetes/igconfig.txt

27
pkg/model/tests/data/bootstrapscript_1.txt
View File

@ -149,15 +149,15 @@ cloudConfig:
nodeTags: something
docker:
logLevel: INFO
fileAssets:
- content: 4/KwQntXJBIluh/8K2f+zWTQdhM= (fingerprint)
name: iptables-restore
path: /var/lib/iptables/rules-save
hooks:
- execContainer:
command:
- sh
- -c
- chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common
- ChIIt0bv6sTY/wwaEWTBNaObBgM= (fingerprint)
image: busybox
roles:
- Master
kubeAPIServer:
image: CoreOS
kubeControllerManager:
@ -176,19 +176,20 @@ masterKubelet:
__EOF_CLUSTER_SPEC
cat > ig_spec.yaml << '__EOF_IG_SPEC'
fileAssets:
- content: 4/KwQntXJBIluh/8K2f+zWTQdhM= (fingerprint)
name: iptables-restore
path: /var/lib/iptables/rules-save
- content: tre8+iQw12cCsccJY3cQk4HQV3g= (fingerprint)
name: tokens
path: /kube/tokens.csv
hooks:
- before:
- update-engine.service
- kubelet.service
manifest: |-
Type=oneshot
ExecStart=/usr/bin/systemctl stop update-engine.service
manifest: LSdpmOQebIkYoG0lRv8AUHCBIyg= (fingerprint)
name: disable-update-engine.service
roles:
- Master
- manifest: |-
Type=oneshot
ExecStart=/usr/bin/systemctl start apply-to-all.service
- manifest: FdicqEXLciSI1yRjQxsrye3QivU= (fingerprint)
name: apply-to-all.service
kubelet:
kubeconfigPath: /etc/kubernetes/igconfig.txt

29
pkg/model/tests/data/bootstrapscript_2.txt
View File

@ -149,16 +149,15 @@ cloudConfig:
nodeTags: something
docker:
logLevel: INFO
fileAssets:
- content: 4/KwQntXJBIluh/8K2f+zWTQdhM= (fingerprint)
name: iptables-restore
path: /var/lib/iptables/rules-save
hooks:
- execContainer:
command:
- sh
- -c
- chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common
- ChIIt0bv6sTY/wwaEWTBNaObBgM= (fingerprint)
image: busybox
roles:
- Master
- Node
kubeAPIServer:
image: CoreOS
kubeControllerManager:
@ -177,20 +176,20 @@ masterKubelet:
__EOF_CLUSTER_SPEC
cat > ig_spec.yaml << '__EOF_IG_SPEC'
fileAssets:
- content: 4/KwQntXJBIluh/8K2f+zWTQdhM= (fingerprint)
name: iptables-restore
path: /var/lib/iptables/rules-save
- content: tre8+iQw12cCsccJY3cQk4HQV3g= (fingerprint)
name: tokens
path: /kube/tokens.csv
hooks:
- before:
- update-engine.service
- kubelet.service
manifest: |-
Type=oneshot
ExecStart=/usr/bin/systemctl stop update-engine.service
manifest: LSdpmOQebIkYoG0lRv8AUHCBIyg= (fingerprint)
name: disable-update-engine.service
roles:
- Master
- Node
- manifest: |-
Type=oneshot
ExecStart=/usr/bin/systemctl start apply-to-all.service
- manifest: FdicqEXLciSI1yRjQxsrye3QivU= (fingerprint)
name: apply-to-all.service
kubelet:
kubeconfigPath: /etc/kubernetes/igconfig.txt

8
pkg/model/tests/data/bootstrapscript_3.txt
View File

@ -159,10 +159,12 @@ kubelet:
__EOF_CLUSTER_SPEC
cat > ig_spec.yaml << '__EOF_IG_SPEC'
fileAssets:
- content: tre8+iQw12cCsccJY3cQk4HQV3g= (fingerprint)
name: tokens
path: /kube/tokens.csv
hooks:
- manifest: |-
Type=oneshot
ExecStart=/usr/bin/systemctl start apply-to-all.service
- manifest: FdicqEXLciSI1yRjQxsrye3QivU= (fingerprint)
name: apply-to-all.service
kubelet:
kubeconfigPath: /etc/kubernetes/igconfig.txt

27
pkg/model/tests/data/bootstrapscript_4.txt
View File

@ -149,15 +149,15 @@ cloudConfig:
nodeTags: something
docker:
logLevel: INFO
fileAssets:
- content: 4/KwQntXJBIluh/8K2f+zWTQdhM= (fingerprint)
name: iptables-restore
path: /var/lib/iptables/rules-save
hooks:
- execContainer:
command:
- sh
- -c
- chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common
- ChIIt0bv6sTY/wwaEWTBNaObBgM= (fingerprint)
image: busybox
roles:
- Node
kubeProxy:
cpuRequest: 30m
featureGates:
@ -168,19 +168,20 @@ kubelet:
__EOF_CLUSTER_SPEC
cat > ig_spec.yaml << '__EOF_IG_SPEC'
fileAssets:
- content: 4/KwQntXJBIluh/8K2f+zWTQdhM= (fingerprint)
name: iptables-restore
path: /var/lib/iptables/rules-save
- content: tre8+iQw12cCsccJY3cQk4HQV3g= (fingerprint)
name: tokens
path: /kube/tokens.csv
hooks:
- before:
- update-engine.service
- kubelet.service
manifest: |-
Type=oneshot
ExecStart=/usr/bin/systemctl stop update-engine.service
manifest: LSdpmOQebIkYoG0lRv8AUHCBIyg= (fingerprint)
name: disable-update-engine.service
roles:
- Node
- manifest: |-
Type=oneshot
ExecStart=/usr/bin/systemctl start apply-to-all.service
- manifest: FdicqEXLciSI1yRjQxsrye3QivU= (fingerprint)
name: apply-to-all.service
kubelet:
kubeconfigPath: /etc/kubernetes/igconfig.txt

29
pkg/model/tests/data/bootstrapscript_5.txt
View File

@ -149,16 +149,15 @@ cloudConfig:
nodeTags: something
docker:
logLevel: INFO
fileAssets:
- content: 4/KwQntXJBIluh/8K2f+zWTQdhM= (fingerprint)
name: iptables-restore
path: /var/lib/iptables/rules-save
hooks:
- execContainer:
command:
- sh
- -c
- chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common
- ChIIt0bv6sTY/wwaEWTBNaObBgM= (fingerprint)
image: busybox
roles:
- Master
- Node
kubeProxy:
cpuRequest: 30m
featureGates:
@ -169,20 +168,20 @@ kubelet:
__EOF_CLUSTER_SPEC
cat > ig_spec.yaml << '__EOF_IG_SPEC'
fileAssets:
- content: 4/KwQntXJBIluh/8K2f+zWTQdhM= (fingerprint)
name: iptables-restore
path: /var/lib/iptables/rules-save
- content: tre8+iQw12cCsccJY3cQk4HQV3g= (fingerprint)
name: tokens
path: /kube/tokens.csv
hooks:
- before:
- update-engine.service
- kubelet.service
manifest: |-
Type=oneshot
ExecStart=/usr/bin/systemctl stop update-engine.service
manifest: LSdpmOQebIkYoG0lRv8AUHCBIyg= (fingerprint)
name: disable-update-engine.service
roles:
- Master
- Node
- manifest: |-
Type=oneshot
ExecStart=/usr/bin/systemctl start apply-to-all.service
- manifest: FdicqEXLciSI1yRjQxsrye3QivU= (fingerprint)
name: apply-to-all.service
kubelet:
kubeconfigPath: /etc/kubernetes/igconfig.txt