diff --git a/Makefile b/Makefile index 6882a8787b..efff5337df 100644 --- a/Makefile +++ b/Makefile @@ -183,7 +183,8 @@ push-aws-dry: push ssh ${TARGET} sudo SKIP_PACKAGE_UPDATE=1 /tmp/nodeup --conf=/var/cache/kubernetes-install/kube_env.yaml --dryrun --v=8 push-gce-run: push - ssh ${TARGET} sudo SKIP_PACKAGE_UPDATE=1 /tmp/nodeup --conf=metadata://gce/config --v=8 + ssh ${TARGET} sudo cp /tmp/nodeup /home/kubernetes/bin/nodeup + ssh ${TARGET} sudo SKIP_PACKAGE_UPDATE=1 /home/kubernetes/bin/nodeup --conf=/var/cache/kubernetes-install/kube_env.yaml --v=8 # -t is for CentOS http://unix.stackexchange.com/questions/122616/why-do-i-need-a-tty-to-run-sudo-if-i-can-sudo-without-a-password push-aws-run: push diff --git a/channels/alpha b/channels/alpha index 13da154b82..7983f85c75 100644 --- a/channels/alpha +++ b/channels/alpha @@ -7,6 +7,8 @@ spec: - name: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09 providerID: aws kubernetesVersion: ">=1.5.0" + - providerID: gce + name: "cos-cloud/cos-stable-56-9000-84-2" cluster: kubernetesVersion: v1.5.4 networking: diff --git a/hack/.packages b/hack/.packages index 6f7276f6f1..b408d93479 100644 --- a/hack/.packages +++ b/hack/.packages @@ -28,6 +28,7 @@ k8s.io/kops/pkg/apis/kops/util k8s.io/kops/pkg/apis/kops/v1alpha1 k8s.io/kops/pkg/apis/kops/v1alpha2 k8s.io/kops/pkg/apis/kops/validation +k8s.io/kops/pkg/apis/nodeup k8s.io/kops/pkg/client/simple k8s.io/kops/pkg/client/simple/vfsclientset k8s.io/kops/pkg/diff diff --git a/nodeup/pkg/distros/distribution.go b/nodeup/pkg/distros/distribution.go index a099bf0558..5bf3701870 100644 --- a/nodeup/pkg/distros/distribution.go +++ b/nodeup/pkg/distros/distribution.go @@ -24,11 +24,12 @@ import ( type Distribution string var ( - DistributionJessie Distribution = "jessie" - DistributionXenial Distribution = "xenial" - DistributionRhel7 Distribution = "rhel7" - DistributionCentos7 Distribution = "centos7" - DistributionCoreOS Distribution = "coreos" + DistributionJessie Distribution = "jessie" + DistributionXenial Distribution = "xenial" + DistributionRhel7 Distribution = "rhel7" + DistributionCentos7 Distribution = "centos7" + DistributionCoreOS Distribution = "coreos" + DistributionContainerOS Distribution = "containeros" ) func (d Distribution) BuildTags() []string { @@ -45,6 +46,8 @@ func (d Distribution) BuildTags() []string { t = []string{"_rhel7"} case DistributionCoreOS: t = []string{"_coreos"} + case DistributionContainerOS: + t = []string{"_containeros"} default: glog.Fatalf("unknown distribution: %s", d) return nil @@ -67,7 +70,7 @@ func (d Distribution) IsDebianFamily() bool { switch d { case DistributionJessie, DistributionXenial: return true - case DistributionCentos7, DistributionRhel7, DistributionCoreOS: + case DistributionCentos7, DistributionRhel7, DistributionCoreOS, DistributionContainerOS: return false default: glog.Fatalf("unknown distribution: %s", d) @@ -79,7 +82,7 @@ func (d Distribution) IsRHELFamily() bool { switch d { case DistributionCentos7, DistributionRhel7: return true - case DistributionJessie, DistributionXenial, DistributionCoreOS: + case DistributionJessie, DistributionXenial, DistributionCoreOS, DistributionContainerOS: return false default: glog.Fatalf("unknown distribution: %s", d) @@ -95,6 +98,8 @@ func (d Distribution) IsSystemd() bool { return true case DistributionCoreOS: return true + case DistributionContainerOS: + return true default: glog.Fatalf("unknown distribution: %s", d) return false diff --git a/nodeup/pkg/distros/identify.go b/nodeup/pkg/distros/identify.go index 862aa5cb6f..e64227c880 100644 --- a/nodeup/pkg/distros/identify.go +++ b/nodeup/pkg/distros/identify.go @@ -87,5 +87,21 @@ func FindDistribution(rootfs string) (Distribution, error) { glog.Warningf("error reading /usr/lib/os-release: %v", err) } + // ContainerOS uses /etc/os-release + { + osRelease, err := ioutil.ReadFile(path.Join(rootfs, "etc/os-release")) + if err == nil { + for _, line := range strings.Split(string(osRelease), "\n") { + line = strings.TrimSpace(line) + if line == "ID=cos" { + return DistributionContainerOS, nil + } + } + glog.Warningf("unhandled /etc/os-release info %q", string(osRelease)) + } else if !os.IsNotExist(err) { + glog.Warningf("error reading /etc/os-release: %v", err) + } + } + return "", fmt.Errorf("cannot identify distro") } diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 9024ece6cd..607bcad5bc 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -19,10 +19,13 @@ package model import ( "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/upup/pkg/fi" ) type NodeupModelContext struct { + NodeupConfig *nodeup.NodeUpConfig + Cluster *kops.Cluster InstanceGroup *kops.InstanceGroup Architecture Architecture @@ -46,9 +49,39 @@ func (c *NodeupModelContext) SSLHostPaths() []string { paths = append(paths, "/usr/share/ca-certificates") + case distros.DistributionContainerOS: + paths = append(paths, "/usr/share/ca-certificates") + default: paths = append(paths, "/usr/share/ssl", "/usr/ssl", "/usr/lib/ssl", "/usr/local/openssl", "/var/ssl", "/etc/openssl") } return paths } + +func (c *NodeupModelContext) PathSrvKubernetes() string { + switch c.Distribution { + case distros.DistributionContainerOS: + return "/etc/srv/kubernetes" + default: + return "/srv/kubernetes" + } +} + +func (c *NodeupModelContext) PathSrvSshproxy() string { + switch c.Distribution { + case distros.DistributionContainerOS: + return "/etc/srv/sshproxy" + default: + return "/srv/sshproxy" + } +} + +func (c *NodeupModelContext) NetworkPluginDir() string { + switch c.Distribution { + case distros.DistributionContainerOS: + return "/home/kubernetes/bin/" + default: + return "/opt/cni/bin/" + } +} diff --git a/nodeup/pkg/model/directories.go b/nodeup/pkg/model/directories.go new file mode 100644 index 0000000000..5c4f402acb --- /dev/null +++ b/nodeup/pkg/model/directories.go @@ -0,0 +1,50 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "k8s.io/kops/nodeup/pkg/distros" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) + +// DirectoryBuilder creates required directories +type DirectoryBuilder struct { + *NodeupModelContext +} + +var _ fi.ModelBuilder = &DirectoryBuilder{} + +func (b *DirectoryBuilder) Build(c *fi.ModelBuilderContext) error { + if b.Distribution == distros.DistributionContainerOS { + dir := "/home/kubernetes/bin" + + t := &nodetasks.File{ + Path: dir, + Type: nodetasks.FileType_Directory, + Mode: s("0755"), + + OnChangeExecute: [][]string{ + {"/bin/mount", "--bind", "/home/kubernetes/bin", "/home/kubernetes/bin"}, + {"/bin/mount", "-o", "remount,exec", "/home/kubernetes/bin"}, + }, + } + c.AddTask(t) + } + + return nil +} diff --git a/nodeup/pkg/model/docker.go b/nodeup/pkg/model/docker.go index f6903342f3..c6dc1596b4 100644 --- a/nodeup/pkg/model/docker.go +++ b/nodeup/pkg/model/docker.go @@ -227,9 +227,14 @@ func (d *dockerVersion) matches(arch Architecture, dockerVersion string, distro } func (b *DockerBuilder) Build(c *fi.ModelBuilderContext) error { - if b.Distribution == distros.DistributionCoreOS { + switch b.Distribution { + case distros.DistributionCoreOS: glog.Infof("Detected CoreOS; won't install Docker") return nil + + case distros.DistributionContainerOS: + glog.Infof("Detected ContainerOS; won't install Docker") + return nil } // Add Apache2 license diff --git a/nodeup/pkg/model/etcd.go b/nodeup/pkg/model/etcd.go index fbf54f6036..f4ce591565 100644 --- a/nodeup/pkg/model/etcd.go +++ b/nodeup/pkg/model/etcd.go @@ -35,9 +35,14 @@ func (b *EtcdBuilder) Build(c *fi.ModelBuilderContext) error { return nil } - if b.Distribution == distros.DistributionCoreOS { + switch b.Distribution { + case distros.DistributionCoreOS: glog.Infof("Detected CoreOS; skipping etcd user installation") return nil + + case distros.DistributionContainerOS: + glog.Infof("Detected ContainerOS; skipping etcd user installation") + return nil } // TODO: Do we actually use the user anywhere? diff --git a/nodeup/pkg/model/firewall.go b/nodeup/pkg/model/firewall.go new file mode 100644 index 0000000000..9aa6e9015e --- /dev/null +++ b/nodeup/pkg/model/firewall.go @@ -0,0 +1,100 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "github.com/golang/glog" + "k8s.io/kops/nodeup/pkg/distros" + "k8s.io/kops/pkg/systemd" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) + +// FirewallBuilder configures the firewall (iptables) +type FirewallBuilder struct { + *NodeupModelContext +} + +var _ fi.ModelBuilder = &FirewallBuilder{} + +func (b *FirewallBuilder) Build(c *fi.ModelBuilderContext) error { + if b.Distribution == distros.DistributionContainerOS { + c.AddTask(b.buildFirewallScript()) + + c.AddTask(b.buildSystemdService()) + } + + return nil +} + +func (b *FirewallBuilder) buildSystemdService() *nodetasks.Service { + manifest := &systemd.Manifest{} + manifest.Set("Unit", "Description", "Configure iptables for kubernetes") + manifest.Set("Unit", "Documentation", "https://github.com/kubernetes/kops") + manifest.Set("Unit", "Before", "network.target") + + manifest.Set("Service", "Type", "oneshot") + manifest.Set("Service", "RemainAfterExit", "yes") + manifest.Set("Service", "ExecStart", "/home/kubernetes/bin/iptables-setup") + + manifest.Set("Install", "WantedBy", "basic.target") + + manifestString := manifest.Render() + glog.V(8).Infof("Built service manifest %q\n%s", "kubernetes-iptables-setup", manifestString) + + service := &nodetasks.Service{ + Name: "kubernetes-iptables-setup.service", + Definition: s(manifestString), + } + + service.InitDefaults() + + return service +} + +func (b *FirewallBuilder) buildFirewallScript() *nodetasks.File { + // TODO: Do we want to rely on running nodeup on every boot, or do we want to install systemd units? + + // TODO: The if statement in the script doesn't make it idempotent + + // This is borrowed from gce/gci/configure-helper.sh + script := `#!/bin/bash +# Built by kops - do not edit + +# The GCI image has host firewall which drop most inbound/forwarded packets. +# We need to add rules to accept all TCP/UDP/ICMP packets. +if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then +echo "Add rules to accept all inbound TCP/UDP/ICMP packets" +iptables -A INPUT -w -p TCP -j ACCEPT +iptables -A INPUT -w -p UDP -j ACCEPT +iptables -A INPUT -w -p ICMP -j ACCEPT +fi +if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then +echo "Add rules to accept all forwarded TCP/UDP/ICMP packets" +iptables -A FORWARD -w -p TCP -j ACCEPT +iptables -A FORWARD -w -p UDP -j ACCEPT +iptables -A FORWARD -w -p ICMP -j ACCEPT +fi +` + t := &nodetasks.File{ + Path: "/home/kubernetes/bin/iptables-setup", + Contents: fi.NewStringResource(script), + Type: nodetasks.FileType_File, + Mode: s("0755"), + } + return t +} diff --git a/nodeup/pkg/model/kubeapiserver.go b/nodeup/pkg/model/kubeapiserver.go index 4be0cb6bfc..eb4634ff11 100644 --- a/nodeup/pkg/model/kubeapiserver.go +++ b/nodeup/pkg/model/kubeapiserver.go @@ -18,6 +18,7 @@ package model import ( "fmt" + "path/filepath" "strings" "k8s.io/apimachinery/pkg/api/resource" @@ -64,6 +65,15 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { } func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { + kubeAPIServer := b.Cluster.Spec.KubeAPIServer + + kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.cert") + kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key") + + kubeAPIServer.BasicAuthFile = filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv") + kubeAPIServer.TokenAuthFile = filepath.Join(b.PathSrvKubernetes(), "known_tokens.csv") + flags, err := flagbuilder.BuildFlags(b.Cluster.Spec.KubeAPIServer) if err != nil { return nil, fmt.Errorf("error building kube-apiserver flags: %v", err) @@ -141,12 +151,14 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { addHostPathMapping(pod, container, "cloudconfig", CloudConfigFilePath, true) } - if b.Cluster.Spec.KubeAPIServer.PathSrvKubernetes != "" { - addHostPathMapping(pod, container, "srvkube", b.Cluster.Spec.KubeAPIServer.PathSrvKubernetes, true) + pathSrvKubernetes := b.PathSrvKubernetes() + if pathSrvKubernetes != "" { + addHostPathMapping(pod, container, "srvkube", pathSrvKubernetes, true) } - if b.Cluster.Spec.KubeAPIServer.PathSrvSshproxy != "" { - addHostPathMapping(pod, container, "srvsshproxy", b.Cluster.Spec.KubeAPIServer.PathSrvSshproxy, false) + pathSrvSshproxy := b.PathSrvSshproxy() + if pathSrvSshproxy != "" { + addHostPathMapping(pod, container, "srvsshproxy", pathSrvSshproxy, false) } addHostPathMapping(pod, container, "logfile", "/var/log/kube-apiserver.log", false) diff --git a/nodeup/pkg/model/kubecontrollermanager.go b/nodeup/pkg/model/kubecontrollermanager.go index 59a696f302..911576becc 100644 --- a/nodeup/pkg/model/kubecontrollermanager.go +++ b/nodeup/pkg/model/kubecontrollermanager.go @@ -18,6 +18,9 @@ package model import ( "fmt" + "path/filepath" + "strings" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -25,7 +28,6 @@ import ( "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" - "strings" ) // KubeControllerManagerBuilder install kube-controller-manager (just the manifest at the moment) @@ -63,7 +65,12 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { } func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) { - flags, err := flagbuilder.BuildFlags(b.Cluster.Spec.KubeControllerManager) + kcm := b.Cluster.Spec.KubeControllerManager + + kcm.RootCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + kcm.ServiceAccountPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key") + + flags, err := flagbuilder.BuildFlags(kcm) if err != nil { return nil, fmt.Errorf("error building kube-controller-manager flags: %v", err) } @@ -127,8 +134,9 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) { addHostPathMapping(pod, container, "cloudconfig", CloudConfigFilePath, true) } - if b.Cluster.Spec.KubeControllerManager.PathSrvKubernetes != "" { - addHostPathMapping(pod, container, "srvkube", b.Cluster.Spec.KubeControllerManager.PathSrvKubernetes, true) + pathSrvKubernetes := b.PathSrvKubernetes() + if pathSrvKubernetes != "" { + addHostPathMapping(pod, container, "srvkube", pathSrvKubernetes, true) } addHostPathMapping(pod, container, "logfile", "/var/log/kube-controller-manager.log", false) diff --git a/nodeup/pkg/model/kubectl.go b/nodeup/pkg/model/kubectl.go index c594e8c74e..5a6e45ec53 100644 --- a/nodeup/pkg/model/kubectl.go +++ b/nodeup/pkg/model/kubectl.go @@ -66,5 +66,8 @@ func (b *KubectlBuilder) kubectlPath() string { if b.Distribution == distros.DistributionCoreOS { kubeletCommand = "/opt/bin/kubectl" } + if b.Distribution == distros.DistributionContainerOS { + kubeletCommand = "/home/kubernetes/bin/kubectl" + } return kubeletCommand } diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index 93310d3957..0bd37d4443 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -57,6 +57,8 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { flags += " --cloud-config=" + CloudConfigFilePath } + flags += " --network-plugin-dir=" + b.NetworkPluginDir() + sysconfig := "DAEMON_ARGS=\"" + flags + "\"\n" t := &nodetasks.File{ @@ -126,6 +128,9 @@ func (b *KubeletBuilder) kubeletPath() string { if b.Distribution == distros.DistributionCoreOS { kubeletCommand = "/opt/kubernetes/bin/kubelet" } + if b.Distribution == distros.DistributionContainerOS { + kubeletCommand = "/home/kubernetes/bin/kubelet" + } return kubeletCommand } @@ -138,7 +143,7 @@ func (b *KubeletBuilder) buildSystemdService() *nodetasks.Service { manifest.Set("Unit", "After", "docker.service") if b.Distribution == distros.DistributionCoreOS { - // We add /opt/kubernetes/bin for our utilities + // We add /opt/kubernetes/bin for our utilities (socat) manifest.Set("Service", "Environment", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kubernetes/bin") } @@ -150,7 +155,7 @@ func (b *KubeletBuilder) buildSystemdService() *nodetasks.Service { manifest.Set("Service", "KillMode", "process") manifestString := manifest.Render() - glog.V(8).Infof("Built service manifest %q\n%s", "docker", manifestString) + glog.V(8).Infof("Built service manifest %q\n%s", "kubelet", manifestString) service := &nodetasks.Service{ Name: "kubelet.service", diff --git a/nodeup/pkg/model/logrotate.go b/nodeup/pkg/model/logrotate.go index e288d1602c..fb63d39cc5 100644 --- a/nodeup/pkg/model/logrotate.go +++ b/nodeup/pkg/model/logrotate.go @@ -36,6 +36,11 @@ func (b *LogrotateBuilder) Build(c *fi.ModelBuilderContext) error { return nil } + if b.Distribution == distros.DistributionContainerOS { + glog.Infof("Detected ContainerOS; won't install logrotate") + return nil + } + c.AddTask(&nodetasks.Package{Name: "logrotate"}) return nil diff --git a/nodeup/pkg/model/network.go b/nodeup/pkg/model/network.go new file mode 100644 index 0000000000..45f9a42d60 --- /dev/null +++ b/nodeup/pkg/model/network.go @@ -0,0 +1,85 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "fmt" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "path/filepath" +) + +// NetworkBuilder writes CNI assets +type NetworkBuilder struct { + *NodeupModelContext +} + +var _ fi.ModelBuilder = &NetworkBuilder{} + +func (b *NetworkBuilder) Build(c *fi.ModelBuilderContext) error { + var assetNames []string + + networking := b.Cluster.Spec.Networking + if networking == nil || networking.Classic != nil { + } else if networking.Kubenet != nil { + assetNames = append(assetNames, "bridge", "host-local", "loopback") + } else if networking.External != nil { + // external is based on kubenet + assetNames = append(assetNames, "bridge", "host-local", "loopback") + } else if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil { + assetNames = append(assetNames, "bridge", "host-local", "loopback", "ptp") + // Do we need tuning? + + // TODO: Only when using flannel ? + assetNames = append(assetNames, "flannel") + } else if networking.Kopeio != nil { + // TODO combine with External + // Kopeio is based on kubenet / external + assetNames = append(assetNames, "bridge", "host-local", "loopback") + } else { + return fmt.Errorf("No networking mode set") + } + + for _, assetName := range assetNames { + if err := b.addAsset(c, assetName); err != nil { + return err + } + } + + return nil +} + +func (b *NetworkBuilder) addAsset(c *fi.ModelBuilderContext, assetName string) error { + assetPath := "" + asset, err := b.Assets.Find(assetName, assetPath) + if err != nil { + return fmt.Errorf("error trying to locate asset %q: %v", assetName, err) + } + if asset == nil { + return fmt.Errorf("unable to locate asset %q", assetName) + } + + t := &nodetasks.File{ + Path: filepath.Join(b.NetworkPluginDir(), assetName), + Contents: asset, + Type: nodetasks.FileType_File, + Mode: s("0755"), + } + c.AddTask(t) + + return nil +} diff --git a/nodeup/pkg/model/protokube.go b/nodeup/pkg/model/protokube.go new file mode 100644 index 0000000000..0744d94eb8 --- /dev/null +++ b/nodeup/pkg/model/protokube.go @@ -0,0 +1,182 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "fmt" + "github.com/blang/semver" + "github.com/golang/glog" + "k8s.io/kops" + "k8s.io/kops/pkg/apis/kops/util" + "k8s.io/kops/pkg/flagbuilder" + "k8s.io/kops/pkg/systemd" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "strings" +) + +// ProtokubeBuilder configures protokube +type ProtokubeBuilder struct { + *NodeupModelContext +} + +var _ fi.ModelBuilder = &ProtokubeBuilder{} + +func (b *ProtokubeBuilder) Build(c *fi.ModelBuilderContext) error { + // TODO: Should we run _protokube on the nodes? + service, err := b.buildSystemdService() + if err != nil { + return err + } + c.AddTask(service) + + return nil +} + +func (b *ProtokubeBuilder) buildSystemdService() (*nodetasks.Service, error) { + k8sVersion, err := util.ParseKubernetesVersion(b.Cluster.Spec.KubernetesVersion) + if err != nil || k8sVersion == nil { + return nil, fmt.Errorf("unable to parse KubernetesVersion %q", b.Cluster.Spec.KubernetesVersion) + } + + protokubeFlags := b.ProtokubeFlags(*k8sVersion) + protokubeFlagsArgs, err := flagbuilder.BuildFlags(protokubeFlags) + if err != nil { + return nil, err + } + + protokubeCommand := "/usr/bin/docker run -v /:/rootfs/ -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd --net=host --privileged " + protokubeCommand += b.ProtokubeImageName() + " /usr/bin/protokube " + protokubeCommand += protokubeFlagsArgs + + manifest := &systemd.Manifest{} + manifest.Set("Unit", "Description", "Kubernetes Protokube Service") + manifest.Set("Unit", "Documentation", "https://github.com/kubernetes/kops") + + //manifest.Set("Service", "EnvironmentFile", "/etc/sysconfig/protokube") + manifest.Set("Service", "ExecStartPre", b.ProtokubeImagePullCommand()) + manifest.Set("Service", "ExecStart", protokubeCommand) + manifest.Set("Service", "Restart", "always") + manifest.Set("Service", "RestartSec", "2s") + manifest.Set("Service", "StartLimitInterval", "0") + + manifest.Set("Install", "WantedBy", "multi-user.target") + + manifestString := manifest.Render() + glog.V(8).Infof("Built service manifest %q\n%s", "protokube", manifestString) + + service := &nodetasks.Service{ + Name: "protokube.service", + Definition: s(manifestString), + } + + service.InitDefaults() + + return service, nil +} + +// ProtokubeImageName returns the docker image for protokube +func (t *ProtokubeBuilder) ProtokubeImageName() string { + name := "" + if t.NodeupConfig.ProtokubeImage != nil && t.NodeupConfig.ProtokubeImage.Name != "" { + name = t.NodeupConfig.ProtokubeImage.Name + } + if name == "" { + // use current default corresponding to this version of nodeup + name = kops.DefaultProtokubeImageName() + } + return name +} + +// ProtokubeImagePullCommand returns the command to pull the image +func (t *ProtokubeBuilder) ProtokubeImagePullCommand() string { + source := "" + if t.NodeupConfig.ProtokubeImage != nil { + source = t.NodeupConfig.ProtokubeImage.Source + } + if source == "" { + // Nothing to pull; return dummy value + return "/bin/true" + } + if strings.HasPrefix(source, "http:") || strings.HasPrefix(source, "https:") || strings.HasPrefix(source, "s3:") { + // We preloaded the image; return a dummy value + return "/bin/true" + } + return "/usr/bin/docker pull " + t.NodeupConfig.ProtokubeImage.Source +} + +type ProtokubeFlags struct { + Master *bool `json:"master,omitempty" flag:"master"` + Containerized *bool `json:"containerized,omitempty" flag:"containerized"` + LogLevel *int32 `json:"logLevel,omitempty" flag:"v"` + + DNSProvider *string `json:"dnsProvider,omitempty" flag:"dns"` + + Zone []string `json:"zone,omitempty" flag:"zone"` + + Channels []string `json:"channels,omitempty" flag:"channels"` + + DNSInternalSuffix *string `json:"dnsInternalSuffix,omitempty" flag:"dns-internal-suffix"` + Cloud *string `json:"cloud,omitempty" flag:"cloud"` +} + +// ProtokubeFlags returns the flags object for protokube +func (t *ProtokubeBuilder) ProtokubeFlags(k8sVersion semver.Version) *ProtokubeFlags { + f := &ProtokubeFlags{} + + master := t.IsMaster + + f.Master = fi.Bool(master) + if master { + f.Channels = t.NodeupConfig.Channels + } + + f.LogLevel = fi.Int32(4) + f.Containerized = fi.Bool(true) + + zone := t.Cluster.Spec.DNSZone + if zone != "" { + if strings.Contains(zone, ".") { + // match by name + f.Zone = append(f.Zone, zone) + } else { + // match by id + f.Zone = append(f.Zone, "*/"+zone) + } + } else { + glog.Warningf("DNSZone not specified; protokube won't be able to update DNS") + // TODO: Should we permit wildcard updates if zone is not specified? + //argv = append(argv, "--zone=*/*") + } + + if t.Cluster.Spec.CloudProvider != "" { + f.Cloud = fi.String(t.Cluster.Spec.CloudProvider) + + switch fi.CloudProviderID(t.Cluster.Spec.CloudProvider) { + case fi.CloudProviderAWS: + f.DNSProvider = fi.String("aws-route53") + case fi.CloudProviderGCE: + f.DNSProvider = fi.String("google-clouddns") + default: + glog.Warningf("Unknown cloudprovider %q; won't set DNS provider") + } + } + + f.DNSInternalSuffix = fi.String(".internal." + t.Cluster.ObjectMeta.Name) + + return f +} diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go new file mode 100644 index 0000000000..22608a29c7 --- /dev/null +++ b/nodeup/pkg/model/secrets.go @@ -0,0 +1,154 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "fmt" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "path/filepath" + "strings" +) + +// SecretBuilder writes secrets +type SecretBuilder struct { + *NodeupModelContext +} + +var _ fi.ModelBuilder = &SecretBuilder{} + +func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { + if b.KeyStore == nil { + return fmt.Errorf("KeyStore not set") + } + + { + ca, err := b.KeyStore.CertificatePool(fi.CertificateId_CA) + if err != nil { + return err + } + + serialized, err := ca.AsString() + if err != nil { + return err + } + + t := &nodetasks.File{ + Path: filepath.Join(b.PathSrvKubernetes(), "ca.crt"), + Contents: fi.NewStringResource(serialized), + Type: nodetasks.FileType_File, + } + c.AddTask(t) + } + + { + cert, err := b.KeyStore.Cert("master") + if err != nil { + return err + } + + serialized, err := cert.AsString() + if err != nil { + return err + } + + t := &nodetasks.File{ + Path: filepath.Join(b.PathSrvKubernetes(), "server.cert"), + Contents: fi.NewStringResource(serialized), + Type: nodetasks.FileType_File, + } + c.AddTask(t) + } + { + k, err := b.KeyStore.PrivateKey("master") + if err != nil { + return err + } + + serialized, err := k.AsString() + if err != nil { + return err + } + + t := &nodetasks.File{ + Path: filepath.Join(b.PathSrvKubernetes(), "server.key"), + Contents: fi.NewStringResource(serialized), + Type: nodetasks.FileType_File, + } + c.AddTask(t) + } + + if b.SecretStore != nil { + key := "kube" + token, err := b.SecretStore.FindSecret(key) + if err != nil { + return err + } + if token == nil { + return fmt.Errorf("token not found: %q", key) + } + csv := string(token.Data) + ",admin,admin" + + t := &nodetasks.File{ + Path: filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv"), + Contents: fi.NewStringResource(csv), + Type: nodetasks.FileType_File, + Mode: s("0600"), + } + c.AddTask(t) + } + + if b.SecretStore != nil { + allTokens, err := b.allTokens() + if err != nil { + return err + } + + var lines []string + for id, token := range allTokens { + lines = append(lines, token+","+id+","+id) + } + csv := strings.Join(lines, "\n") + + t := &nodetasks.File{ + Path: filepath.Join(b.PathSrvKubernetes(), "known_tokens.csv"), + Contents: fi.NewStringResource(csv), + Type: nodetasks.FileType_File, + Mode: s("0600"), + } + c.AddTask(t) + } + + return nil +} + +// allTokens returns a map of all tokens +func (b *SecretBuilder) allTokens() (map[string]string, error) { + tokens := make(map[string]string) + ids, err := b.SecretStore.ListSecrets() + if err != nil { + return nil, err + } + for _, id := range ids { + token, err := b.SecretStore.FindSecret(id) + if err != nil { + return nil, err + } + tokens[id] = string(token.Data) + } + return tokens, nil +} diff --git a/nodeup/pkg/model/sysctls.go b/nodeup/pkg/model/sysctls.go index 3834ef61bd..009b526c16 100644 --- a/nodeup/pkg/model/sysctls.go +++ b/nodeup/pkg/model/sysctls.go @@ -123,7 +123,7 @@ func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error { Path: "/etc/sysctl.d/99-k8s-general.conf", Contents: fi.NewStringResource(strings.Join(sysctls, "\n")), Type: nodetasks.FileType_File, - OnChangeExecute: []string{"sysctl", "--system"}, + OnChangeExecute: [][]string{{"sysctl", "--system"}}, } c.AddTask(t) diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 71168882ff..bfb66d5a25 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -377,9 +377,12 @@ type KubeProxyConfig struct { } type KubeAPIServerConfig struct { + // TODO: Remove PathSrvKubernetes - unused PathSrvKubernetes string `json:"pathSrvKubernetes,omitempty"` - PathSrvSshproxy string `json:"pathSrvSshproxy,omitempty"` - Image string `json:"image,omitempty"` + // TODO: Remove PathSrvSshProxy - unused + PathSrvSshproxy string `json:"pathSrvSshproxy,omitempty"` + + Image string `json:"image,omitempty"` LogLevel int32 `json:"logLevel,omitempty" flag:"v"` @@ -390,13 +393,22 @@ type KubeAPIServerConfig struct { EtcdServersOverrides []string `json:"etcdServersOverrides,omitempty" flag:"etcd-servers-overrides"` AdmissionControl []string `json:"admissionControl,omitempty" flag:"admission-control"` ServiceClusterIPRange string `json:"serviceClusterIPRange,omitempty" flag:"service-cluster-ip-range"` - ClientCAFile string `json:"clientCAFile,omitempty" flag:"client-ca-file"` - BasicAuthFile string `json:"basicAuthFile,omitempty" flag:"basic-auth-file"` - TLSCertFile string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` - TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty" flag:"tls-private-key-file"` - TokenAuthFile string `json:"tokenAuthFile,omitempty" flag:"token-auth-file"` - AllowPrivileged *bool `json:"allowPrivileged,omitempty" flag:"allow-privileged"` - APIServerCount *int32 `json:"apiServerCount,omitempty" flag:"apiserver-count"` + + // TODO: Remove unused BasicAuthFile + BasicAuthFile string `json:"basicAuthFile,omitempty" flag:"basic-auth-file"` + + // TODO: Remove unused ClientCAFile + ClientCAFile string `json:"clientCAFile,omitempty" flag:"client-ca-file"` + // TODO: Remove unused TLSCertFile + TLSCertFile string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` + // TODO: Remove unused TLSPrivateKeyFile + TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty" flag:"tls-private-key-file"` + + // TODO: Remove unused TokenAuthFile + TokenAuthFile string `json:"tokenAuthFile,omitempty" flag:"token-auth-file"` + + AllowPrivileged *bool `json:"allowPrivileged,omitempty" flag:"allow-privileged"` + APIServerCount *int32 `json:"apiServerCount,omitempty" flag:"apiserver-count"` // keys and values in RuntimeConfig are parsed into the `--runtime-config` parameter // for KubeAPIServer, concatenated with commas. ex: `--runtime-config=key1=value1,key2=value2`. // Use this to enable alpha resources on kube-apiserver @@ -445,10 +457,12 @@ type KubeControllerManagerConfig struct { Master string `json:"master,omitempty" flag:"master"` LogLevel int32 `json:"logLevel,omitempty" flag:"v" flag-empty:"0"` + // TODO: Remove as unused ServiceAccountPrivateKeyFile string `json:"serviceAccountPrivateKeyFile,omitempty" flag:"service-account-private-key-file"` Image string `json:"image,omitempty"` + // TODO: Remove PathSrvKubernetes - unused PathSrvKubernetes string `json:"pathSrvKubernetes,omitempty"` // Configuration flags - a subset of https://github.com/kubernetes/kubernetes/blob/master/pkg/apis/componentconfig/types.go @@ -563,9 +577,12 @@ type KubeControllerManagerConfig struct { // configureCloudRoutes enables CIDRs allocated with allocateNodeCIDRs // to be configured on the cloud provider. ConfigureCloudRoutes *bool `json:"configureCloudRoutes,omitempty" flag:"configure-cloud-routes"` + + // TODO: Remove as unused // rootCAFile is the root certificate authority will be included in service // account's token secret. This must be a valid PEM-encoded CA bundle. RootCAFile string `json:"rootCAFile,omitempty" flag:"root-ca-file"` + //// contentType is contentType of requests sent to apiserver. //ContentType string `json:"contentType"` //// kubeAPIQPS is the QPS to use while talking with kubernetes apiserver. diff --git a/upup/pkg/fi/nodeup/config.go b/pkg/apis/nodeup/config.go similarity index 100% rename from upup/pkg/fi/nodeup/config.go rename to pkg/apis/nodeup/config.go diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 7ce41ef608..1175b54483 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -18,17 +18,16 @@ package model import ( "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/model/resources" "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/nodeup" "text/template" ) // BootstrapScript creates the bootstrap script type BootstrapScript struct { - NodeUpSource string - NodeUpSourceHash string - + NodeUpSource string + NodeUpSourceHash string NodeUpConfigBuilder func(ig *kops.InstanceGroup) (*nodeup.NodeUpConfig, error) } diff --git a/pkg/model/components/kubecontrollermanager.go b/pkg/model/components/kubecontrollermanager.go index 2fb39614ee..a344172b8c 100644 --- a/pkg/model/components/kubecontrollermanager.go +++ b/pkg/model/components/kubecontrollermanager.go @@ -108,10 +108,6 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error return fmt.Errorf("unknown cloud provider %q", clusterSpec.CloudProvider) } - kcm.PathSrvKubernetes = "/srv/kubernetes" - kcm.RootCAFile = "/srv/kubernetes/ca.crt" - kcm.ServiceAccountPrivateKeyFile = "/srv/kubernetes/server.key" - kcm.Master = "127.0.0.1:8080" kcm.LogLevel = 2 diff --git a/pkg/model/resources/nodeup.go b/pkg/model/resources/nodeup.go index 0eaa91a90b..85329a6ad7 100644 --- a/pkg/model/resources/nodeup.go +++ b/pkg/model/resources/nodeup.go @@ -40,6 +40,10 @@ NODEUP_HASH={{ NodeUpSourceHash }} function ensure-install-dir() { INSTALL_DIR="/var/cache/kubernetes-install" + # On ContainerOS, we install to /var/lib/toolbox install (because of noexec) + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kubernetes-install" + fi mkdir -p ${INSTALL_DIR} cd ${INSTALL_DIR} } @@ -122,7 +126,7 @@ function download-release() { echo "Running nodeup" # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=/var/cache/kubernetes-install/kube_env.yaml --v=8 ) + ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 ) } #################################################################################### diff --git a/protokube/pkg/protokube/etcd_manifest.go b/protokube/pkg/protokube/etcd_manifest.go index 56bae4f899..bd8c56e38b 100644 --- a/protokube/pkg/protokube/etcd_manifest.go +++ b/protokube/pkg/protokube/etcd_manifest.go @@ -73,7 +73,7 @@ func BuildEtcdManifest(c *EtcdCluster) *v1.Pod { container.Env = append(container.Env, v1.EnvVar{Name: "ETCD_INITIAL_CLUSTER", Value: strings.Join(initialCluster, ",")}) container.LivenessProbe = &v1.Probe{ - InitialDelaySeconds: 600, + InitialDelaySeconds: 15, TimeoutSeconds: 15, } container.LivenessProbe.HTTPGet = &v1.HTTPGetAction{ diff --git a/protokube/pkg/protokube/volume_mounter.go b/protokube/pkg/protokube/volume_mounter.go index 72586fdbc9..a7b7bce5a4 100644 --- a/protokube/pkg/protokube/volume_mounter.go +++ b/protokube/pkg/protokube/volume_mounter.go @@ -58,6 +58,17 @@ func (k *VolumeMountController) mountMasterVolumes() ([]*Volume, error) { glog.V(2).Infof("Master volume %q is attached at %q", v.ID, v.LocalDevice) mountpoint := "/mnt/master-" + v.ID + + // On ContainerOS, we mount to /mnt/disks instead (/mnt is readonly) + _, err := os.Stat(PathFor("/mnt/disks")) + if err != nil { + if !os.IsNotExist(err) { + return nil, fmt.Errorf("error checking for /mnt/disks: %v", err) + } + } else { + mountpoint = "/mnt/disks/master-" + v.ID + } + glog.Infof("Doing safe-format-and-mount of %s to %s", v.LocalDevice, mountpoint) fstype := "" err = k.safeFormatAndMount(v.LocalDevice, mountpoint, fstype) diff --git a/protokube/tests/integration/build_etcd_manifest/main/manifest.yaml b/protokube/tests/integration/build_etcd_manifest/main/manifest.yaml index 85aba3ce16..66e2d504ad 100644 --- a/protokube/tests/integration/build_etcd_manifest/main/manifest.yaml +++ b/protokube/tests/integration/build_etcd_manifest/main/manifest.yaml @@ -37,7 +37,7 @@ spec: host: 127.0.0.1 path: /health port: 4001 - initialDelaySeconds: 600 + initialDelaySeconds: 15 timeoutSeconds: 15 name: etcd-container ports: @@ -63,4 +63,4 @@ spec: - hostPath: path: /var/log/main.log name: varlogetcd -status: {} \ No newline at end of file +status: {} diff --git a/tests/integration/minimal/cloudformation.json b/tests/integration/minimal/cloudformation.json index 36f4d12533..b144a697cc 100644 --- a/tests/integration/minimal/cloudformation.json +++ b/tests/integration/minimal/cloudformation.json @@ -93,7 +93,7 @@ "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" } ], - "UserData": "IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9rdWJldXB2Mi5zMy5hbWF6b25hd3MuY29tL2tvcHMvMS41LjAvbGludXgvYW1kNjQvbm9kZXVwCk5PREVVUF9IQVNIPQoKZnVuY3Rpb24gZW5zdXJlLWluc3RhbGwtZGlyKCkgewogIElOU1RBTExfRElSPSIvdmFyL2NhY2hlL2t1YmVybmV0ZXMtaW5zdGFsbCIKICBta2RpciAtcCAke0lOU1RBTExfRElSfQogIGNkICR7SU5TVEFMTF9ESVJ9Cn0KCiMgUmV0cnkgYSBkb3dubG9hZCB1bnRpbCB3ZSBnZXQgaXQuIFRha2VzIGEgaGFzaCBhbmQgYSBzZXQgb2YgVVJMcy4KIwojICQxIGlzIHRoZSBzaGExIG9mIHRoZSBVUkwuIENhbiBiZSAiIiBpZiB0aGUgc2hhMSBpcyB1bmtub3duLgojICQyKyBhcmUgdGhlIFVSTHMgdG8gZG93bmxvYWQuCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgaGFzaD0iJDEiCiAgc2hpZnQgMQoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgbG9jYWwgZmlsZT0iJHt1cmwjIyovfSIKICAgICAgcm0gLWYgIiR7ZmlsZX0iCiAgICAgIGlmICEgY3VybCAtZiAtLWlwdjQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCAiJHt1cmx9IjsgdGhlbgogICAgICAgIGVjaG8gIj09IEZhaWxlZCB0byBkb3dubG9hZCAke3VybH0uIFJldHJ5aW5nLiA9PSIKICAgICAgZWxpZiBbWyAtbiAiJHtoYXNofSIgXV0gJiYgISB2YWxpZGF0ZS1oYXNoICIke2ZpbGV9IiAiJHtoYXNofSI7IHRoZW4KICAgICAgICBlY2hvICI9PSBIYXNoIHZhbGlkYXRpb24gb2YgJHt1cmx9IGZhaWxlZC4gUmV0cnlpbmcuID09IgogICAgICBlbHNlCiAgICAgICAgaWYgW1sgLW4gIiR7aGFzaH0iIF1dOyB0aGVuCiAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgIGVsc2UKICAgICAgICAgIGVjaG8gIj09IERvd25sb2FkZWQgJHt1cmx9ID09IgogICAgICAgIGZpCiAgICAgICAgcmV0dXJuCiAgICAgIGZpCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMXN1bSAke2ZpbGV9IHwgYXdrICd7IHByaW50ICQxIH0nKSB8fCB0cnVlCiAgaWYgW1sgIiR7YWN0dWFsfSIgIT0gIiR7ZXhwZWN0ZWR9IiBdXTsgdGhlbgogICAgZWNobyAiPT0gJHtmaWxlfSBjb3JydXB0ZWQsIHNoYTEgJHthY3R1YWx9IGRvZXNuJ3QgbWF0Y2ggZXhwZWN0ZWQgJHtleHBlY3RlZH0gPT0iCiAgICByZXR1cm4gMQogIGZpCn0KCmZ1bmN0aW9uIHNwbGl0LWNvbW1hcygpIHsKICBlY2hvICQxIHwgdHIgIiwiICJcbiIKfQoKZnVuY3Rpb24gdHJ5LWRvd25sb2FkLXJlbGVhc2UoKSB7CiAgIyBUT0RPKHptZXJseW5uKTogTm93IHdlIFJFQUxMWSBoYXZlIG5vIGV4Y3VzZSBub3QgdG8gZG8gdGhlIHJlYm9vdAogICMgb3B0aW1pemF0aW9uLgoKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGxvY2FsIC1yIG5vZGV1cF9maWxlbmFtZT0iJHtub2RldXBfdXJsc1swXSMjKi99IgogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTEgKG5vdCBmb3VuZCBpbiBlbnYpIgogICAgZG93bmxvYWQtb3ItYnVzdCAiIiAiJHtub2RldXBfdXJsc1tAXS8lLy5zaGExfSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0ICIke25vZGV1cF9maWxlbmFtZX0uc2hhMSIpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCAiJHtub2RldXBfaGFzaH0iICIke25vZGV1cF91cmxzW0BdfSIKCiAgY2htb2QgK3ggbm9kZXVwCn0KCmZ1bmN0aW9uIGRvd25sb2FkLXJlbGVhc2UoKSB7CiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9OyAuL25vZGV1cCAtLWluc3RhbGwtc3lzdGVtZC11bml0IC0tY29uZj0vdmFyL2NhY2hlL2t1YmVybmV0ZXMtaW5zdGFsbC9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGt1YmVfZW52LnlhbWwgPDwgX19FT0ZfS1VCRV9FTlYKQXNzZXRzOgotIDdkNzBlMDkwOTUxNDg2Y2FlNTJkOWE4MmI3YWFmNTA1NmY4NGY4ZWRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjQuNi9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDlhZGNkMTIwZmRiN2FkNmU2NGMwNjFlNTZhMDVmZWZjMTJlOTYxOGJAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjQuNi9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDE5ZDQ5ZjdiMmI5OWNkMjQ5M2Q1YWUwYWNlODk2YzY0ZTI4OWNjYmJAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLTA3YThhMjg2MzdlOTdiMjJlYjhkZmU3MTBlZWFlMTM0NGY2OWQxNmUudGFyLmd6Ci0gY2JiYTg1Njc0NmE0NDFjN2QxYTllOTVlMTQxYzk4MmExYjg4NjRlNkBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjUuMC9saW51eC9hbWQ2NC91dGlscy50YXIuZ3oKQ2x1c3Rlck5hbWU6IG1pbmltYWwuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taW5pbWFsLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBtYXN0ZXItdXMtdGVzdC0xYQpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCi0gX2NuaV9icmlkZ2UKLSBfY25pX2ZsYW5uZWwKLSBfY25pX2hvc3RfbG9jYWwKLSBfY25pX2xvb3BiYWNrCi0gX2NuaV9wdHAKLSBfa3ViZXJuZXRlc19tYXN0ZXIKLSBfa3ViZXJuZXRlc19wb29sCi0gX3Byb3Rva3ViZQpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21pbmltYWwuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKcHJvdG9rdWJlSW1hZ2U6CiAgaGFzaDogN2MzYTBlYzA3MjNmZDM1MDYwOWIyOTU4YmM1YjhhYjAyNTgzODUxYwogIG5hbWU6IHByb3Rva3ViZToxLjUuMAogIHNvdXJjZTogaHR0cHM6Ly9rdWJldXB2Mi5zMy5hbWF6b25hd3MuY29tL2tvcHMvMS41LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo=" + "UserData": "IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9rdWJldXB2Mi5zMy5hbWF6b25hd3MuY29tL2tvcHMvMS41LjAvbGludXgvYW1kNjQvbm9kZXVwCk5PREVVUF9IQVNIPQoKZnVuY3Rpb24gZW5zdXJlLWluc3RhbGwtZGlyKCkgewogIElOU1RBTExfRElSPSIvdmFyL2NhY2hlL2t1YmVybmV0ZXMtaW5zdGFsbCIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHRvIC92YXIvbGliL3Rvb2xib3ggaW5zdGFsbCAoYmVjYXVzZSBvZiBub2V4ZWMpCiAgaWYgW1sgLWQgL3Zhci9saWIvdG9vbGJveCBdXTsgdGhlbgogICAgSU5TVEFMTF9ESVI9Ii92YXIvbGliL3Rvb2xib3gva3ViZXJuZXRlcy1pbnN0YWxsIgogIGZpCiAgbWtkaXIgLXAgJHtJTlNUQUxMX0RJUn0KICBjZCAke0lOU1RBTExfRElSfQp9CgojIFJldHJ5IGEgZG93bmxvYWQgdW50aWwgd2UgZ2V0IGl0LiBUYWtlcyBhIGhhc2ggYW5kIGEgc2V0IG9mIFVSTHMuCiMKIyAkMSBpcyB0aGUgc2hhMSBvZiB0aGUgVVJMLiBDYW4gYmUgIiIgaWYgdGhlIHNoYTEgaXMgdW5rbm93bi4KIyAkMisgYXJlIHRoZSBVUkxzIHRvIGRvd25sb2FkLgpkb3dubG9hZC1vci1idXN0KCkgewogIGxvY2FsIC1yIGhhc2g9IiQxIgogIHNoaWZ0IDEKCiAgdXJscz0oICQqICkKICB3aGlsZSB0cnVlOyBkbwogICAgZm9yIHVybCBpbiAiJHt1cmxzW0BdfSI7IGRvCiAgICAgIGxvY2FsIGZpbGU9IiR7dXJsIyMqL30iCiAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICBpZiAhIGN1cmwgLWYgLS1pcHY0IC1MbyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQgMjAgLS1yZXRyeSA2IC0tcmV0cnktZGVsYXkgMTAgIiR7dXJsfSI7IHRoZW4KICAgICAgICBlY2hvICI9PSBGYWlsZWQgdG8gZG93bmxvYWQgJHt1cmx9LiBSZXRyeWluZy4gPT0iCiAgICAgIGVsaWYgW1sgLW4gIiR7aGFzaH0iIF1dICYmICEgdmFsaWRhdGUtaGFzaCAiJHtmaWxlfSIgIiR7aGFzaH0iOyB0aGVuCiAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgZWxzZQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWRlZCAke3VybH0gKFNIQTEgPSAke2hhc2h9KSA9PSIKICAgICAgICBlbHNlCiAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICBmaQogICAgICAgIHJldHVybgogICAgICBmaQogICAgZG9uZQoKICAgIGVjaG8gIkFsbCBkb3dubG9hZHMgZmFpbGVkOyBzbGVlcGluZyBiZWZvcmUgcmV0cnlpbmciCiAgICBzbGVlcCA2MAogIGRvbmUKfQoKdmFsaWRhdGUtaGFzaCgpIHsKICBsb2NhbCAtciBmaWxlPSIkMSIKICBsb2NhbCAtciBleHBlY3RlZD0iJDIiCiAgbG9jYWwgYWN0dWFsCgogIGFjdHVhbD0kKHNoYTFzdW0gJHtmaWxlfSB8IGF3ayAneyBwcmludCAkMSB9JykgfHwgdHJ1ZQogIGlmIFtbICIke2FjdHVhbH0iICE9ICIke2V4cGVjdGVkfSIgXV07IHRoZW4KICAgIGVjaG8gIj09ICR7ZmlsZX0gY29ycnVwdGVkLCBzaGExICR7YWN0dWFsfSBkb2Vzbid0IG1hdGNoIGV4cGVjdGVkICR7ZXhwZWN0ZWR9ID09IgogICAgcmV0dXJuIDEKICBmaQp9CgpmdW5jdGlvbiBzcGxpdC1jb21tYXMoKSB7CiAgZWNobyAkMSB8IHRyICIsIiAiXG4iCn0KCmZ1bmN0aW9uIHRyeS1kb3dubG9hZC1yZWxlYXNlKCkgewogICMgVE9ETyh6bWVybHlubik6IE5vdyB3ZSBSRUFMTFkgaGF2ZSBubyBleGN1c2Ugbm90IHRvIGRvIHRoZSByZWJvb3QKICAjIG9wdGltaXphdGlvbi4KCiAgbG9jYWwgLXIgbm9kZXVwX3VybHM9KCAkKHNwbGl0LWNvbW1hcyAiJHtOT0RFVVBfVVJMfSIpICkKICBsb2NhbCAtciBub2RldXBfZmlsZW5hbWU9IiR7bm9kZXVwX3VybHNbMF0jIyovfSIKICBpZiBbWyAtbiAiJHtOT0RFVVBfSEFTSDotfSIgXV07IHRoZW4KICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSIke05PREVVUF9IQVNIfSIKICBlbHNlCiAgIyBUT0RPOiBSZW1vdmU/CiAgICBlY2hvICJEb3dubG9hZGluZyBzaGExIChub3QgZm91bmQgaW4gZW52KSIKICAgIGRvd25sb2FkLW9yLWJ1c3QgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMX0iCiAgICBsb2NhbCAtciBub2RldXBfaGFzaD0kKGNhdCAiJHtub2RldXBfZmlsZW5hbWV9LnNoYTEiKQogIGZpCgogIGVjaG8gIkRvd25sb2FkaW5nIG5vZGV1cCAoJHtub2RldXBfdXJsc1tAXX0pIgogIGRvd25sb2FkLW9yLWJ1c3QgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogICMgSW4gY2FzZSBvZiBmYWlsdXJlIGNoZWNraW5nIGludGVncml0eSBvZiByZWxlYXNlLCByZXRyeS4KICB1bnRpbCB0cnktZG93bmxvYWQtcmVsZWFzZTsgZG8KICAgIHNsZWVwIDE1CiAgICBlY2hvICJDb3VsZG4ndCBkb3dubG9hZCByZWxlYXNlLiBSZXRyeWluZy4uLiIKICBkb25lCgogIGVjaG8gIlJ1bm5pbmcgbm9kZXVwIgogICMgV2UgY2FuJ3QgcnVuIGluIHRoZSBmb3JlZ3JvdW5kIGJlY2F1c2Ugb2YgaHR0cHM6Ly9naXRodWIuY29tL2RvY2tlci9kb2NrZXIvaXNzdWVzLzIzNzkzCiAgKCBjZCAke0lOU1RBTExfRElSfTsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0va3ViZV9lbnYueWFtbCAtLXY9OCAgKQp9CgojIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMKCi9iaW4vc3lzdGVtZC1tYWNoaW5lLWlkLXNldHVwIHx8IGVjaG8gImZhaWxlZCB0byBzZXQgdXAgZW5zdXJlIG1hY2hpbmUtaWQgY29uZmlndXJlZCIKCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBzdGFydGluZyA9PSIKZW5zdXJlLWluc3RhbGwtZGlyCgpjYXQgPiBrdWJlX2Vudi55YW1sIDw8IF9fRU9GX0tVQkVfRU5WCkFzc2V0czoKLSA3ZDcwZTA5MDk1MTQ4NmNhZTUyZDlhODJiN2FhZjUwNTZmODRmOGVkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS40LjYvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA5YWRjZDEyMGZkYjdhZDZlNjRjMDYxZTU2YTA1ZmVmYzEyZTk2MThiQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS40LjYvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSAxOWQ0OWY3YjJiOTljZDI0OTNkNWFlMGFjZTg5NmM2NGUyODljY2JiQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS0wN2E4YTI4NjM3ZTk3YjIyZWI4ZGZlNzEwZWVhZTEzNDRmNjlkMTZlLnRhci5negotIGNiYmE4NTY3NDZhNDQxYzdkMWE5ZTk1ZTE0MWM5ODJhMWI4ODY0ZTZAaHR0cHM6Ly9rdWJldXB2Mi5zMy5hbWF6b25hd3MuY29tL2tvcHMvMS41LjAvbGludXgvYW1kNjQvdXRpbHMudGFyLmd6CkNsdXN0ZXJOYW1lOiBtaW5pbWFsLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vbWluaW1hbC5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwotIF9rdWJlcm5ldGVzX21hc3RlcgotIF9rdWJlcm5ldGVzX3Bvb2wKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taW5pbWFsLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDdjM2EwZWMwNzIzZmQzNTA2MDliMjk1OGJjNWI4YWIwMjU4Mzg1MWMKICBuYW1lOiBwcm90b2t1YmU6MS41LjAKICBzb3VyY2U6IGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuNS4wL2ltYWdlcy9wcm90b2t1YmUudGFyLmd6CgpfX0VPRl9LVUJFX0VOVgoKZG93bmxvYWQtcmVsZWFzZQplY2hvICI9PSBub2RldXAgbm9kZSBjb25maWcgZG9uZSA9PSIK" } }, "AWSAutoScalingLaunchConfigurationnodesminimalexamplecom": { @@ -121,7 +121,7 @@ "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" } ], - "UserData": "IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9rdWJldXB2Mi5zMy5hbWF6b25hd3MuY29tL2tvcHMvMS41LjAvbGludXgvYW1kNjQvbm9kZXVwCk5PREVVUF9IQVNIPQoKZnVuY3Rpb24gZW5zdXJlLWluc3RhbGwtZGlyKCkgewogIElOU1RBTExfRElSPSIvdmFyL2NhY2hlL2t1YmVybmV0ZXMtaW5zdGFsbCIKICBta2RpciAtcCAke0lOU1RBTExfRElSfQogIGNkICR7SU5TVEFMTF9ESVJ9Cn0KCiMgUmV0cnkgYSBkb3dubG9hZCB1bnRpbCB3ZSBnZXQgaXQuIFRha2VzIGEgaGFzaCBhbmQgYSBzZXQgb2YgVVJMcy4KIwojICQxIGlzIHRoZSBzaGExIG9mIHRoZSBVUkwuIENhbiBiZSAiIiBpZiB0aGUgc2hhMSBpcyB1bmtub3duLgojICQyKyBhcmUgdGhlIFVSTHMgdG8gZG93bmxvYWQuCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgaGFzaD0iJDEiCiAgc2hpZnQgMQoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgbG9jYWwgZmlsZT0iJHt1cmwjIyovfSIKICAgICAgcm0gLWYgIiR7ZmlsZX0iCiAgICAgIGlmICEgY3VybCAtZiAtLWlwdjQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCAiJHt1cmx9IjsgdGhlbgogICAgICAgIGVjaG8gIj09IEZhaWxlZCB0byBkb3dubG9hZCAke3VybH0uIFJldHJ5aW5nLiA9PSIKICAgICAgZWxpZiBbWyAtbiAiJHtoYXNofSIgXV0gJiYgISB2YWxpZGF0ZS1oYXNoICIke2ZpbGV9IiAiJHtoYXNofSI7IHRoZW4KICAgICAgICBlY2hvICI9PSBIYXNoIHZhbGlkYXRpb24gb2YgJHt1cmx9IGZhaWxlZC4gUmV0cnlpbmcuID09IgogICAgICBlbHNlCiAgICAgICAgaWYgW1sgLW4gIiR7aGFzaH0iIF1dOyB0aGVuCiAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgIGVsc2UKICAgICAgICAgIGVjaG8gIj09IERvd25sb2FkZWQgJHt1cmx9ID09IgogICAgICAgIGZpCiAgICAgICAgcmV0dXJuCiAgICAgIGZpCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMXN1bSAke2ZpbGV9IHwgYXdrICd7IHByaW50ICQxIH0nKSB8fCB0cnVlCiAgaWYgW1sgIiR7YWN0dWFsfSIgIT0gIiR7ZXhwZWN0ZWR9IiBdXTsgdGhlbgogICAgZWNobyAiPT0gJHtmaWxlfSBjb3JydXB0ZWQsIHNoYTEgJHthY3R1YWx9IGRvZXNuJ3QgbWF0Y2ggZXhwZWN0ZWQgJHtleHBlY3RlZH0gPT0iCiAgICByZXR1cm4gMQogIGZpCn0KCmZ1bmN0aW9uIHNwbGl0LWNvbW1hcygpIHsKICBlY2hvICQxIHwgdHIgIiwiICJcbiIKfQoKZnVuY3Rpb24gdHJ5LWRvd25sb2FkLXJlbGVhc2UoKSB7CiAgIyBUT0RPKHptZXJseW5uKTogTm93IHdlIFJFQUxMWSBoYXZlIG5vIGV4Y3VzZSBub3QgdG8gZG8gdGhlIHJlYm9vdAogICMgb3B0aW1pemF0aW9uLgoKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGxvY2FsIC1yIG5vZGV1cF9maWxlbmFtZT0iJHtub2RldXBfdXJsc1swXSMjKi99IgogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTEgKG5vdCBmb3VuZCBpbiBlbnYpIgogICAgZG93bmxvYWQtb3ItYnVzdCAiIiAiJHtub2RldXBfdXJsc1tAXS8lLy5zaGExfSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0ICIke25vZGV1cF9maWxlbmFtZX0uc2hhMSIpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCAiJHtub2RldXBfaGFzaH0iICIke25vZGV1cF91cmxzW0BdfSIKCiAgY2htb2QgK3ggbm9kZXVwCn0KCmZ1bmN0aW9uIGRvd25sb2FkLXJlbGVhc2UoKSB7CiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9OyAuL25vZGV1cCAtLWluc3RhbGwtc3lzdGVtZC11bml0IC0tY29uZj0vdmFyL2NhY2hlL2t1YmVybmV0ZXMtaW5zdGFsbC9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGt1YmVfZW52LnlhbWwgPDwgX19FT0ZfS1VCRV9FTlYKQXNzZXRzOgotIDdkNzBlMDkwOTUxNDg2Y2FlNTJkOWE4MmI3YWFmNTA1NmY4NGY4ZWRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjQuNi9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDlhZGNkMTIwZmRiN2FkNmU2NGMwNjFlNTZhMDVmZWZjMTJlOTYxOGJAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjQuNi9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDE5ZDQ5ZjdiMmI5OWNkMjQ5M2Q1YWUwYWNlODk2YzY0ZTI4OWNjYmJAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLTA3YThhMjg2MzdlOTdiMjJlYjhkZmU3MTBlZWFlMTM0NGY2OWQxNmUudGFyLmd6Ci0gY2JiYTg1Njc0NmE0NDFjN2QxYTllOTVlMTQxYzk4MmExYjg4NjRlNkBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjUuMC9saW51eC9hbWQ2NC91dGlscy50YXIuZ3oKQ2x1c3Rlck5hbWU6IG1pbmltYWwuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taW5pbWFsLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBub2RlcwpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCi0gX2NuaV9icmlkZ2UKLSBfY25pX2ZsYW5uZWwKLSBfY25pX2hvc3RfbG9jYWwKLSBfY25pX2xvb3BiYWNrCi0gX2NuaV9wdHAKLSBfa3ViZXJuZXRlc19wb29sCi0gX3Byb3Rva3ViZQpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21pbmltYWwuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKcHJvdG9rdWJlSW1hZ2U6CiAgaGFzaDogN2MzYTBlYzA3MjNmZDM1MDYwOWIyOTU4YmM1YjhhYjAyNTgzODUxYwogIG5hbWU6IHByb3Rva3ViZToxLjUuMAogIHNvdXJjZTogaHR0cHM6Ly9rdWJldXB2Mi5zMy5hbWF6b25hd3MuY29tL2tvcHMvMS41LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo=" + "UserData": "IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9rdWJldXB2Mi5zMy5hbWF6b25hd3MuY29tL2tvcHMvMS41LjAvbGludXgvYW1kNjQvbm9kZXVwCk5PREVVUF9IQVNIPQoKZnVuY3Rpb24gZW5zdXJlLWluc3RhbGwtZGlyKCkgewogIElOU1RBTExfRElSPSIvdmFyL2NhY2hlL2t1YmVybmV0ZXMtaW5zdGFsbCIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHRvIC92YXIvbGliL3Rvb2xib3ggaW5zdGFsbCAoYmVjYXVzZSBvZiBub2V4ZWMpCiAgaWYgW1sgLWQgL3Zhci9saWIvdG9vbGJveCBdXTsgdGhlbgogICAgSU5TVEFMTF9ESVI9Ii92YXIvbGliL3Rvb2xib3gva3ViZXJuZXRlcy1pbnN0YWxsIgogIGZpCiAgbWtkaXIgLXAgJHtJTlNUQUxMX0RJUn0KICBjZCAke0lOU1RBTExfRElSfQp9CgojIFJldHJ5IGEgZG93bmxvYWQgdW50aWwgd2UgZ2V0IGl0LiBUYWtlcyBhIGhhc2ggYW5kIGEgc2V0IG9mIFVSTHMuCiMKIyAkMSBpcyB0aGUgc2hhMSBvZiB0aGUgVVJMLiBDYW4gYmUgIiIgaWYgdGhlIHNoYTEgaXMgdW5rbm93bi4KIyAkMisgYXJlIHRoZSBVUkxzIHRvIGRvd25sb2FkLgpkb3dubG9hZC1vci1idXN0KCkgewogIGxvY2FsIC1yIGhhc2g9IiQxIgogIHNoaWZ0IDEKCiAgdXJscz0oICQqICkKICB3aGlsZSB0cnVlOyBkbwogICAgZm9yIHVybCBpbiAiJHt1cmxzW0BdfSI7IGRvCiAgICAgIGxvY2FsIGZpbGU9IiR7dXJsIyMqL30iCiAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICBpZiAhIGN1cmwgLWYgLS1pcHY0IC1MbyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQgMjAgLS1yZXRyeSA2IC0tcmV0cnktZGVsYXkgMTAgIiR7dXJsfSI7IHRoZW4KICAgICAgICBlY2hvICI9PSBGYWlsZWQgdG8gZG93bmxvYWQgJHt1cmx9LiBSZXRyeWluZy4gPT0iCiAgICAgIGVsaWYgW1sgLW4gIiR7aGFzaH0iIF1dICYmICEgdmFsaWRhdGUtaGFzaCAiJHtmaWxlfSIgIiR7aGFzaH0iOyB0aGVuCiAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgZWxzZQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWRlZCAke3VybH0gKFNIQTEgPSAke2hhc2h9KSA9PSIKICAgICAgICBlbHNlCiAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICBmaQogICAgICAgIHJldHVybgogICAgICBmaQogICAgZG9uZQoKICAgIGVjaG8gIkFsbCBkb3dubG9hZHMgZmFpbGVkOyBzbGVlcGluZyBiZWZvcmUgcmV0cnlpbmciCiAgICBzbGVlcCA2MAogIGRvbmUKfQoKdmFsaWRhdGUtaGFzaCgpIHsKICBsb2NhbCAtciBmaWxlPSIkMSIKICBsb2NhbCAtciBleHBlY3RlZD0iJDIiCiAgbG9jYWwgYWN0dWFsCgogIGFjdHVhbD0kKHNoYTFzdW0gJHtmaWxlfSB8IGF3ayAneyBwcmludCAkMSB9JykgfHwgdHJ1ZQogIGlmIFtbICIke2FjdHVhbH0iICE9ICIke2V4cGVjdGVkfSIgXV07IHRoZW4KICAgIGVjaG8gIj09ICR7ZmlsZX0gY29ycnVwdGVkLCBzaGExICR7YWN0dWFsfSBkb2Vzbid0IG1hdGNoIGV4cGVjdGVkICR7ZXhwZWN0ZWR9ID09IgogICAgcmV0dXJuIDEKICBmaQp9CgpmdW5jdGlvbiBzcGxpdC1jb21tYXMoKSB7CiAgZWNobyAkMSB8IHRyICIsIiAiXG4iCn0KCmZ1bmN0aW9uIHRyeS1kb3dubG9hZC1yZWxlYXNlKCkgewogICMgVE9ETyh6bWVybHlubik6IE5vdyB3ZSBSRUFMTFkgaGF2ZSBubyBleGN1c2Ugbm90IHRvIGRvIHRoZSByZWJvb3QKICAjIG9wdGltaXphdGlvbi4KCiAgbG9jYWwgLXIgbm9kZXVwX3VybHM9KCAkKHNwbGl0LWNvbW1hcyAiJHtOT0RFVVBfVVJMfSIpICkKICBsb2NhbCAtciBub2RldXBfZmlsZW5hbWU9IiR7bm9kZXVwX3VybHNbMF0jIyovfSIKICBpZiBbWyAtbiAiJHtOT0RFVVBfSEFTSDotfSIgXV07IHRoZW4KICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSIke05PREVVUF9IQVNIfSIKICBlbHNlCiAgIyBUT0RPOiBSZW1vdmU/CiAgICBlY2hvICJEb3dubG9hZGluZyBzaGExIChub3QgZm91bmQgaW4gZW52KSIKICAgIGRvd25sb2FkLW9yLWJ1c3QgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMX0iCiAgICBsb2NhbCAtciBub2RldXBfaGFzaD0kKGNhdCAiJHtub2RldXBfZmlsZW5hbWV9LnNoYTEiKQogIGZpCgogIGVjaG8gIkRvd25sb2FkaW5nIG5vZGV1cCAoJHtub2RldXBfdXJsc1tAXX0pIgogIGRvd25sb2FkLW9yLWJ1c3QgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogICMgSW4gY2FzZSBvZiBmYWlsdXJlIGNoZWNraW5nIGludGVncml0eSBvZiByZWxlYXNlLCByZXRyeS4KICB1bnRpbCB0cnktZG93bmxvYWQtcmVsZWFzZTsgZG8KICAgIHNsZWVwIDE1CiAgICBlY2hvICJDb3VsZG4ndCBkb3dubG9hZCByZWxlYXNlLiBSZXRyeWluZy4uLiIKICBkb25lCgogIGVjaG8gIlJ1bm5pbmcgbm9kZXVwIgogICMgV2UgY2FuJ3QgcnVuIGluIHRoZSBmb3JlZ3JvdW5kIGJlY2F1c2Ugb2YgaHR0cHM6Ly9naXRodWIuY29tL2RvY2tlci9kb2NrZXIvaXNzdWVzLzIzNzkzCiAgKCBjZCAke0lOU1RBTExfRElSfTsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0va3ViZV9lbnYueWFtbCAtLXY9OCAgKQp9CgojIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMKCi9iaW4vc3lzdGVtZC1tYWNoaW5lLWlkLXNldHVwIHx8IGVjaG8gImZhaWxlZCB0byBzZXQgdXAgZW5zdXJlIG1hY2hpbmUtaWQgY29uZmlndXJlZCIKCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBzdGFydGluZyA9PSIKZW5zdXJlLWluc3RhbGwtZGlyCgpjYXQgPiBrdWJlX2Vudi55YW1sIDw8IF9fRU9GX0tVQkVfRU5WCkFzc2V0czoKLSA3ZDcwZTA5MDk1MTQ4NmNhZTUyZDlhODJiN2FhZjUwNTZmODRmOGVkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS40LjYvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA5YWRjZDEyMGZkYjdhZDZlNjRjMDYxZTU2YTA1ZmVmYzEyZTk2MThiQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS40LjYvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSAxOWQ0OWY3YjJiOTljZDI0OTNkNWFlMGFjZTg5NmM2NGUyODljY2JiQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS0wN2E4YTI4NjM3ZTk3YjIyZWI4ZGZlNzEwZWVhZTEzNDRmNjlkMTZlLnRhci5negotIGNiYmE4NTY3NDZhNDQxYzdkMWE5ZTk1ZTE0MWM5ODJhMWI4ODY0ZTZAaHR0cHM6Ly9rdWJldXB2Mi5zMy5hbWF6b25hd3MuY29tL2tvcHMvMS41LjAvbGludXgvYW1kNjQvdXRpbHMudGFyLmd6CkNsdXN0ZXJOYW1lOiBtaW5pbWFsLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vbWluaW1hbC5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbm9kZXMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwotIF9rdWJlcm5ldGVzX3Bvb2wKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taW5pbWFsLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDdjM2EwZWMwNzIzZmQzNTA2MDliMjk1OGJjNWI4YWIwMjU4Mzg1MWMKICBuYW1lOiBwcm90b2t1YmU6MS41LjAKICBzb3VyY2U6IGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuNS4wL2ltYWdlcy9wcm90b2t1YmUudGFyLmd6CgpfX0VPRl9LVUJFX0VOVgoKZG93bmxvYWQtcmVsZWFzZQplY2hvICI9PSBub2RldXAgbm9kZSBjb25maWcgZG9uZSA9PSIK" } }, "AWSEC2DHCPOptionsminimalexamplecom": { diff --git a/upup/models/config/components/kube-apiserver/kube-apiserver.options b/upup/models/config/components/kube-apiserver/kube-apiserver.options index 1d04728427..5b89c7a370 100644 --- a/upup/models/config/components/kube-apiserver/kube-apiserver.options +++ b/upup/models/config/components/kube-apiserver/kube-apiserver.options @@ -1,18 +1,11 @@ KubeAPIServer: SecurePort: 443 - PathSrvKubernetes: /srv/kubernetes - PathSrvSshproxy: /srv/sshproxy Address: 127.0.0.1 EtcdServers: - http://127.0.0.1:4001 EtcdServersOverrides: - /events#http://127.0.0.1:4002 ServiceClusterIPRange: {{ .ServiceClusterIPRange }} - ClientCAFile: /srv/kubernetes/ca.crt - BasicAuthFile: /srv/kubernetes/basic_auth.csv - TLSCertFile: /srv/kubernetes/server.cert - TLSPrivateKeyFile: /srv/kubernetes/server.key - TokenAuthFile: /srv/kubernetes/known_tokens.csv LogLevel: 2 AllowPrivileged: true Image: {{ Image "kube-apiserver" }} diff --git a/upup/models/nodeup/_kubernetes_master/certs/files/srv/kubernetes/ca.crt.template b/upup/models/nodeup/_kubernetes_master/certs/files/srv/kubernetes/ca.crt.template deleted file mode 100644 index 81eacbd433..0000000000 --- a/upup/models/nodeup/_kubernetes_master/certs/files/srv/kubernetes/ca.crt.template +++ /dev/null @@ -1 +0,0 @@ -{{ CACertificatePool.AsString }} \ No newline at end of file diff --git a/upup/models/nodeup/_kubernetes_master/certs/files/srv/kubernetes/server.cert.template b/upup/models/nodeup/_kubernetes_master/certs/files/srv/kubernetes/server.cert.template deleted file mode 100644 index 45820a290e..0000000000 --- a/upup/models/nodeup/_kubernetes_master/certs/files/srv/kubernetes/server.cert.template +++ /dev/null @@ -1 +0,0 @@ -{{ (Certificate "master").AsString }} \ No newline at end of file diff --git a/upup/models/nodeup/_kubernetes_master/certs/files/srv/kubernetes/server.key.template b/upup/models/nodeup/_kubernetes_master/certs/files/srv/kubernetes/server.key.template deleted file mode 100644 index ed5c0552c4..0000000000 --- a/upup/models/nodeup/_kubernetes_master/certs/files/srv/kubernetes/server.key.template +++ /dev/null @@ -1 +0,0 @@ -{{ (PrivateKey "master").AsString }} \ No newline at end of file diff --git a/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/etc/kubernetes/manifests/etcd-events.manifest b/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/etc/kubernetes/manifests/etcd-events.manifest deleted file mode 100644 index be5bb0c486..0000000000 --- a/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/etc/kubernetes/manifests/etcd-events.manifest +++ /dev/null @@ -1,68 +0,0 @@ -{ -"apiVersion": "v1", -"kind": "Pod", -"metadata": { - "name":"etcd-server-events", - "namespace": "kube-system", - "labels": { - "k8s-app" : "etcd-server-events" - } -}, -"spec":{ -"hostNetwork": true, -"containers":[ - { - "name": "etcd-container", - "image": "gcr.io/google_containers/etcd:2.2.1", - "resources": { - "requests": { - "cpu": "100m" - } - }, - "command": [ - "/bin/sh", - "-c", - "/usr/local/bin/etcd --listen-peer-urls http://127.0.0.1:2381 --addr 127.0.0.1:4002 --bind-addr 127.0.0.1:4002 --data-dir /var/etcd/data-events 1>>/var/log/etcd-events.log 2>&1" - ], - "livenessProbe": { - "httpGet": { - "host": "127.0.0.1", - "port": 4002, - "path": "/health" - }, - "initialDelaySeconds": 15, - "timeoutSeconds": 15 - }, - "ports":[ - { "name": "serverport", - "containerPort": 2381, - "hostPort": 2381 - },{ - "name": "clientport", - "containerPort": 4002, - "hostPort": 4002 - } - ], - "volumeMounts": [ - {"name": "varetcd", - "mountPath": "/var/etcd", - "readOnly": false - }, - {"name": "varlogetcd", - "mountPath": "/var/log/etcd-events.log", - "readOnly": false - } - ] - } -], -"volumes":[ - { "name": "varetcd", - "hostPath": { - "path": "/mnt/master-pd/var/etcd"} - }, - { "name": "varlogetcd", - "hostPath": { - "path": "/var/log/etcd-events.log"} - } -] -}} diff --git a/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/etc/kubernetes/manifests/etcd.manifest b/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/etc/kubernetes/manifests/etcd.manifest deleted file mode 100644 index 91519208e8..0000000000 --- a/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/etc/kubernetes/manifests/etcd.manifest +++ /dev/null @@ -1,68 +0,0 @@ -{ -"apiVersion": "v1", -"kind": "Pod", -"metadata": { - "name":"etcd-server", - "namespace": "kube-system", - "labels": { - "k8s-app" : "etcd-server" - } -}, -"spec":{ -"hostNetwork": true, -"containers":[ - { - "name": "etcd-container", - "image": "gcr.io/google_containers/etcd:2.2.1", - "resources": { - "requests": { - "cpu": "200m" - } - }, - "command": [ - "/bin/sh", - "-c", - "/usr/local/bin/etcd --listen-peer-urls http://127.0.0.1:2380 --addr 127.0.0.1:4001 --bind-addr 127.0.0.1:4001 --data-dir /var/etcd/data 1>>/var/log/etcd.log 2>&1" - ], - "livenessProbe": { - "httpGet": { - "host": "127.0.0.1", - "port": 4001, - "path": "/health" - }, - "initialDelaySeconds": 15, - "timeoutSeconds": 15 - }, - "ports":[ - { "name": "serverport", - "containerPort": 2380, - "hostPort": 2380 - },{ - "name": "clientport", - "containerPort": 4001, - "hostPort": 4001 - } - ], - "volumeMounts": [ - {"name": "varetcd", - "mountPath": "/var/etcd", - "readOnly": false - }, - {"name": "varlogetcd", - "mountPath": "/var/log/etcd.log", - "readOnly": false - } - ] - } -], -"volumes":[ - { "name": "varetcd", - "hostPath": { - "path": "/mnt/master-pd/var/etcd"} - }, - { "name": "varlogetcd", - "hostPath": { - "path": "/var/log/etcd.log"} - } -] -}} diff --git a/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/var/log/etcd-events.log b/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/var/log/etcd-events.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/var/log/etcd-events.log.meta b/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/var/log/etcd-events.log.meta deleted file mode 100644 index 56d0e34103..0000000000 --- a/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/var/log/etcd-events.log.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "ifNotExists": true -} \ No newline at end of file diff --git a/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/var/log/etcd.log b/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/var/log/etcd.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/var/log/etcd.log.meta b/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/var/log/etcd.log.meta deleted file mode 100644 index 56d0e34103..0000000000 --- a/upup/models/nodeup/_kubernetes_master/etcd/_not_protokube/var/log/etcd.log.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "ifNotExists": true -} \ No newline at end of file diff --git a/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/basic_auth.csv.template b/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/basic_auth.csv.template deleted file mode 100644 index fe42d43648..0000000000 --- a/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/basic_auth.csv.template +++ /dev/null @@ -1 +0,0 @@ -{{ GetToken "kube" }},admin,admin diff --git a/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/basic_auth.csv.template.meta b/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/basic_auth.csv.template.meta deleted file mode 100644 index 26aab40054..0000000000 --- a/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/basic_auth.csv.template.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mode": "0600" -} \ No newline at end of file diff --git a/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/known_tokens.csv.template b/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/known_tokens.csv.template deleted file mode 100644 index 45b9a540b1..0000000000 --- a/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/known_tokens.csv.template +++ /dev/null @@ -1,3 +0,0 @@ -{{ range $id, $token := AllTokens }} -{{ $token }},{{ $id }},{{ $id }} -{{ end }} diff --git a/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/known_tokens.csv.template.meta b/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/known_tokens.csv.template.meta deleted file mode 100644 index 26aab40054..0000000000 --- a/upup/models/nodeup/_kubernetes_master/kube-apiserver/files/srv/kubernetes/known_tokens.csv.template.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mode": "0600" -} \ No newline at end of file diff --git a/upup/models/nodeup/_protokube/files/etc/sysconfig/protokube.template b/upup/models/nodeup/_protokube/files/etc/sysconfig/protokube.template deleted file mode 100644 index fbbdee42a6..0000000000 --- a/upup/models/nodeup/_protokube/files/etc/sysconfig/protokube.template +++ /dev/null @@ -1 +0,0 @@ -DAEMON_ARGS="{{ BuildFlags ProtokubeFlags }}" diff --git a/upup/models/nodeup/_protokube/services/protokube.service.template b/upup/models/nodeup/_protokube/services/protokube.service.template deleted file mode 100644 index 684f4541ed..0000000000 --- a/upup/models/nodeup/_protokube/services/protokube.service.template +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Kubernetes Protokube Service -Documentation=https://github.com/kubernetes/kube-deploy/protokube -After=docker.service - -[Service] -EnvironmentFile=/etc/sysconfig/protokube -ExecStartPre={{ ProtokubeImagePullCommand }} -ExecStart=/usr/bin/docker run -v /:/rootfs/ -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd --net=host --privileged {{ ProtokubeImageName }} /usr/bin/protokube "$DAEMON_ARGS" -Restart=always -RestartSec=2s -StartLimitInterval=0 - -[Install] -WantedBy=multi-user.target diff --git a/upup/models/nodeup/networking/_cni_bridge/files/opt/cni/bin/bridge.asset b/upup/models/nodeup/networking/_cni_bridge/files/opt/cni/bin/bridge.asset deleted file mode 100644 index 7a73a41bfd..0000000000 --- a/upup/models/nodeup/networking/_cni_bridge/files/opt/cni/bin/bridge.asset +++ /dev/null @@ -1,2 +0,0 @@ -{ -} \ No newline at end of file diff --git a/upup/models/nodeup/networking/_cni_bridge/files/opt/cni/bin/bridge.asset.meta b/upup/models/nodeup/networking/_cni_bridge/files/opt/cni/bin/bridge.asset.meta deleted file mode 100644 index aac9c2ebd8..0000000000 --- a/upup/models/nodeup/networking/_cni_bridge/files/opt/cni/bin/bridge.asset.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mode": "0755" -} \ No newline at end of file diff --git a/upup/models/nodeup/networking/_cni_flannel/files/opt/cni/bin/flannel.asset b/upup/models/nodeup/networking/_cni_flannel/files/opt/cni/bin/flannel.asset deleted file mode 100644 index 7a73a41bfd..0000000000 --- a/upup/models/nodeup/networking/_cni_flannel/files/opt/cni/bin/flannel.asset +++ /dev/null @@ -1,2 +0,0 @@ -{ -} \ No newline at end of file diff --git a/upup/models/nodeup/networking/_cni_flannel/files/opt/cni/bin/flannel.asset.meta b/upup/models/nodeup/networking/_cni_flannel/files/opt/cni/bin/flannel.asset.meta deleted file mode 100644 index aac9c2ebd8..0000000000 --- a/upup/models/nodeup/networking/_cni_flannel/files/opt/cni/bin/flannel.asset.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mode": "0755" -} \ No newline at end of file diff --git a/upup/models/nodeup/networking/_cni_host_local/files/opt/cni/bin/host-local.asset b/upup/models/nodeup/networking/_cni_host_local/files/opt/cni/bin/host-local.asset deleted file mode 100644 index 7a73a41bfd..0000000000 --- a/upup/models/nodeup/networking/_cni_host_local/files/opt/cni/bin/host-local.asset +++ /dev/null @@ -1,2 +0,0 @@ -{ -} \ No newline at end of file diff --git a/upup/models/nodeup/networking/_cni_host_local/files/opt/cni/bin/host-local.asset.meta b/upup/models/nodeup/networking/_cni_host_local/files/opt/cni/bin/host-local.asset.meta deleted file mode 100644 index aac9c2ebd8..0000000000 --- a/upup/models/nodeup/networking/_cni_host_local/files/opt/cni/bin/host-local.asset.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mode": "0755" -} \ No newline at end of file diff --git a/upup/models/nodeup/networking/_cni_loopback/files/opt/cni/bin/loopback.asset b/upup/models/nodeup/networking/_cni_loopback/files/opt/cni/bin/loopback.asset deleted file mode 100644 index 7a73a41bfd..0000000000 --- a/upup/models/nodeup/networking/_cni_loopback/files/opt/cni/bin/loopback.asset +++ /dev/null @@ -1,2 +0,0 @@ -{ -} \ No newline at end of file diff --git a/upup/models/nodeup/networking/_cni_loopback/files/opt/cni/bin/loopback.asset.meta b/upup/models/nodeup/networking/_cni_loopback/files/opt/cni/bin/loopback.asset.meta deleted file mode 100644 index aac9c2ebd8..0000000000 --- a/upup/models/nodeup/networking/_cni_loopback/files/opt/cni/bin/loopback.asset.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mode": "0755" -} \ No newline at end of file diff --git a/upup/models/nodeup/networking/_cni_ptp/files/opt/cni/bin/ptp.asset b/upup/models/nodeup/networking/_cni_ptp/files/opt/cni/bin/ptp.asset deleted file mode 100644 index 7a73a41bfd..0000000000 --- a/upup/models/nodeup/networking/_cni_ptp/files/opt/cni/bin/ptp.asset +++ /dev/null @@ -1,2 +0,0 @@ -{ -} \ No newline at end of file diff --git a/upup/models/nodeup/networking/_cni_ptp/files/opt/cni/bin/ptp.asset.meta b/upup/models/nodeup/networking/_cni_ptp/files/opt/cni/bin/ptp.asset.meta deleted file mode 100644 index aac9c2ebd8..0000000000 --- a/upup/models/nodeup/networking/_cni_ptp/files/opt/cni/bin/ptp.asset.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mode": "0755" -} \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 28c64b3b13..56e1d3ddbd 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -30,6 +30,7 @@ import ( "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/kops/validation" + "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/client/simple" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model" @@ -44,7 +45,6 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/gcetasks" "k8s.io/kops/upup/pkg/fi/cloudup/terraform" "k8s.io/kops/upup/pkg/fi/fitasks" - "k8s.io/kops/upup/pkg/fi/nodeup" "k8s.io/kops/util/pkg/hashing" "k8s.io/kops/util/pkg/vfs" ) diff --git a/upup/pkg/fi/cloudup/tagbuilder.go b/upup/pkg/fi/cloudup/tagbuilder.go index 6a2ebac6fa..08b88b0a54 100644 --- a/upup/pkg/fi/cloudup/tagbuilder.go +++ b/upup/pkg/fi/cloudup/tagbuilder.go @@ -116,9 +116,6 @@ func buildNodeupTags(role api.InstanceGroupRole, cluster *api.Cluster, clusterTa case api.InstanceGroupRoleNode: tags.Insert("_kubernetes_pool") - // TODO: Should we run _protokube on the nodes? - tags.Insert("_protokube") - case api.InstanceGroupRoleMaster: tags.Insert("_kubernetes_master") @@ -127,8 +124,6 @@ func buildNodeupTags(role api.InstanceGroupRole, cluster *api.Cluster, clusterTa tags.Insert("_kubernetes_pool") } - tags.Insert("_protokube") - case api.InstanceGroupRoleBastion: // No tags @@ -136,12 +131,6 @@ func buildNodeupTags(role api.InstanceGroupRole, cluster *api.Cluster, clusterTa return nil, fmt.Errorf("Unrecognized role: %v", role) } - // TODO: Replace with list of CNI plugins ? - if usesCNI(cluster) { - tags.Insert("_cni_bridge", "_cni_host_local", "_cni_loopback", "_cni_ptp", "_cni_flannel") - //tags.Insert("_cni_tuning") - } - switch fi.StringValue(cluster.Spec.UpdatePolicy) { case "": // default tags.Insert("_automatic_upgrades") diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 2c0d757d82..d9d3a0a010 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -31,6 +31,7 @@ import ( "k8s.io/kops/nodeup/pkg/model" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/registry" + "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/cloudinit" "k8s.io/kops/upup/pkg/fi/nodeup/local" @@ -44,7 +45,7 @@ import ( const MaxTaskDuration = 365 * 24 * time.Hour type NodeUpCommand struct { - config *NodeUpConfig + config *nodeup.NodeUpConfig cluster *api.Cluster instanceGroup *api.InstanceGroup ConfigLocation string @@ -195,6 +196,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { } modelContext := &model.NodeupModelContext{ + NodeupConfig: c.config, Cluster: c.cluster, Distribution: distribution, Architecture: model.ArchitectureAmd64, @@ -207,12 +209,18 @@ func (c *NodeUpCommand) Run(out io.Writer) error { } loader := NewLoader(c.config, c.cluster, assets, nodeTags) + loader.Builders = append(loader.Builders, &model.DirectoryBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.DockerBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.ProtokubeBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.CloudConfigBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeletBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubectlBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.EtcdBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.LogrotateBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.PackagesBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.SecretBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.FirewallBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.NetworkBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.SysctlBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeAPIServerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeControllerManagerBuilder{NodeupModelContext: modelContext}) diff --git a/upup/pkg/fi/nodeup/loader.go b/upup/pkg/fi/nodeup/loader.go index f3fb546631..b62e3da2d0 100644 --- a/upup/pkg/fi/nodeup/loader.go +++ b/upup/pkg/fi/nodeup/loader.go @@ -23,6 +23,7 @@ import ( "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/sets" api "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/loader" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" @@ -35,7 +36,7 @@ type Loader struct { Builders []fi.ModelBuilder templates []*template.Template - config *NodeUpConfig + config *nodeup.NodeUpConfig cluster *api.Cluster assets *fi.AssetStore @@ -45,7 +46,7 @@ type Loader struct { TemplateFunctions template.FuncMap } -func NewLoader(config *NodeUpConfig, cluster *api.Cluster, assets *fi.AssetStore, tags sets.String) *Loader { +func NewLoader(config *nodeup.NodeUpConfig, cluster *api.Cluster, assets *fi.AssetStore, tags sets.String) *Loader { l := &Loader{} l.assets = assets l.tasks = make(map[string]fi.Task) diff --git a/upup/pkg/fi/nodeup/nodetasks/file.go b/upup/pkg/fi/nodeup/nodetasks/file.go index 40027eb2ef..162363beea 100644 --- a/upup/pkg/fi/nodeup/nodetasks/file.go +++ b/upup/pkg/fi/nodeup/nodetasks/file.go @@ -42,7 +42,7 @@ type File struct { Mode *string `json:"mode,omitempty"` IfNotExists bool `json:"ifNotExists,omitempty"` - OnChangeExecute []string `json:"onChangeExecute,omitempty"` + OnChangeExecute [][]string `json:"onChangeExecute,omitempty"` Symlink *string `json:"symlink,omitempty"` Owner *string `json:"owner,omitempty"` @@ -96,6 +96,23 @@ func (f *File) GetDependencies(tasks map[string]fi.Task) []fi.Task { } } + // Files depend on parent directories + for _, v := range tasks { + dir, ok := v.(*File) + if !ok { + continue + } + if dir.Type == FileType_Directory { + dirPath := dir.Path + if !strings.HasSuffix(dirPath, "/") { + dirPath += "/" + } + if strings.HasPrefix(f.Path, dirPath) { + deps = append(deps, v) + } + } + } + return deps } @@ -261,15 +278,16 @@ func (_ *File) RenderLocal(t *local.LocalTarget, a, e, changes *File) error { } if changed && e.OnChangeExecute != nil { - args := e.OnChangeExecute - human := strings.Join(args, " ") + for _, args := range e.OnChangeExecute { + human := strings.Join(args, " ") - glog.Infof("Changed; will execute OnChangeExecute command: %q", human) + glog.Infof("Changed; will execute OnChangeExecute command: %q", human) - cmd := exec.Command(args[0], args[1:]...) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("error executing command %q: %v\nOutput: %s", human, err, output) + cmd := exec.Command(args[0], args[1:]...) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error executing command %q: %v\nOutput: %s", human, err, output) + } } } @@ -303,7 +321,8 @@ func (_ *File) RenderCloudInit(t *cloudinit.CloudInitTarget, a, e, changes *File } if e.OnChangeExecute != nil { - t.AddCommand(cloudinit.Always, e.OnChangeExecute...) + return fmt.Errorf("OnChangeExecute not supported with CloudInit") + //t.AddCommand(cloudinit.Always, e.OnChangeExecute...) } return nil diff --git a/upup/pkg/fi/nodeup/nodetasks/service.go b/upup/pkg/fi/nodeup/nodetasks/service.go index ad9bc4b034..e708304143 100644 --- a/upup/pkg/fi/nodeup/nodetasks/service.go +++ b/upup/pkg/fi/nodeup/nodetasks/service.go @@ -42,6 +42,8 @@ const ( centosSystemdSystemPath = "/usr/lib/systemd/system" coreosSystemdSystemPath = "/etc/systemd/system" + + containerosSystemdSystemPath = "/etc/systemd/system" ) type Service struct { @@ -146,6 +148,8 @@ func (e *Service) systemdSystemPath(target tags.HasTags) (string, error) { return centosSystemdSystemPath, nil } else if target.HasTag("_coreos") { return coreosSystemdSystemPath, nil + } else if target.HasTag("_containeros") { + return containerosSystemdSystemPath, nil } else { return "", fmt.Errorf("unsupported systemd system") } diff --git a/upup/pkg/fi/nodeup/protokube_flags.go b/upup/pkg/fi/nodeup/protokube_flags.go deleted file mode 100644 index ade733371a..0000000000 --- a/upup/pkg/fi/nodeup/protokube_flags.go +++ /dev/null @@ -1,32 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nodeup - -type ProtokubeFlags struct { - Master *bool `json:"master,omitempty" flag:"master"` - Containerized *bool `json:"containerized,omitempty" flag:"containerized"` - LogLevel *int32 `json:"logLevel,omitempty" flag:"v"` - - DNSProvider *string `json:"dnsProvider,omitempty" flag:"dns"` - - Zone []string `json:"zone,omitempty" flag:"zone"` - - Channels []string `json:"channels,omitempty" flag:"channels"` - - DNSInternalSuffix *string `json:"dnsInternalSuffix,omitempty" flag:"dns-internal-suffix"` - Cloud *string `json:"cloud,omitempty" flag:"cloud"` -} diff --git a/upup/pkg/fi/nodeup/template_functions.go b/upup/pkg/fi/nodeup/template_functions.go index 4de3de6537..a0e4d709e8 100644 --- a/upup/pkg/fi/nodeup/template_functions.go +++ b/upup/pkg/fi/nodeup/template_functions.go @@ -20,13 +20,12 @@ import ( "encoding/base64" "fmt" "runtime" - "strings" "text/template" "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/kops" api "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/secrets" @@ -37,7 +36,7 @@ const TagMaster = "_kubernetes_master" // templateFunctions is a simple helper-class for the functions accessible to templates type templateFunctions struct { - nodeupConfig *NodeUpConfig + nodeupConfig *nodeup.NodeUpConfig // cluster is populated with the current cluster cluster *api.Cluster @@ -53,7 +52,7 @@ type templateFunctions struct { } // newTemplateFunctions is the constructor for templateFunctions -func newTemplateFunctions(nodeupConfig *NodeUpConfig, cluster *api.Cluster, instanceGroup *api.InstanceGroup, tags sets.String) (*templateFunctions, error) { +func newTemplateFunctions(nodeupConfig *nodeup.NodeUpConfig, cluster *api.Cluster, instanceGroup *api.InstanceGroup, tags sets.String) (*templateFunctions, error) { t := &templateFunctions{ nodeupConfig: nodeupConfig, cluster: cluster, @@ -93,11 +92,9 @@ func (t *templateFunctions) populate(dest template.FuncMap) { return runtime.GOARCH } - dest["CACertificatePool"] = t.CACertificatePool dest["CACertificate"] = t.CACertificate dest["PrivateKey"] = t.PrivateKey dest["Certificate"] = t.Certificate - dest["AllTokens"] = t.AllTokens dest["GetToken"] = t.GetToken dest["BuildFlags"] = flagbuilder.BuildFlags @@ -123,31 +120,6 @@ func (t *templateFunctions) populate(dest template.FuncMap) { dest["ClusterName"] = func() string { return t.cluster.ObjectMeta.Name } - - dest["ProtokubeImageName"] = t.ProtokubeImageName - dest["ProtokubeImagePullCommand"] = t.ProtokubeImagePullCommand - - dest["ProtokubeFlags"] = t.ProtokubeFlags -} - -// CACertificatePool returns the set of valid CA certificates for the cluster -func (t *templateFunctions) CACertificatePool() (*fi.CertificatePool, error) { - if t.keyStore != nil { - return t.keyStore.CertificatePool(fi.CertificateId_CA) - } - - // Fallback to direct properties - glog.Infof("Falling back to direct configuration for keystore") - cert, err := t.CACertificate() - if err != nil { - return nil, err - } - if cert == nil { - return nil, fmt.Errorf("CA certificate not found (with fallback)") - } - pool := &fi.CertificatePool{} - pool.Primary = cert - return pool, nil } // CACertificate returns the primary CA certificate for the cluster @@ -165,23 +137,6 @@ func (t *templateFunctions) Certificate(id string) (*fi.Certificate, error) { return t.keyStore.Cert(id) } -// AllTokens returns a map of all tokens -func (t *templateFunctions) AllTokens() (map[string]string, error) { - tokens := make(map[string]string) - ids, err := t.secretStore.ListSecrets() - if err != nil { - return nil, err - } - for _, id := range ids { - token, err := t.secretStore.FindSecret(id) - if err != nil { - return nil, err - } - tokens[id] = string(token.Data) - } - return tokens, nil -} - // GetToken returns the specified token func (t *templateFunctions) GetToken(key string) (string, error) { token, err := t.secretStore.FindSecret(key) @@ -194,36 +149,6 @@ func (t *templateFunctions) GetToken(key string) (string, error) { return string(token.Data), nil } -// ProtokubeImageName returns the docker image for protokube -func (t *templateFunctions) ProtokubeImageName() string { - name := "" - if t.nodeupConfig.ProtokubeImage != nil && t.nodeupConfig.ProtokubeImage.Name != "" { - name = t.nodeupConfig.ProtokubeImage.Name - } - if name == "" { - // use current default corresponding to this version of nodeup - name = kops.DefaultProtokubeImageName() - } - return name -} - -// ProtokubeImagePullCommand returns the command to pull the image -func (t *templateFunctions) ProtokubeImagePullCommand() string { - source := "" - if t.nodeupConfig.ProtokubeImage != nil { - source = t.nodeupConfig.ProtokubeImage.Source - } - if source == "" { - // Nothing to pull; return dummy value - return "/bin/true" - } - if strings.HasPrefix(source, "http:") || strings.HasPrefix(source, "https:") || strings.HasPrefix(source, "s3:") { - // We preloaded the image; return a dummy value - return "/bin/true" - } - return "/usr/bin/docker pull " + t.nodeupConfig.ProtokubeImage.Source -} - // IsMaster returns true if we are tagged as a master func (t *templateFunctions) isMaster() bool { return t.hasTag(TagMaster) @@ -235,53 +160,6 @@ func (t *templateFunctions) hasTag(tag string) bool { return found } -// ProtokubeFlags returns the flags object for protokube -func (t *templateFunctions) ProtokubeFlags() *ProtokubeFlags { - f := &ProtokubeFlags{} - - master := t.isMaster() - - f.Master = fi.Bool(master) - if master { - f.Channels = t.nodeupConfig.Channels - } - - f.LogLevel = fi.Int32(4) - f.Containerized = fi.Bool(true) - - zone := t.cluster.Spec.DNSZone - if zone != "" { - if strings.Contains(zone, ".") { - // match by name - f.Zone = append(f.Zone, zone) - } else { - // match by id - f.Zone = append(f.Zone, "*/"+zone) - } - } else { - glog.Warningf("DNSZone not specified; protokube won't be able to update DNS") - // TODO: Should we permit wildcard updates if zone is not specified? - //argv = append(argv, "--zone=*/*") - } - - if t.cluster.Spec.CloudProvider != "" { - f.Cloud = fi.String(t.cluster.Spec.CloudProvider) - - switch fi.CloudProviderID(t.cluster.Spec.CloudProvider) { - case fi.CloudProviderAWS: - f.DNSProvider = fi.String("aws-route53") - case fi.CloudProviderGCE: - f.DNSProvider = fi.String("google-clouddns") - default: - glog.Warningf("Unknown cloudprovider %q; won't set DNS provider") - } - } - - f.DNSInternalSuffix = fi.String(".internal." + t.cluster.ObjectMeta.Name) - - return f -} - // KubeProxyConfig builds the KubeProxyConfig configuration object func (t *templateFunctions) KubeProxyConfig() *api.KubeProxyConfig { config := &api.KubeProxyConfig{}