protokube: build etcd manifest in code

images/protokube/Dockerfile

@@ -20,9 +20,6 @@ RUN apt-get update && apt-get install --yes ca-certificates e2fsprogs
 
 COPY /.build/artifacts/kubectl /usr/bin/kubectl
 
-COPY protokube/model/ /model/
-COPY protokube/templates/ /templates/
-
 COPY /.build/artifacts/protokube /usr/bin/protokube
 COPY /.build/artifacts/channels /usr/bin/channels
 

nodeup/pkg/model/codecs.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package model
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/runtime"
+
+	_ "k8s.io/kubernetes/pkg/api/install"
+)
+
+func encoder() runtime.Encoder {
+	// TODO: Which is better way to build yaml?
+	//yaml := json.NewYAMLSerializer(json.DefaultMetaFactory, k8sapi.Scheme, k8sapi.Scheme)
+
+	// TODO: Cache?
+	yaml, ok := runtime.SerializerInfoForMediaType(api.Codecs.SupportedMediaTypes(), "application/yaml")
+	if !ok {
+		glog.Fatalf("no YAML serializer registered")
+	}
+	gv := v1.SchemeGroupVersion
+	return api.Codecs.EncoderForVersion(yaml.Serializer, gv)
+}
+
+// ToVersionedYamlWithVersion encodes the object to YAML
+func ToVersionedYaml(obj runtime.Object) ([]byte, error) {
+	var w bytes.Buffer
+	err := encoder().Encode(obj, &w)
+	if err != nil {
+		return nil, fmt.Errorf("error encoding %T: %v", obj, err)
+	}
+	return w.Bytes(), nil
+}

protokube/model/etcd/events.config

@@ -1,6 +0,0 @@
-ClusterName: etcd-events
-ClientPort: 4002
-PeerPort: 2381
-DataDirName: data-events
-PodName: etcd-server-events
-CPURequest: 100m

protokube/model/etcd/main.config

@@ -1,6 +0,0 @@
-ClusterName: etcd
-ClientPort: 4001
-PeerPort: 2380
-DataDirName: data
-PodName: etcd-server
-CPURequest: 150m

protokube/pkg/protokube/codecs.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protokube
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/runtime"
+
+	_ "k8s.io/kubernetes/pkg/api/install"
+)
+
+func encoder() runtime.Encoder {
+	yaml, ok := runtime.SerializerInfoForMediaType(api.Codecs.SupportedMediaTypes(), "application/yaml")
+	if !ok {
+		glog.Fatalf("no YAML serializer registered")
+	}
+	gv := v1.SchemeGroupVersion
+	return api.Codecs.EncoderForVersion(yaml.Serializer, gv)
+}
+
+// ToVersionedYaml encodes the object to YAML
+func ToVersionedYaml(obj runtime.Object) ([]byte, error) {
+	var w bytes.Buffer
+	err := encoder().Encode(obj, &w)
+	if err != nil {
+		return nil, fmt.Errorf("error encoding %T: %v", obj, err)
+	}
+	return w.Bytes(), nil
+}

protokube/pkg/protokube/etcd_cluster.go

@@ -19,15 +19,26 @@ package protokube
 import (
 	"bytes"
 	"fmt"
-	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
 	"io/ioutil"
+	"k8s.io/kubernetes/pkg/api/resource"
 	"os"
 	"path"
 	"strings"
 	"time"
 )
 
+type EtcdClusterSpec struct {
+	ClusterKey string `json:"clusterKey,omitempty"`
+
+	NodeName  string   `json:"nodeName,omitempty"`
+	NodeNames []string `json:"nodeNames,omitempty"`
+}
+
+func (e *EtcdClusterSpec) String() string {
+	return DebugString(e)
+}
+
 type EtcdCluster struct {
 	PeerPort     int
 	ClientPort   int
@@ -38,7 +49,7 @@ type EtcdCluster struct {
 	Me           *EtcdNode
 	Nodes        []*EtcdNode
 	PodName      string
-	CPURequest   string
+	CPURequest   resource.Quantity
 
 	Spec *EtcdClusterSpec
 
@@ -71,25 +82,34 @@ func newEtcdController(kubeBoot *KubeBoot, v *Volume, spec *EtcdClusterSpec) (*E
 		kubeBoot: kubeBoot,
 	}
 
-	modelTemplatePath := path.Join(kubeBoot.ModelDir, spec.ClusterKey+".config")
-	modelTemplate, err := ioutil.ReadFile(modelTemplatePath)
-	if err != nil {
-		return nil, fmt.Errorf("error reading model template %q: %v", modelTemplatePath, err)
-	}
-
 	cluster := &EtcdCluster{}
 	cluster.Spec = spec
 	cluster.VolumeMountPath = v.Mountpoint
 
-	model, err := ExecuteTemplate("model-etcd-"+spec.ClusterKey, string(modelTemplate), cluster)
-	if err != nil {
-		return nil, fmt.Errorf("error executing etcd model template %q: %v", modelTemplatePath, err)
+	cluster.ClusterName = "etcd-" + spec.ClusterKey
+	cluster.DataDirName = "data-" + spec.ClusterKey
+	cluster.PodName = "etcd-server-" + spec.ClusterKey
+	cluster.CPURequest = resource.MustParse("100m")
+	cluster.ClientPort = 4001
+	cluster.PeerPort = 2380
+
+	// We used to build this through text files ... it turns out to just be more complicated than code!
+	switch spec.ClusterKey {
+	case "main":
+		cluster.ClusterName = "etcd"
+		cluster.DataDirName = "data"
+		cluster.PodName = "etcd-server"
+		cluster.CPURequest = resource.MustParse("200m")
+
+	case "events":
+		cluster.ClientPort = 4002
+		cluster.PeerPort = 2381
+
+	default:
+		return nil, fmt.Errorf("unknown Etcd ClusterKey %q", spec.ClusterKey)
+
 	}
 
-	err = yaml.Unmarshal([]byte(model), cluster)
-	if err != nil {
-		return nil, fmt.Errorf("error parsing etcd model template %q: %v", modelTemplatePath, err)
-	}
 	k.cluster = cluster
 
 	return k, nil
@@ -124,10 +144,6 @@ func (c *EtcdCluster) configure(k *KubeBoot) error {
 		c.PodName = c.ClusterName
 	}
 
-	if c.CPURequest == "" {
-		c.CPURequest = "100m"
-	}
-
 	err := touchFile(PathFor(c.LogFile))
 	if err != nil {
 		return fmt.Errorf("error touching log-file %q: %v", c.LogFile, err)
@@ -163,14 +179,10 @@ func (c *EtcdCluster) configure(k *KubeBoot) error {
 		return fmt.Errorf("my node name %s not found in cluster %v", c.Spec.NodeName, strings.Join(c.Spec.NodeNames, ","))
 	}
 
-	manifestTemplatePath := "templates/etcd/manifest.template"
-	manifestTemplate, err := ioutil.ReadFile(manifestTemplatePath)
+	pod := BuildEtcdManifest(c)
+	manifest, err := ToVersionedYaml(pod)
 	if err != nil {
-		return fmt.Errorf("error reading etcd manifest template %q: %v", manifestTemplatePath, err)
-	}
-	manifest, err := ExecuteTemplate("etcd-manifest", string(manifestTemplate), c)
-	if err != nil {
-		return fmt.Errorf("error executing etcd manifest template: %v", err)
+		return fmt.Errorf("error marshalling pod to yaml: %v", err)
 	}
 
 	// Time to write the manifest!

protokube/pkg/protokube/etcd_manifest.go

@@ -0,0 +1,112 @@
+package protokube
+
+import (
+	"fmt"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/util/intstr"
+	"strings"
+)
+
+// BuildEtcdManifest creates the pod spec, based on the etcd cluster
+func BuildEtcdManifest(c *EtcdCluster) *v1.Pod {
+	pod := &v1.Pod{}
+	pod.APIVersion = "v1"
+	pod.Kind = "Pod"
+	pod.Name = c.PodName
+	pod.Namespace = "kube-system"
+
+	pod.Labels = map[string]string{
+		"k8s-app": c.PodName,
+	}
+
+	pod.Spec.HostNetwork = true
+
+	{
+		container := v1.Container{
+			Name:  "etcd-container",
+			Image: "gcr.io/google_containers/etcd:2.2.1",
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceCPU: c.CPURequest,
+				},
+			},
+			Command: []string{
+				"/bin/sh",
+				"-c",
+				"/usr/local/bin/etcd 1>>/var/log/etcd.log 2>&1",
+			},
+
+			// Note that we listen on 0.0.0.0, not 127.0.0.1, so we can support etcd clusters
+			Env: []v1.EnvVar{
+				{Name: "ETCD_NAME", Value: c.Me.Name},
+				{Name: "ETCD_DATA_DIR", Value: "/var/etcd/" + c.DataDirName},
+				{Name: "ETCD_LISTEN_PEER_URLS", Value: fmt.Sprintf("http://0.0.0.0:%d", c.PeerPort)},
+				{Name: "ETCD_LISTEN_CLIENT_URLS", Value: fmt.Sprintf("http://0.0.0.0:%d", c.ClientPort)},
+				{Name: "ETCD_ADVERTISE_CLIENT_URLS", Value: fmt.Sprintf("http://%s:%d", c.Me.InternalName, c.ClientPort)},
+				{Name: "ETCD_INITIAL_ADVERTISE_PEER_URLS", Value: fmt.Sprintf("http://%s:%d", c.Me.InternalName, c.PeerPort)},
+				{Name: "ETCD_INITIAL_CLUSTER_STATE", Value: "new"},
+				{Name: "ETCD_INITIAL_CLUSTER_TOKEN", Value: c.ClusterToken},
+			},
+		}
+
+		var initialCluster []string
+		for _, node := range c.Nodes {
+			// TODO: Use localhost for ourselves?  Does the cluster view have to be symmetric?
+			initialCluster = append(initialCluster, node.Name+"="+fmt.Sprintf("http://%s:%d", node.InternalName, c.PeerPort))
+		}
+		container.Env = append(container.Env, v1.EnvVar{Name: "ETCD_INITIAL_CLUSTER", Value: strings.Join(initialCluster, ",")})
+
+		container.LivenessProbe = &v1.Probe{
+			InitialDelaySeconds: 600,
+			TimeoutSeconds:      15,
+		}
+		container.LivenessProbe.HTTPGet = &v1.HTTPGetAction{
+			Host: "127.0.0.1",
+			Port: intstr.FromInt(c.ClientPort),
+			Path: "/health",
+		}
+
+		container.Ports = append(container.Ports, v1.ContainerPort{
+			Name:          "serverport",
+			ContainerPort: int32(c.PeerPort),
+			HostPort:      int32(c.PeerPort),
+		})
+		container.Ports = append(container.Ports, v1.ContainerPort{
+			Name:          "clientport",
+			ContainerPort: int32(c.ClientPort),
+			HostPort:      int32(c.ClientPort),
+		})
+
+		container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
+			Name:      "varetcdata",
+			MountPath: "/var/etcd/" + c.DataDirName,
+			ReadOnly:  false,
+		})
+		pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
+			Name: "varetcdata",
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: c.VolumeMountPath + "/var/etcd/" + c.DataDirName,
+				},
+			},
+		})
+
+		container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
+			Name:      "varlogetcd",
+			MountPath: "/var/log/etcd.log",
+			ReadOnly:  false,
+		})
+		pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
+			Name: "varlogetcd",
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: c.LogFile,
+				},
+			},
+		})
+
+		pod.Spec.Containers = append(pod.Spec.Containers, container)
+	}
+
+	return pod
+}

protokube/pkg/protokube/kube_model.go

@@ -1,41 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package protokube
-
-//
-//import (
-//	"fmt"
-//	"github.com/golang/glog"
-//)
-//
-//// ApplyModel applies the configuration as specified in the model
-//func (k *KubeBoot) ApplyModel() error {
-//	etcdClusters, err := k.BuildEtcdClusters(modelDir)
-//	if err != nil {
-//		return fmt.Errorf("error building etcd models: %v", err)
-//	}
-//
-//	for _, etcdCluster := range etcdClusters {
-//		glog.Infof("configuring etcd cluster %s", etcdCluster.ClusterName)
-//		err := etcdCluster.configure(k)
-//		if err != nil {
-//			return fmt.Errorf("error applying etcd model: %v", err)
-//		}
-//	}
-//
-//	return nil
-//}

protokube/pkg/protokube/volumes.go

@@ -61,17 +61,6 @@ func (v *VolumeInfo) String() string {
 	return DebugString(v)
 }
 
-type EtcdClusterSpec struct {
-	ClusterKey string
-
-	NodeName  string
-	NodeNames []string
-}
-
-func (e *EtcdClusterSpec) String() string {
-	return DebugString(e)
-}
-
 // Parses a tag on a volume that encodes an etcd cluster role
 // The format is "<myname>/<allnames>", e.g. "node1/node1,node2,node3"
 func ParseEtcdClusterSpec(clusterKey, v string) (*EtcdClusterSpec, error) {

protokube/templates/etcd/manifest.template

@@ -1,71 +0,0 @@
-# etcd podspec
-apiVersion: v1
-kind: Pod
-metadata:
-  name: {{ .PodName }}
-  namespace: kube-system
-  labels:
-    k8s-app: {{ .PodName }}
-spec:
-  hostNetwork: true
-  containers:
-  - name: etcd-container
-    image: gcr.io/google_containers/etcd:2.2.1
-    resources:
-      requests:
-        cpu: {{ .CPURequest }}
-    command:
-    - /bin/sh
-    - -c
-    - /usr/local/bin/etcd 1>>/var/log/etcd.log 2>&1
-    env:
-    - name: ETCD_NAME
-      value: {{ .Me.Name }}
-    - name: ETCD_DATA_DIR
-      value: /var/etcd/{{ .DataDirName}}
-    - name: ETCD_LISTEN_PEER_URLS
-      value: http://0.0.0.0:{{ .PeerPort }}
-    - name: ETCD_LISTEN_CLIENT_URLS
-      value: http://0.0.0.0:{{ .ClientPort }}
-    - name: ETCD_ADVERTISE_CLIENT_URLS
-      value: http://{{ .Me.InternalName }}:{{ .ClientPort }}
-    - name: ETCD_INITIAL_ADVERTISE_PEER_URLS
-      value: http://{{ .Me.InternalName }}:{{ .PeerPort }}
-    - name: ETCD_INITIAL_CLUSTER_STATE
-      value: new
-    - name: ETCD_INITIAL_CLUSTER_TOKEN
-      value: {{ .ClusterToken }}
-    - name: ETCD_INITIAL_CLUSTER
-      value: {{ range $index, $node := .Nodes -}}
-             {{- if $index }},{{ end -}}
-             {{ $node.Name }}=http://{{ $node.InternalName }}:{{ $.PeerPort }}
-             {{- end }}
-    livenessProbe:
-      httpGet:
-        host: 127.0.0.1
-        port: {{ .ClientPort }}
-        path: /health
-      initialDelaySeconds: 600
-      timeoutSeconds: 15
-    ports:
-    - name: serverport
-      containerPort: {{ .PeerPort }}
-      hostPort: {{ .PeerPort }}
-    - name: clientport
-      containerPort: {{ .ClientPort }}
-      hostPort: {{ .ClientPort }}
-    volumeMounts:
-    - mountPath: /var/etcd/{{ .DataDirName }}
-      name: varetcddata
-      readOnly: false
-    - mountPath: /var/log/etcd.log
-      name: varlogetcd
-      readOnly: false
-  volumes:
-  - name: varetcddata
-    hostPath:
-      path: {{ .VolumeMountPath }}/var/etcd/{{ .DataDirName }}
-  - name: varlogetcd
-    hostPath:
-      path: {{ .LogFile }}
-

protokube/tests/integration/build_etcd_manifest/integration_test.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"io/ioutil"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/diff"
+	"path"
+	"testing"
+
+	"strings"
+	"k8s.io/kops/protokube/pkg/protokube"
+
+	_ "k8s.io/kubernetes/pkg/api/install"
+	"strconv"
+)
+
+func TestBuildEtcdManifest(t *testing.T) {
+	runTest(t, "main")
+}
+
+func runTest(t *testing.T, srcDir string) {
+	sourcePath := path.Join(srcDir, "cluster.yaml")
+	sourceBytes, err := ioutil.ReadFile(sourcePath)
+	if err != nil {
+		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
+	}
+
+	expectedPath := path.Join(srcDir, "manifest.yaml")
+	expectedBytes, err := ioutil.ReadFile(expectedPath)
+	if err != nil {
+		t.Fatalf("unexpected error reading expectedPath %q: %v", expectedPath, err)
+	}
+
+	cluster := &protokube.EtcdCluster{}
+	err = kops.ParseRawYaml(sourceBytes, cluster)
+	if err != nil {
+		t.Fatalf("error parsing options yaml: %v", err)
+	}
+
+	cluster.Me = &protokube.EtcdNode{
+		Name: "node0",
+		InternalName: "node0" + ".internal",
+	}
+
+	for i := 0; i <= 2; i++ {
+		node := &protokube.EtcdNode{
+			Name: "node" + strconv.Itoa(i),
+			InternalName: "node"  + strconv.Itoa(i) + ".internal",
+		}
+		cluster.Nodes = append(cluster.Nodes, node)
+	}
+
+	pod := protokube.BuildEtcdManifest(cluster)
+	actual, err := protokube.ToVersionedYaml(pod)
+	if err != nil {
+		t.Fatalf("error marshalling to yaml: %v", err)
+	}
+
+	actualString := strings.TrimSpace(string(actual))
+	expectedString := strings.TrimSpace(string(expectedBytes))
+
+	if actualString != expectedString {
+		diffString := diff.FormatDiff(expectedString, actualString)
+		t.Logf("diff:\n%s\n", diffString)
+
+		t.Fatalf("manifest differed from expected")
+	}
+}

protokube/tests/integration/build_etcd_manifest/main/cluster.yaml

@@ -0,0 +1,9 @@
+volumeMountPath: /mnt/main
+clusterName: etcd-main
+dataDirName: data-main
+podName: etcd-server-main
+cpuRequest: "200m"
+clientPort: 4001
+peerPort: 2380
+clusterToken: token-main
+logFile: /var/log/main.log

protokube/tests/integration/build_etcd_manifest/main/manifest.yaml

@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  creationTimestamp: null
+  labels:
+    k8s-app: etcd-server-main
+  name: etcd-server-main
+  namespace: kube-system
+spec:
+  containers:
+  - command:
+    - /bin/sh
+    - -c
+    - /usr/local/bin/etcd 1>>/var/log/etcd.log 2>&1
+    env:
+    - name: ETCD_NAME
+      value: node0
+    - name: ETCD_DATA_DIR
+      value: /var/etcd/data-main
+    - name: ETCD_LISTEN_PEER_URLS
+      value: http://0.0.0.0:2380
+    - name: ETCD_LISTEN_CLIENT_URLS
+      value: http://0.0.0.0:4001
+    - name: ETCD_ADVERTISE_CLIENT_URLS
+      value: http://node0.internal:4001
+    - name: ETCD_INITIAL_ADVERTISE_PEER_URLS
+      value: http://node0.internal:2380
+    - name: ETCD_INITIAL_CLUSTER_STATE
+      value: new
+    - name: ETCD_INITIAL_CLUSTER_TOKEN
+      value: token-main
+    - name: ETCD_INITIAL_CLUSTER
+      value: node0=http://node0.internal:2380,node1=http://node1.internal:2380,node2=http://node2.internal:2380
+    image: gcr.io/google_containers/etcd:2.2.1
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /health
+        port: 4001
+      initialDelaySeconds: 600
+      timeoutSeconds: 15
+    name: etcd-container
+    ports:
+    - containerPort: 2380
+      hostPort: 2380
+      name: serverport
+    - containerPort: 4001
+      hostPort: 4001
+      name: clientport
+    resources:
+      requests:
+        cpu: 200m
+    volumeMounts:
+    - mountPath: /var/etcd/data-main
+      name: varetcdata
+    - mountPath: /var/log/etcd.log
+      name: varlogetcd
+  hostNetwork: true
+  volumes:
+  - hostPath:
+      path: /mnt/main/var/etcd/data-main
+    name: varetcdata
+  - hostPath:
+      path: /var/log/main.log
+    name: varlogetcd
+status: {}