From 5d6d0f5f5d679d3561b824dd3847d62cd6fcc8e3 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 30 Dec 2016 14:45:11 -0500 Subject: [PATCH] Load protokube from http/https This helps us treat protokube as being paired with nodeup, and is a step towards registry-less deployments (and isolated deployments) along with moving away from our deprecated gcr.io usage. --- Makefile | 36 ++++--- cmd/kops/main.go | 6 -- cmd/kops/version.go | 3 +- docs/development/adding_a_feature.md | 2 +- .../services/protokube.service.template | 4 +- upup/pkg/fi/cloudup/apply_cluster.go | 32 +++---- upup/pkg/fi/cloudup/urls.go | 94 +++++++++++++++++++ upup/pkg/fi/nodeup/command.go | 6 ++ upup/pkg/fi/nodeup/config.go | 3 + upup/pkg/fi/nodeup/template_functions.go | 42 ++++++--- version.go | 20 ++++ 11 files changed, 192 insertions(+), 56 deletions(-) create mode 100644 upup/pkg/fi/cloudup/urls.go create mode 100644 version.go diff --git a/Makefile b/Makefile index 7f6b99384b..3eb02c7b9d 100644 --- a/Makefile +++ b/Makefile @@ -30,12 +30,12 @@ MAKEDIR:=$(strip $(shell dirname "$(realpath $(lastword $(MAKEFILE_LIST)))")) # Keep in sync with upup/models/cloudup/resources/addons/dns-controller/ DNS_CONTROLLER_TAG=1.4.1 -PROTOKUBE_TAG=1.5.0 ifndef VERSION VERSION := git-$(shell git describe --always) endif + # Go exports: GO15VENDOREXPERIMENT=1 @@ -49,7 +49,7 @@ ifdef STATIC_BUILD endif kops: kops-gobindata - go install ${EXTRA_BUILDFLAGS} -ldflags "-X main.BuildVersion=${VERSION} ${EXTRA_LDFLAGS}" k8s.io/kops/cmd/kops/... + go install ${EXTRA_BUILDFLAGS} -ldflags "-X k8s.io/kops.Version=${VERSION} ${EXTRA_LDFLAGS}" k8s.io/kops/cmd/kops/... gobindata-tool: go build ${EXTRA_BUILDFLAGS} -ldflags "${EXTRA_LDFLAGS}" -o ${GOPATH_1ST}/bin/go-bindata k8s.io/kops/vendor/github.com/jteeuwen/go-bindata/go-bindata @@ -84,9 +84,9 @@ test: crossbuild-nodeup: mkdir -p .build/dist/ - #GOOS=darwin GOARCH=amd64 go build -a ${EXTRA_BUILDFLAGS} -o .build/dist/darwin/amd64/nodeup -ldflags "${EXTRA_LDFLAGS} -X main.BuildVersion=${VERSION}" k8s.io/kops/cmd/nodeup - GOOS=linux GOARCH=amd64 go build -a ${EXTRA_BUILDFLAGS} -o .build/dist/linux/amd64/nodeup -ldflags "${EXTRA_LDFLAGS} -X main.BuildVersion=${VERSION}" k8s.io/kops/cmd/nodeup - #GOOS=windows GOARCH=amd64 go build -o .build/dist/windows/amd64/kops -ldflags "-X main.BuildVersion=${VERSION}" -v k8s.io/kops/cmd/kops/... + #GOOS=darwin GOARCH=amd64 go build -a ${EXTRA_BUILDFLAGS} -o .build/dist/darwin/amd64/nodeup -ldflags "${EXTRA_LDFLAGS} -X k8s.io/kops.Version=${VERSION}" k8s.io/kops/cmd/nodeup + GOOS=linux GOARCH=amd64 go build -a ${EXTRA_BUILDFLAGS} -o .build/dist/linux/amd64/nodeup -ldflags "${EXTRA_LDFLAGS} -X k8s.io/kops.Version=${VERSION}" k8s.io/kops/cmd/nodeup + #GOOS=windows GOARCH=amd64 go build -o .build/dist/windows/amd64/kops -ldflags "-X k8s.io/kops.Version=${VERSION}" -v k8s.io/kops/cmd/kops/... crossbuild-nodeup-in-docker: docker pull golang:${GOVERSION} # Keep golang image up to date @@ -95,9 +95,9 @@ crossbuild-nodeup-in-docker: crossbuild: mkdir -p .build/dist/ - GOOS=darwin GOARCH=amd64 go build -a ${EXTRA_BUILDFLAGS} -o .build/dist/darwin/amd64/kops -ldflags "${EXTRA_LDFLAGS} -X main.BuildVersion=${VERSION}" k8s.io/kops/cmd/kops - GOOS=linux GOARCH=amd64 go build -a ${EXTRA_BUILDFLAGS} -o .build/dist/linux/amd64/kops -ldflags "${EXTRA_LDFLAGS} -X main.BuildVersion=${VERSION}" k8s.io/kops/cmd/kops - #GOOS=windows GOARCH=amd64 go build -o .build/dist/windows/amd64/kops -ldflags "-X main.BuildVersion=${VERSION}" -v k8s.io/kops/cmd/kops/... + GOOS=darwin GOARCH=amd64 go build -a ${EXTRA_BUILDFLAGS} -o .build/dist/darwin/amd64/kops -ldflags "${EXTRA_LDFLAGS} -X k8s.io/kops.Version=${VERSION}" k8s.io/kops/cmd/kops + GOOS=linux GOARCH=amd64 go build -a ${EXTRA_BUILDFLAGS} -o .build/dist/linux/amd64/kops -ldflags "${EXTRA_LDFLAGS} -X k8s.io/kops.Version=${VERSION}" k8s.io/kops/cmd/kops + #GOOS=windows GOARCH=amd64 go build -o .build/dist/windows/amd64/kops -ldflags "-X k8s.io/kops.Version=${VERSION}" -v k8s.io/kops/cmd/kops/... crossbuild-in-docker: docker pull golang:${GOVERSION} # Keep golang image up to date @@ -109,12 +109,15 @@ kops-dist: crossbuild-in-docker (sha1sum .build/dist/darwin/amd64/kops | cut -d' ' -f1) > .build/dist/darwin/amd64/kops.sha1 (sha1sum .build/dist/linux/amd64/kops | cut -d' ' -f1) > .build/dist/linux/amd64/kops.sha1 -version-dist: nodeup-dist kops-dist +version-dist: nodeup-dist kops-dist protokube-export rm -rf .build/upload mkdir -p .build/upload/kops/${VERSION}/linux/amd64/ mkdir -p .build/upload/kops/${VERSION}/darwin/amd64/ + mkdir -p .build/upload/kops/${VERSION}/images/ cp .build/dist/nodeup .build/upload/kops/${VERSION}/linux/amd64/nodeup cp .build/dist/nodeup.sha1 .build/upload/kops/${VERSION}/linux/amd64/nodeup.sha1 + cp .build/dist/images/protokube.tar.gz .build/upload/kops/${VERSION}/images/protokube.tar.gz + cp .build/dist/images/protokube.tar.gz.sha1 .build/upload/kops/${VERSION}/images/protokube.tar.gz.sha1 cp .build/dist/linux/amd64/kops .build/upload/kops/${VERSION}/linux/amd64/kops cp .build/dist/linux/amd64/kops.sha1 .build/upload/kops/${VERSION}/linux/amd64/kops.sha1 cp .build/dist/darwin/amd64/kops .build/upload/kops/${VERSION}/darwin/amd64/kops @@ -154,7 +157,6 @@ push-aws-run: push ssh -t ${TARGET} sudo SKIP_PACKAGE_UPDATE=1 /tmp/nodeup --conf=/var/cache/kubernetes-install/kube_env.yaml --v=8 - protokube-gocode: go install k8s.io/kops/protokube/cmd/protokube @@ -165,15 +167,21 @@ protokube-build-in-docker: protokube-builder-image docker run -it -e VERSION=${VERSION} -v `pwd`:/src protokube-builder /onbuild.sh protokube-image: protokube-build-in-docker - docker build -t ${DOCKER_REGISTRY}/protokube:${PROTOKUBE_TAG} -f images/protokube/Dockerfile . + docker build -t protokube:${VERSION} -f images/protokube/Dockerfile . + +protokube-export: protokube-image + mkdir -p .build/dist/images + docker save protokube:${VERSION} | gzip -c > .build/dist/images/protokube.tar.gz + (sha1sum .build/dist/images/protokube.tar.gz | cut -d' ' -f1) > .build/dist/images/protokube.tar.gz.sha1 protokube-push: protokube-image - docker push ${DOCKER_REGISTRY}/protokube:${PROTOKUBE_TAG} + docker tag protokube:${VERSION} ${DOCKER_REGISTRY}/protokube:${VERSION} + docker push ${DOCKER_REGISTRY}/protokube:${VERSION} nodeup: nodeup-dist nodeup-gocode: kops-gobindata - go install ${EXTRA_BUILDFLAGS} -ldflags "${EXTRA_LDFLAGS} -X main.BuildVersion=${VERSION}" k8s.io/kops/cmd/nodeup + go install ${EXTRA_BUILDFLAGS} -ldflags "${EXTRA_LDFLAGS} -X k8s.io/kops.Version=${VERSION}" k8s.io/kops/cmd/nodeup nodeup-dist: docker pull golang:${GOVERSION} # Keep golang image up to date @@ -250,7 +258,7 @@ ci: kops nodeup-gocode examples test govet verify-boilerplate channels: channels-gocode channels-gocode: - go install ${EXTRA_BUILDFLAGS} -ldflags "-X main.BuildVersion=${VERSION} ${EXTRA_LDFLAGS}" k8s.io/kops/channels/cmd/channels + go install ${EXTRA_BUILDFLAGS} -ldflags "-X k8s.io/kops.Version=${VERSION} ${EXTRA_LDFLAGS}" k8s.io/kops/channels/cmd/channels # -------------------------------------------------- # API / embedding examples diff --git a/cmd/kops/main.go b/cmd/kops/main.go index 41c25ee7de..e130a91dd2 100644 --- a/cmd/kops/main.go +++ b/cmd/kops/main.go @@ -18,15 +18,9 @@ package main // import "k8s.io/kops/cmd/kops" import ( "fmt" - "k8s.io/kops/upup/pkg/fi/cloudup" "os" ) -var ( - // value overwritten during build. This can be used to resolve issues. - BuildVersion = cloudup.NodeUpVersion -) - func main() { Execute() } diff --git a/cmd/kops/version.go b/cmd/kops/version.go index 15aedd47f1..8caf3185e7 100644 --- a/cmd/kops/version.go +++ b/cmd/kops/version.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/spf13/cobra" + "k8s.io/kops" ) type VersionCmd struct { @@ -46,7 +47,7 @@ func init() { } func (c *VersionCmd) Run() error { - fmt.Printf("Version %s\n", BuildVersion) + fmt.Printf("Version %s\n", kops.Version) return nil } diff --git a/docs/development/adding_a_feature.md b/docs/development/adding_a_feature.md index 0bb92994d5..6c40840d9b 100644 --- a/docs/development/adding_a_feature.md +++ b/docs/development/adding_a_feature.md @@ -198,7 +198,7 @@ and then push nodeup using: export S3_BUCKET_NAME= make upload S3_BUCKET=s3://${S3_BUCKET_NAME} VERSION=dev -export NODEUP_URL=https://${S3_BUCKET_NAME}.s3.amazonaws.com/kops/dev/linux/amd64/nodeup +export KOPS_URL=https://${S3_BUCKET_NAME}.s3.amazonaws.com/kops/dev/ kops create cluster --zones us-east-1b ... diff --git a/upup/models/nodeup/_protokube/services/protokube.service.template b/upup/models/nodeup/_protokube/services/protokube.service.template index 555817fa78..684f4541ed 100644 --- a/upup/models/nodeup/_protokube/services/protokube.service.template +++ b/upup/models/nodeup/_protokube/services/protokube.service.template @@ -5,8 +5,8 @@ After=docker.service [Service] EnvironmentFile=/etc/sysconfig/protokube -ExecStartPre=/usr/bin/docker pull {{ ProtokubeImage }} -ExecStart=/usr/bin/docker run -v /:/rootfs/ -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd --net=host --privileged {{ ProtokubeImage }} /usr/bin/protokube "$DAEMON_ARGS" +ExecStartPre={{ ProtokubeImagePullCommand }} +ExecStart=/usr/bin/docker run -v /:/rootfs/ -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd --net=host --privileged {{ ProtokubeImageName }} /usr/bin/protokube "$DAEMON_ARGS" Restart=always RestartSec=2s StartLimitInterval=0 diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 57b48970c5..b10177a6f4 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -23,6 +23,7 @@ import ( "time" "github.com/golang/glog" + "k8s.io/kops" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/client/simple" @@ -40,10 +41,6 @@ import ( k8sapi "k8s.io/kubernetes/pkg/api" ) -const ( - NodeUpVersion = "1.4.2" -) - const DefaultMaxTaskDuration = 10 * time.Minute var CloudupModels = []string{"config", "proto", "cloudup"} @@ -199,14 +196,7 @@ func (c *ApplyClusterCmd) Run() error { } if c.NodeUpSource == "" { - location := os.Getenv("NODEUP_URL") - if location == "" { - location = "https://kubeupv2.s3.amazonaws.com/kops/" + NodeUpVersion + "/linux/amd64/nodeup" - glog.V(2).Infof("Using default nodeup location: %q", location) - } else { - glog.Warningf("Using nodeup location from NODEUP_URL env var: %q", location) - } - c.NodeUpSource = location + c.NodeUpSource = NodeUpLocation() } checkExisting := true @@ -439,20 +429,22 @@ func (c *ApplyClusterCmd) Run() error { } } - config.Images = images - { - protokubeImage := os.Getenv("PROTOKUBE_IMAGE") - if protokubeImage != "" { - glog.Warningf("Using protokube image specified in PROTOKUBE_IMAGE env var: %q", protokubeImage) - } else { - protokubeImage = nodeup.DefaultProtokubeImage + location := ProtokubeImageSource() + + hash, err := findHash(location) + if err != nil { + return nil, err } + config.ProtokubeImage = &nodeup.Image{ - Source: protokubeImage, + Name: ProtokubeImageName(), + Source: location, + Hash: hash.Hex(), } } + config.Images = images config.Channels = channels return config, nil diff --git a/upup/pkg/fi/cloudup/urls.go b/upup/pkg/fi/cloudup/urls.go new file mode 100644 index 0000000000..ffb70351f8 --- /dev/null +++ b/upup/pkg/fi/cloudup/urls.go @@ -0,0 +1,94 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudup + +import ( + "github.com/golang/glog" + "k8s.io/kops" + "os" + "strings" +) + +// ProtokubeImageName is the name of the protokube image, as referenced by docker +func ProtokubeImageName() string { + return "protokube:" + kops.Version +} + +// baseUrl caches the BaseUrl value +var baseUrl string + +// BaseUrl returns the base url for the distribution of kops - in particular for nodeup & docker images +func BaseUrl() string { + if baseUrl != "" { + // Avoid repeated logging + return baseUrl + } + + baseUrl = os.Getenv("KOPS_URL") + if baseUrl == "" { + baseUrl = "https://kubeupv2.s3.amazonaws.com/kops/" + kops.Version + "/" + glog.V(4).Infof("Using default base url: %q", baseUrl) + } else { + glog.Warningf("Using base url from KOPS_BASE_URL env var: %q", baseUrl) + } + + if !strings.HasSuffix(baseUrl, "/") { + baseUrl += "/" + } + + return baseUrl +} + +// nodeUpLocation caches the NodeUpLocation value +var nodeUpLocation string + +// NodeUpLocation returns the URL where nodeup should be downloaded +func NodeUpLocation() string { + if nodeUpLocation != "" { + // Avoid repeated logging + return nodeUpLocation + } + nodeUpLocation = os.Getenv("NODEUP_URL") + if nodeUpLocation == "" { + nodeUpLocation = BaseUrl() + "linux/amd64/nodeup" + glog.V(4).Infof("Using default nodeup location: %q", nodeUpLocation) + } else { + glog.Warningf("Using nodeup location from NODEUP_URL env var: %q", nodeUpLocation) + } + return nodeUpLocation +} + +// protokubeImageSource caches the ProtokubeImageSource value +var protokubeImageSource string + +// ProtokubeImageSource returns the source for the docker image for protokube. +// Either a docker name (e.g. gcr.io/protokube:1.4), or a URL (https://...) in which case we download +// the contents of the url and docker load it +func ProtokubeImageSource() string { + if protokubeImageSource != "" { + // Avoid repeated logging + return protokubeImageSource + } + protokubeImageSource = os.Getenv("PROTOKUBE_IMAGE") + if protokubeImageSource == "" { + protokubeImageSource = BaseUrl() + "images/protokube.tar.gz" + glog.V(4).Infof("Using default protokube location: %q", protokubeImageSource) + } else { + glog.Warningf("Using protokube location from PROTOKUBE_IMAGE env var: %q", protokubeImageSource) + } + return protokubeImageSource +} diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 2e6e6297ac..0dbef7b905 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -213,6 +213,12 @@ func (c *NodeUpCommand) Run(out io.Writer) error { Hash: image.Hash, } } + if c.config.ProtokubeImage != nil { + taskMap["LoadImage.protokube"] = &nodetasks.LoadImageTask{ + Source: c.config.ProtokubeImage.Source, + Hash: c.config.ProtokubeImage.Hash, + } + } var cloud fi.Cloud var keyStore fi.Keystore diff --git a/upup/pkg/fi/nodeup/config.go b/upup/pkg/fi/nodeup/config.go index b625b5a74a..76d20b7f33 100644 --- a/upup/pkg/fi/nodeup/config.go +++ b/upup/pkg/fi/nodeup/config.go @@ -54,6 +54,9 @@ type NodeUpConfig struct { // Image is a docker image we should pre-load type Image struct { + // Name is the name of the tagged image + Name string `json:"name,omitempty"` + // Source is the URL from which we should download the image Source string `json:"source,omitempty"` diff --git a/upup/pkg/fi/nodeup/template_functions.go b/upup/pkg/fi/nodeup/template_functions.go index 73daf0b85b..bc8c06c7b8 100644 --- a/upup/pkg/fi/nodeup/template_functions.go +++ b/upup/pkg/fi/nodeup/template_functions.go @@ -19,21 +19,21 @@ package nodeup import ( "encoding/base64" "fmt" + "runtime" + "strings" "text/template" "github.com/golang/glog" + "k8s.io/kops" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/secrets" "k8s.io/kops/util/pkg/vfs" "k8s.io/kubernetes/pkg/util/sets" - "runtime" ) const TagMaster = "_kubernetes_master" -const DefaultProtokubeImage = "b.gcr.io/kops-images/protokube:1.5.0" - // templateFunctions is a simple helper-class for the functions accessible to templates type templateFunctions struct { nodeupConfig *NodeUpConfig @@ -155,7 +155,8 @@ func (t *templateFunctions) populate(dest template.FuncMap) { return t.cluster.ObjectMeta.Name } - dest["ProtokubeImage"] = t.ProtokubeImage + dest["ProtokubeImageName"] = t.ProtokubeImageName + dest["ProtokubeImagePullCommand"] = t.ProtokubeImagePullCommand dest["ProtokubeFlags"] = t.ProtokubeFlags } @@ -235,17 +236,34 @@ func (t *templateFunctions) GetToken(key string) (string, error) { return string(token.Data), nil } -// ProtokubeImage returns the docker image for protokube -func (t *templateFunctions) ProtokubeImage() string { - image := "" - if t.nodeupConfig.ProtokubeImage != nil { - image = t.nodeupConfig.ProtokubeImage.Source +// ProtokubeImageName returns the docker image for protokube +func (t *templateFunctions) ProtokubeImageName() string { + name := "" + if t.nodeupConfig.ProtokubeImage != nil && t.nodeupConfig.ProtokubeImage.Name != "" { + name = t.nodeupConfig.ProtokubeImage.Name } - if image == "" { + if name == "" { // use current default corresponding to this version of nodeup - image = DefaultProtokubeImage + name = kops.DefaultProtokubeImageName() } - return image + return name +} + +// ProtokubeImagePullCommand returns the command to pull the image +func (t *templateFunctions) ProtokubeImagePullCommand() string { + source := "" + if t.nodeupConfig.ProtokubeImage != nil { + source = t.nodeupConfig.ProtokubeImage.Source + } + if source == "" { + // Nothing to pull; return dummy value + return "/bin/true" + } + if strings.HasPrefix(source, "http:") || strings.HasPrefix(source, "https:") || strings.HasPrefix(source, "s3:") { + // We preloaded the image; return a dummy value + return "/bin/true" + } + return "/usr/bin/docker pull " + t.nodeupConfig.ProtokubeImage.Source } // ProtokubeFlags returns the flags object for protokube diff --git a/version.go b/version.go new file mode 100644 index 0000000000..3495ad6eb1 --- /dev/null +++ b/version.go @@ -0,0 +1,20 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kops + +// This should be replaced by the makefile +var Version = "dev"