From 7e3221a2fb445c8733411d5e4276c0a16210540e Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Wed, 27 Jun 2018 22:23:33 +0200 Subject: [PATCH 1/9] Add pull-through proxy cache for asset docker images --- pkg/apis/kops/cluster.go | 2 ++ pkg/assets/builder.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 0e60555614..e4471797d1 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -192,6 +192,8 @@ type Assets struct { ContainerRegistry *string `json:"containerRegistry,omitempty"` // FileRepository is the url for a private file serving repository FileRepository *string `json:"fileRepository,omitempty"` + // ContainerProxy is a url for a pull-through proxy of a docker registry + ContainerProxy *string `json:"containerRegistry,omitempty"` } // IAMSpec adds control over the IAM security policies applied to resources diff --git a/pkg/assets/builder.go b/pkg/assets/builder.go index 79723220f4..a8f1134d05 100644 --- a/pkg/assets/builder.go +++ b/pkg/assets/builder.go @@ -34,6 +34,7 @@ import ( "k8s.io/kops/pkg/values" "k8s.io/kops/util/pkg/hashing" "k8s.io/kops/util/pkg/vfs" + "regexp" ) // RewriteManifests controls whether we rewrite manifests @@ -146,6 +147,20 @@ func (a *AssetBuilder) RemapImage(image string) (string, error) { } } + if a.AssetsLocation != nil && a.AssetsLocation.ContainerProxy != nil { + containerProxy := *a.AssetsLocation.ContainerProxy + normalized := image + + // If the image name contains only a single / we can assume ot is an image pulled from the docker hub. eg. "weaveworks/weave-kube" + // In these cases it should be sufficient to just prepend the proxy url, producing eg docker-proxy/weaveworks/weave-kube + if strings.Count(normalized, "/") == 1 { + normalized = containerProxy + "/" + normalized + } else { + var re = regexp.MustCompile(`^[^/]+`) + normalized = re.ReplaceAllString(normalized, containerProxy) + } + } + if a.AssetsLocation != nil && a.AssetsLocation.ContainerRegistry != nil { registryMirror := *a.AssetsLocation.ContainerRegistry normalized := image From e8607b8c7e51cd92c83d082b332b81810575c5a0 Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Sat, 30 Jun 2018 18:18:14 +0200 Subject: [PATCH 2/9] Fix JSON field name in cluster asset struct --- pkg/apis/kops/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index e4471797d1..2b157eb61e 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -193,7 +193,7 @@ type Assets struct { // FileRepository is the url for a private file serving repository FileRepository *string `json:"fileRepository,omitempty"` // ContainerProxy is a url for a pull-through proxy of a docker registry - ContainerProxy *string `json:"containerRegistry,omitempty"` + ContainerProxy *string `json:"containerProxy,omitempty"` } // IAMSpec adds control over the IAM security policies applied to resources From debbbcfb4f05cd7231fc0760eedb42c40e766a19 Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Sat, 30 Jun 2018 18:18:39 +0200 Subject: [PATCH 3/9] Include containerProxy in kops api deep copying --- pkg/apis/kops/v1alpha1/cluster.go | 2 ++ pkg/apis/kops/v1alpha1/zz_generated.conversion.go | 2 ++ pkg/apis/kops/v1alpha2/cluster.go | 2 ++ pkg/apis/kops/v1alpha2/zz_generated.conversion.go | 2 ++ pkg/apis/kops/zz_generated.deepcopy.go | 9 +++++++++ 5 files changed, 17 insertions(+) diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index e175fcea3a..b889bb5bdb 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -191,6 +191,8 @@ type Assets struct { ContainerRegistry *string `json:"containerRegistry,omitempty"` // FileRepository is the url for a private file serving repository FileRepository *string `json:"fileRepository,omitempty"` + // ContainerProxy is a url for a pull-through proxy of a docker registry + ContainerProxy *string `json:"containerProxy,omitempty"` } // IAMSpec adds control over the IAM security policies applied to resources diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 7b042addf8..b492edc176 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -273,6 +273,7 @@ func Convert_kops_AmazonVPCNetworkingSpec_To_v1alpha1_AmazonVPCNetworkingSpec(in func autoConvert_v1alpha1_Assets_To_kops_Assets(in *Assets, out *kops.Assets, s conversion.Scope) error { out.ContainerRegistry = in.ContainerRegistry out.FileRepository = in.FileRepository + out.ContainerProxy = in.ContainerProxy return nil } @@ -284,6 +285,7 @@ func Convert_v1alpha1_Assets_To_kops_Assets(in *Assets, out *kops.Assets, s conv func autoConvert_kops_Assets_To_v1alpha1_Assets(in *kops.Assets, out *Assets, s conversion.Scope) error { out.ContainerRegistry = in.ContainerRegistry out.FileRepository = in.FileRepository + out.ContainerProxy = in.ContainerProxy return nil } diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 2bbc775c36..4ea038c094 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -192,6 +192,8 @@ type Assets struct { ContainerRegistry *string `json:"containerRegistry,omitempty"` // FileRepository is the url for a private file serving repository FileRepository *string `json:"fileRepository,omitempty"` + // ContainerProxy is a url for a pull-through proxy of a docker registry + ContainerProxy *string `json:"containerProxy,omitempty"` } // IAMSpec adds control over the IAM security policies applied to resources diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 31f469c6f8..e1328060c5 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -287,6 +287,7 @@ func Convert_kops_AmazonVPCNetworkingSpec_To_v1alpha2_AmazonVPCNetworkingSpec(in func autoConvert_v1alpha2_Assets_To_kops_Assets(in *Assets, out *kops.Assets, s conversion.Scope) error { out.ContainerRegistry = in.ContainerRegistry out.FileRepository = in.FileRepository + out.ContainerProxy = in.ContainerProxy return nil } @@ -298,6 +299,7 @@ func Convert_v1alpha2_Assets_To_kops_Assets(in *Assets, out *kops.Assets, s conv func autoConvert_kops_Assets_To_v1alpha2_Assets(in *kops.Assets, out *Assets, s conversion.Scope) error { out.ContainerRegistry = in.ContainerRegistry out.FileRepository = in.FileRepository + out.ContainerProxy = in.ContainerProxy return nil } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 61e176a2b7..20f85eba80 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -144,6 +144,15 @@ func (in *Assets) DeepCopyInto(out *Assets) { **out = **in } } + if in.ContainerProxy != nil { + in, out := &in.ContainerProxy, &out.ContainerProxy + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } return } From 4d87fb6a746ce63514ba8f7d765de83085b7471c Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Sat, 30 Jun 2018 18:19:08 +0200 Subject: [PATCH 4/9] Destinguish between docker hub and convenience registry domain --- pkg/assets/builder.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/assets/builder.go b/pkg/assets/builder.go index a8f1134d05..6f93f61287 100644 --- a/pkg/assets/builder.go +++ b/pkg/assets/builder.go @@ -151,9 +151,9 @@ func (a *AssetBuilder) RemapImage(image string) (string, error) { containerProxy := *a.AssetsLocation.ContainerProxy normalized := image - // If the image name contains only a single / we can assume ot is an image pulled from the docker hub. eg. "weaveworks/weave-kube" - // In these cases it should be sufficient to just prepend the proxy url, producing eg docker-proxy/weaveworks/weave-kube - if strings.Count(normalized, "/") == 1 { + // If the image name contains only a single / we need to determine if the image is located on docker-hub or if it's using a convenient URL like k8s.gcr.io/ + // In case of a hub image it should be sufficient to just prepend the proxy url, producing eg docker-proxy.example.com/weaveworks/weave-kube + if strings.Count(normalized, "/") == 1 && !strings.ContainsAny(strings.Split(normalized, "/")[0], ".:"){ normalized = containerProxy + "/" + normalized } else { var re = regexp.MustCompile(`^[^/]+`) From cbcd7d43c04296e705e3408b1bf69047a6fd20ba Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Fri, 6 Jul 2018 10:58:27 +0200 Subject: [PATCH 5/9] Run gofmt --- pkg/assets/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/assets/builder.go b/pkg/assets/builder.go index 6f93f61287..02e8cbd5b5 100644 --- a/pkg/assets/builder.go +++ b/pkg/assets/builder.go @@ -22,6 +22,7 @@ import ( "net/url" "os" "path" + "regexp" "strings" "github.com/blang/semver" @@ -34,7 +35,6 @@ import ( "k8s.io/kops/pkg/values" "k8s.io/kops/util/pkg/hashing" "k8s.io/kops/util/pkg/vfs" - "regexp" ) // RewriteManifests controls whether we rewrite manifests @@ -153,7 +153,7 @@ func (a *AssetBuilder) RemapImage(image string) (string, error) { // If the image name contains only a single / we need to determine if the image is located on docker-hub or if it's using a convenient URL like k8s.gcr.io/ // In case of a hub image it should be sufficient to just prepend the proxy url, producing eg docker-proxy.example.com/weaveworks/weave-kube - if strings.Count(normalized, "/") == 1 && !strings.ContainsAny(strings.Split(normalized, "/")[0], ".:"){ + if strings.Count(normalized, "/") == 1 && !strings.ContainsAny(strings.Split(normalized, "/")[0], ".:") { normalized = containerProxy + "/" + normalized } else { var re = regexp.MustCompile(`^[^/]+`) From 1264ef8fcec11b896ced5cbf06b3fb23c6d51148 Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Fri, 6 Jul 2018 11:28:31 +0200 Subject: [PATCH 6/9] Add cluster spec validation Since containerRegistry and containerProxy follow mutually exclusive concepts, we need to ensure that they are not mistakenly used together. --- pkg/apis/kops/validation/legacy.go | 4 ++++ upup/pkg/fi/cloudup/validation_test.go | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 7339c39211..1f95353521 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -71,6 +71,10 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { } } + if c.Spec.Assets != nil && c.Spec.Assets.ContainerProxy != nil && c.Spec.Assets.ContainerRegistry != nil { + return field.Forbidden(fieldSpec.Child("Assets", "ContainerProxy"), "ContainerProxy cannot be used in conjunction with ContainerRegistry as represent mutually exclusive concepts. Please consult the documentation for details.") + } + if c.Spec.CloudProvider == "" { return field.Required(fieldSpec.Child("CloudProvider"), "") } diff --git a/upup/pkg/fi/cloudup/validation_test.go b/upup/pkg/fi/cloudup/validation_test.go index 0435c1fc55..d71497669c 100644 --- a/upup/pkg/fi/cloudup/validation_test.go +++ b/upup/pkg/fi/cloudup/validation_test.go @@ -174,6 +174,27 @@ func TestValidate_ClusterName_Import(t *testing.T) { expectNoErrorFromValidate(t, c) } +func TestValidate_ContainerRegistry_and_ContainerProxy_exclusivity(t *testing.T) { + c := buildDefaultCluster(t) + + assets := new(api.Assets) + c.Spec.Assets = assets + + expectNoErrorFromValidate(t, c) + + registry := "https://registry.example.com/" + c.Spec.Assets.ContainerRegistry = ®istry + expectNoErrorFromValidate(t, c) + + proxy := "https://proxy.example.com/" + c.Spec.Assets.ContainerProxy = &proxy + expectErrorFromValidate(t, c, "ContainerProxy cannot be used in conjunction with ContainerRegistry") + + c.Spec.Assets.ContainerRegistry = nil + expectNoErrorFromValidate(t, c) + +} + func expectErrorFromValidate(t *testing.T, c *api.Cluster, message string) { err := validation.ValidateCluster(c, false) if err == nil { From 97c12113381485d29659580e8b2abc132704450e Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Fri, 6 Jul 2018 13:40:21 +0200 Subject: [PATCH 7/9] Finish builder implementation and add tests --- pkg/assets/builder.go | 10 ++- pkg/assets/builder_test.go | 132 +++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 pkg/assets/builder_test.go diff --git a/pkg/assets/builder.go b/pkg/assets/builder.go index 02e8cbd5b5..612ea8a6de 100644 --- a/pkg/assets/builder.go +++ b/pkg/assets/builder.go @@ -148,17 +148,23 @@ func (a *AssetBuilder) RemapImage(image string) (string, error) { } if a.AssetsLocation != nil && a.AssetsLocation.ContainerProxy != nil { - containerProxy := *a.AssetsLocation.ContainerProxy + containerProxy := strings.TrimRight(*a.AssetsLocation.ContainerProxy, "/") normalized := image // If the image name contains only a single / we need to determine if the image is located on docker-hub or if it's using a convenient URL like k8s.gcr.io/ // In case of a hub image it should be sufficient to just prepend the proxy url, producing eg docker-proxy.example.com/weaveworks/weave-kube - if strings.Count(normalized, "/") == 1 && !strings.ContainsAny(strings.Split(normalized, "/")[0], ".:") { + if strings.Count(normalized, "/") <= 1 && !strings.ContainsAny(strings.Split(normalized, "/")[0], ".:") { normalized = containerProxy + "/" + normalized } else { var re = regexp.MustCompile(`^[^/]+`) normalized = re.ReplaceAllString(normalized, containerProxy) } + + asset.DockerImage = normalized + asset.CanonicalLocation = image + + // Run the new image + image = asset.DockerImage } if a.AssetsLocation != nil && a.AssetsLocation.ContainerRegistry != nil { diff --git a/pkg/assets/builder_test.go b/pkg/assets/builder_test.go new file mode 100644 index 0000000000..0137f74716 --- /dev/null +++ b/pkg/assets/builder_test.go @@ -0,0 +1,132 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package assets + +import ( + "testing" + + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/util" +) + +func buildAssetBuilder(t *testing.T) *AssetBuilder { + + builder := &AssetBuilder{ + AssetsLocation: &kops.Assets{}, + ContainerAssets: []*ContainerAsset{}, + } + return builder +} + +func TestValidate_RemapImage_ContainerProxy_AppliesToDockerHub(t *testing.T) { + builder := buildAssetBuilder(t) + + proxyURL := "proxy.example.com/" + image := "weaveworks/weave-kube" + expected := "proxy.example.com/weaveworks/weave-kube" + + builder.AssetsLocation.ContainerProxy = &proxyURL + + remapped, err := builder.RemapImage(image) + if err != nil { + t.Error("Error remapping image", err) + } + + if remapped != expected { + t.Errorf("Error remapping image (Expecting: %s, got %s)", expected, remapped) + } +} + +func TestValidate_RemapImage_ContainerProxy_AppliesToSimplifiedDockerHub(t *testing.T) { + builder := buildAssetBuilder(t) + + proxyURL := "proxy.example.com/" + image := "debian" + expected := "proxy.example.com/debian" + + builder.AssetsLocation.ContainerProxy = &proxyURL + + remapped, err := builder.RemapImage(image) + if err != nil { + t.Error("Error remapping image", err) + } + + if remapped != expected { + t.Errorf("Error remapping image (Expecting: %s, got %s)", expected, remapped) + } +} + +func TestValidate_RemapImage_ContainerProxy_AppliesToSimplifiedKubernetesURL(t *testing.T) { + builder := buildAssetBuilder(t) + + proxyURL := "proxy.example.com/" + image := "k8s.gcr.io/kube-apiserver" + expected := "proxy.example.com/kube-apiserver" + version, _ := util.ParseKubernetesVersion("1.10") + + builder.KubernetesVersion = *version + builder.AssetsLocation.ContainerProxy = &proxyURL + + remapped, err := builder.RemapImage(image) + if err != nil { + t.Error("Error remapping image", err) + } + + if remapped != expected { + t.Errorf("Error remapping image (Expecting: %s, got %s)", expected, remapped) + } +} + +func TestValidate_RemapImage_ContainerProxy_AppliesToLegacyKubernetesURL(t *testing.T) { + builder := buildAssetBuilder(t) + + proxyURL := "proxy.example.com/" + image := "gcr.io/google_containers/kube-apiserver" + expected := "proxy.example.com/google_containers/kube-apiserver" + + builder.AssetsLocation.ContainerProxy = &proxyURL + + remapped, err := builder.RemapImage(image) + if err != nil { + t.Error("Error remapping image", err) + } + + if remapped != expected { + t.Errorf("Error remapping image (Expecting: %s, got %s)", expected, remapped) + } +} + +func TestValidate_RemapImage_ContainerProxy_AppliesToImagesWithTags(t *testing.T) { + builder := buildAssetBuilder(t) + + proxyURL := "proxy.example.com/" + image := "k8s.gcr.io/kube-apiserver:1.2.3" + expected := "proxy.example.com/kube-apiserver:1.2.3" + version, _ := util.ParseKubernetesVersion("1.10") + + builder.KubernetesVersion = *version + builder.AssetsLocation.ContainerProxy = &proxyURL + + remapped, err := builder.RemapImage(image) + if err != nil { + t.Error("Error remapping image", err) + } + + if remapped != expected { + t.Errorf("Error remapping image (Expecting: %s, got %s)", expected, remapped) + } +} From f422e660eb6523a069c7822b98c1aa3f4c14e695 Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Fri, 6 Jul 2018 14:53:05 +0200 Subject: [PATCH 8/9] Add documentation for containerRegisty and containerProxy spec --- docs/cluster_spec.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 24e3386d52..a46f9e6191 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -539,3 +539,37 @@ spec: providerExtraConfig: alias: foo ``` + +### assets + +Assets define alernative locations from where to retrieve static files and containers + +#### containerRegistry + +The container registry enables kops / kubernets to pull containers from a managed registry. +This is useful when pulling containers from the internet is not an option, eg. because the +deployment is offline / internet restricted or because of special requirements that apply +for deployed artifacts, eg. auditing of containers. + +For a use case example, see [How to use kops in AWS China Region](https://github.com/kubernetes/kops/blob/master/docs/aws-china.md) + +```yaml +spec: + assets: + containerRegistry: example.com/registry +``` + + +#### containerProxy + +The container proxy is designed to acts as a [pull through cache](https://docs.docker.com/registry/recipes/mirror/) for docker container assets. +Basically, what it does is it remaps the Kubernets image URL to point to you cache so that the docker daemon will pull the image from that location. +If, for example, the containerProxy is set to `proxy.example.com`, the image `k8s.gcr.io/kube-apiserver` will be pulled from `proxy.example.com/kube-apiserver` instead. +Note that the proxy you use has to support this feature for private registries. + + +```yaml +spec: + assets: + containerProxy: proxy.example.com +``` From d112b230cd422243db15b578a8b56885887420ff Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Fri, 6 Jul 2018 15:00:25 +0200 Subject: [PATCH 9/9] Update bazel files --- pkg/assets/BUILD.bazel | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/assets/BUILD.bazel b/pkg/assets/BUILD.bazel index 94b05e365a..9a480e4f2a 100644 --- a/pkg/assets/BUILD.bazel +++ b/pkg/assets/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -17,3 +17,13 @@ go_library( "//vendor/github.com/golang/glog:go_default_library", ], ) + +go_test( + name = "go_default_test", + srcs = ["builder_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/apis/kops:go_default_library", + "//pkg/apis/kops/util:go_default_library", + ], +)