Merge pull request #5390 from kampka/add-container-proxy

Add pull-through proxy cache for asset docker images

docs/cluster_spec.md

@@ -555,3 +555,37 @@ spec:
       providerExtraConfig:
         alias: foo
 ```
+
+### assets
+
+Assets define alernative locations from where to retrieve static files and containers
+
+#### containerRegistry
+
+The container registry enables kops / kubernets to pull containers from a managed registry.
+This is useful when pulling containers from the internet is not an option, eg. because the
+deployment is offline / internet restricted or because of special requirements that apply
+for deployed artifacts, eg. auditing of containers.
+
+For a use case example, see [How to use kops in AWS China Region](https://github.com/kubernetes/kops/blob/master/docs/aws-china.md)
+
+```yaml
+spec:
+  assets:
+    containerRegistry: example.com/registry
+```
+
+
+#### containerProxy
+
+The container proxy is designed to acts as a [pull through cache](https://docs.docker.com/registry/recipes/mirror/) for docker container assets.
+Basically, what it does is it remaps the Kubernets image URL to point to you cache so that the docker daemon will pull the image from that location.
+If, for example, the containerProxy is set to `proxy.example.com`, the image `k8s.gcr.io/kube-apiserver` will be pulled from `proxy.example.com/kube-apiserver` instead.
+Note that the proxy you use has to support this feature for private registries.
+
+
+```yaml
+spec:
+  assets:
+    containerProxy: proxy.example.com
+```

pkg/apis/kops/cluster.go

@@ -218,6 +218,8 @@ type Assets struct {
 	ContainerRegistry *string `json:"containerRegistry,omitempty"`
 	// FileRepository is the url for a private file serving repository
 	FileRepository *string `json:"fileRepository,omitempty"`
+	// ContainerProxy is a url for a pull-through proxy of a docker registry
+	ContainerProxy *string `json:"containerProxy,omitempty"`
 }
 
 // IAMSpec adds control over the IAM security policies applied to resources

pkg/apis/kops/v1alpha1/cluster.go

@@ -217,6 +217,8 @@ type Assets struct {
 	ContainerRegistry *string `json:"containerRegistry,omitempty"`
 	// FileRepository is the url for a private file serving repository
 	FileRepository *string `json:"fileRepository,omitempty"`
+	// ContainerProxy is a url for a pull-through proxy of a docker registry
+	ContainerProxy *string `json:"containerProxy,omitempty"`
 }
 
 // IAMSpec adds control over the IAM security policies applied to resources

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -276,6 +276,7 @@ func Convert_kops_AmazonVPCNetworkingSpec_To_v1alpha1_AmazonVPCNetworkingSpec(in
 func autoConvert_v1alpha1_Assets_To_kops_Assets(in *Assets, out *kops.Assets, s conversion.Scope) error {
 	out.ContainerRegistry = in.ContainerRegistry
 	out.FileRepository = in.FileRepository
+	out.ContainerProxy = in.ContainerProxy
 	return nil
 }
 
@@ -287,6 +288,7 @@ func Convert_v1alpha1_Assets_To_kops_Assets(in *Assets, out *kops.Assets, s conv
 func autoConvert_kops_Assets_To_v1alpha1_Assets(in *kops.Assets, out *Assets, s conversion.Scope) error {
 	out.ContainerRegistry = in.ContainerRegistry
 	out.FileRepository = in.FileRepository
+	out.ContainerProxy = in.ContainerProxy
 	return nil
 }
 

pkg/apis/kops/v1alpha2/cluster.go

@@ -218,6 +218,8 @@ type Assets struct {
 	ContainerRegistry *string `json:"containerRegistry,omitempty"`
 	// FileRepository is the url for a private file serving repository
 	FileRepository *string `json:"fileRepository,omitempty"`
+	// ContainerProxy is a url for a pull-through proxy of a docker registry
+	ContainerProxy *string `json:"containerProxy,omitempty"`
 }
 
 // IAMSpec adds control over the IAM security policies applied to resources

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -290,6 +290,7 @@ func Convert_kops_AmazonVPCNetworkingSpec_To_v1alpha2_AmazonVPCNetworkingSpec(in
 func autoConvert_v1alpha2_Assets_To_kops_Assets(in *Assets, out *kops.Assets, s conversion.Scope) error {
 	out.ContainerRegistry = in.ContainerRegistry
 	out.FileRepository = in.FileRepository
+	out.ContainerProxy = in.ContainerProxy
 	return nil
 }
 
@@ -301,6 +302,7 @@ func Convert_v1alpha2_Assets_To_kops_Assets(in *Assets, out *kops.Assets, s conv
 func autoConvert_kops_Assets_To_v1alpha2_Assets(in *kops.Assets, out *Assets, s conversion.Scope) error {
 	out.ContainerRegistry = in.ContainerRegistry
 	out.FileRepository = in.FileRepository
+	out.ContainerProxy = in.ContainerProxy
 	return nil
 }
 

pkg/apis/kops/validation/legacy.go

@@ -72,6 +72,10 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error {
 		}
 	}
 
+	if c.Spec.Assets != nil && c.Spec.Assets.ContainerProxy != nil && c.Spec.Assets.ContainerRegistry != nil {
+		return field.Forbidden(fieldSpec.Child("Assets", "ContainerProxy"), "ContainerProxy cannot be used in conjunction with ContainerRegistry as represent mutually exclusive concepts. Please consult the documentation for details.")
+	}
+
 	if c.Spec.CloudProvider == "" {
 		return field.Required(fieldSpec.Child("CloudProvider"), "")
 	}

pkg/apis/kops/zz_generated.deepcopy.go

@@ -144,6 +144,15 @@ func (in *Assets) DeepCopyInto(out *Assets) {
 			**out = **in
 		}
 	}
+	if in.ContainerProxy != nil {
+		in, out := &in.ContainerProxy, &out.ContainerProxy
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(string)
+			**out = **in
+		}
+	}
 	return
 }
 

pkg/assets/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -17,3 +17,13 @@ go_library(
         "//vendor/github.com/golang/glog:go_default_library",
     ],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["builder_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/apis/kops:go_default_library",
+        "//pkg/apis/kops/util:go_default_library",
+    ],
+)

pkg/assets/builder.go

@@ -22,6 +22,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"regexp"
 	"strings"
 
 	"github.com/blang/semver"
@@ -146,6 +147,26 @@ func (a *AssetBuilder) RemapImage(image string) (string, error) {
 		}
 	}
 
+	if a.AssetsLocation != nil && a.AssetsLocation.ContainerProxy != nil {
+		containerProxy := strings.TrimRight(*a.AssetsLocation.ContainerProxy, "/")
+		normalized := image
+
+		// If the image name contains only a single / we need to determine if the image is located on docker-hub or if it's using a convenient URL like k8s.gcr.io/<image-name>
+		// In case of a hub image it should be sufficient to just prepend the proxy url, producing eg docker-proxy.example.com/weaveworks/weave-kube
+		if strings.Count(normalized, "/") <= 1 && !strings.ContainsAny(strings.Split(normalized, "/")[0], ".:") {
+			normalized = containerProxy + "/" + normalized
+		} else {
+			var re = regexp.MustCompile(`^[^/]+`)
+			normalized = re.ReplaceAllString(normalized, containerProxy)
+		}
+
+		asset.DockerImage = normalized
+		asset.CanonicalLocation = image
+
+		// Run the new image
+		image = asset.DockerImage
+	}
+
 	if a.AssetsLocation != nil && a.AssetsLocation.ContainerRegistry != nil {
 		registryMirror := *a.AssetsLocation.ContainerRegistry
 		normalized := image

pkg/assets/builder_test.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package assets
+
+import (
+	"testing"
+
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/kops/util"
+)
+
+func buildAssetBuilder(t *testing.T) *AssetBuilder {
+
+	builder := &AssetBuilder{
+		AssetsLocation:  &kops.Assets{},
+		ContainerAssets: []*ContainerAsset{},
+	}
+	return builder
+}
+
+func TestValidate_RemapImage_ContainerProxy_AppliesToDockerHub(t *testing.T) {
+	builder := buildAssetBuilder(t)
+
+	proxyURL := "proxy.example.com/"
+	image := "weaveworks/weave-kube"
+	expected := "proxy.example.com/weaveworks/weave-kube"
+
+	builder.AssetsLocation.ContainerProxy = &proxyURL
+
+	remapped, err := builder.RemapImage(image)
+	if err != nil {
+		t.Error("Error remapping image", err)
+	}
+
+	if remapped != expected {
+		t.Errorf("Error remapping image (Expecting: %s, got %s)", expected, remapped)
+	}
+}
+
+func TestValidate_RemapImage_ContainerProxy_AppliesToSimplifiedDockerHub(t *testing.T) {
+	builder := buildAssetBuilder(t)
+
+	proxyURL := "proxy.example.com/"
+	image := "debian"
+	expected := "proxy.example.com/debian"
+
+	builder.AssetsLocation.ContainerProxy = &proxyURL
+
+	remapped, err := builder.RemapImage(image)
+	if err != nil {
+		t.Error("Error remapping image", err)
+	}
+
+	if remapped != expected {
+		t.Errorf("Error remapping image (Expecting: %s, got %s)", expected, remapped)
+	}
+}
+
+func TestValidate_RemapImage_ContainerProxy_AppliesToSimplifiedKubernetesURL(t *testing.T) {
+	builder := buildAssetBuilder(t)
+
+	proxyURL := "proxy.example.com/"
+	image := "k8s.gcr.io/kube-apiserver"
+	expected := "proxy.example.com/kube-apiserver"
+	version, _ := util.ParseKubernetesVersion("1.10")
+
+	builder.KubernetesVersion = *version
+	builder.AssetsLocation.ContainerProxy = &proxyURL
+
+	remapped, err := builder.RemapImage(image)
+	if err != nil {
+		t.Error("Error remapping image", err)
+	}
+
+	if remapped != expected {
+		t.Errorf("Error remapping image (Expecting: %s, got %s)", expected, remapped)
+	}
+}
+
+func TestValidate_RemapImage_ContainerProxy_AppliesToLegacyKubernetesURL(t *testing.T) {
+	builder := buildAssetBuilder(t)
+
+	proxyURL := "proxy.example.com/"
+	image := "gcr.io/google_containers/kube-apiserver"
+	expected := "proxy.example.com/google_containers/kube-apiserver"
+
+	builder.AssetsLocation.ContainerProxy = &proxyURL
+
+	remapped, err := builder.RemapImage(image)
+	if err != nil {
+		t.Error("Error remapping image", err)
+	}
+
+	if remapped != expected {
+		t.Errorf("Error remapping image (Expecting: %s, got %s)", expected, remapped)
+	}
+}
+
+func TestValidate_RemapImage_ContainerProxy_AppliesToImagesWithTags(t *testing.T) {
+	builder := buildAssetBuilder(t)
+
+	proxyURL := "proxy.example.com/"
+	image := "k8s.gcr.io/kube-apiserver:1.2.3"
+	expected := "proxy.example.com/kube-apiserver:1.2.3"
+	version, _ := util.ParseKubernetesVersion("1.10")
+
+	builder.KubernetesVersion = *version
+	builder.AssetsLocation.ContainerProxy = &proxyURL
+
+	remapped, err := builder.RemapImage(image)
+	if err != nil {
+		t.Error("Error remapping image", err)
+	}
+
+	if remapped != expected {
+		t.Errorf("Error remapping image (Expecting: %s, got %s)", expected, remapped)
+	}
+}

upup/pkg/fi/cloudup/validation_test.go

@@ -174,6 +174,27 @@ func TestValidate_ClusterName_Import(t *testing.T) {
 	expectNoErrorFromValidate(t, c)
 }
 
+func TestValidate_ContainerRegistry_and_ContainerProxy_exclusivity(t *testing.T) {
+	c := buildDefaultCluster(t)
+
+	assets := new(api.Assets)
+	c.Spec.Assets = assets
+
+	expectNoErrorFromValidate(t, c)
+
+	registry := "https://registry.example.com/"
+	c.Spec.Assets.ContainerRegistry = &registry
+	expectNoErrorFromValidate(t, c)
+
+	proxy := "https://proxy.example.com/"
+	c.Spec.Assets.ContainerProxy = &proxy
+	expectErrorFromValidate(t, c, "ContainerProxy cannot be used in conjunction with ContainerRegistry")
+
+	c.Spec.Assets.ContainerRegistry = nil
+	expectNoErrorFromValidate(t, c)
+
+}
+
 func expectErrorFromValidate(t *testing.T, c *api.Cluster, message string) {
 	err := validation.ValidateCluster(c, false)
 	if err == nil {