Add support for cluster using http forward proxy

docs/http_proxy.md

@@ -0,0 +1,40 @@
+
+HTTP Forward Proxy Support
+==========================
+
+It is possible to launch a kubernetes cluster from behind an http forward proxy ("corporate proxy").  To do so, you will need to configure the `egressProxy` for the cluster.
+
+It is assumed the proxy is already existing.  If you want a private topology on AWS, for example, with an proxy instead of a NAT instance, you'll need to create the proxy yourself.  See [Running in a shared VPC](run_in_existing_vpc.md).
+
+This configuration only manages proxy configurations for Kops and the Kubernetes cluster.  We can not handle proxy configuration for application containers and pods.
+
+## Configuration
+
+Add `spec.egressProxy` port and url as follows
+
+``` yaml
+spec:
+  egressProxy:
+    httpProxy:
+      host: proxy.corp.local
+      port: 3128
+```
+
+Currently we assume the same configuration for http and https traffic.
+
+## Proxy Excludes
+
+Most clients will blindly try to use the proxy to make all calls, even to localhost and the local subnet, unless configured otherwise.  Some basic exclusions necessary for successful launch and operation are added for you at initial cluster creation.  If you wish to add additional exclusions, add or edit `egressProxy.excludes` with a comma separated list of hostnames.  Matching is based on suffix, ie, `corp.local` will match `images.corp.local`, and `.corp.local` will match `corp.local` and `images.corp.local`, following typical `no_proxy` environment variable conventions.
+
+``` yaml
+spec:
+  egressProxy:
+    httpProxy:
+      host: proxy.corp.local
+      port: 3128
+    excludes: corp.local,internal.corp.com
+```
+
+## AWS VPC Endpoints and S3 access
+
+If you are hosting on AWS have configured VPC "Endpoints" for S3 or other services, you may want to add these to the `spec.egressProxy.excludes`.  Keep in mind that the S3 bucket must be in the same region as the VPC for it to be accessible via the endpoint.

docs/run_in_existing_vpc.md

@@ -146,3 +146,7 @@ Please note:
   in their route table.  Private subnets should not have public IPs, and will typically have a NAT gateway
   configured as their default route.
 * kops won't create a route-table at all if we're not creating subnets.
+
+### Proxy VPC Egress
+
+See [HTTP Forward Proxy Support](http_proxy.md)

nodeup/pkg/model/convenience.go

@@ -16,7 +16,14 @@ limitations under the License.
 
 package model
 
-import "k8s.io/kops/upup/pkg/fi"
+import (
+	"strconv"
+
+	"github.com/golang/glog"
+	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/upup/pkg/fi"
+)
 
 // s is a helper that builds a *string from a string value
 func s(v string) *string {
@@ -28,6 +35,33 @@ func i64(v int64) *int64 {
 	return fi.Int64(v)
 }
 
+func getProxyEnvVars(proxies *kops.EgressProxySpec) []v1.EnvVar {
+	if proxies == nil {
+		glog.V(8).Info("proxies is == nil, returning empty list")
+		return []v1.EnvVar{}
+	}
+
+	if proxies.HTTPProxy.Host == "" {
+		glog.Warning("EgressProxy set but no proxy host provided")
+	}
+
+	var httpProxyURL string
+	if proxies.HTTPProxy.Port == 0 {
+		httpProxyURL = "http://" + proxies.HTTPProxy.Host
+	} else {
+		httpProxyURL = "http://" + proxies.HTTPProxy.Host + ":" + strconv.Itoa(proxies.HTTPProxy.Port)
+	}
+
+	noProxy := proxies.ProxyExcludes
+
+	return []v1.EnvVar{
+		{Name: "http_proxy", Value: httpProxyURL},
+		{Name: "https_proxy", Value: httpProxyURL},
+		{Name: "NO_PROXY", Value: noProxy},
+		{Name: "no_proxy", Value: noProxy},
+	}
+}
+
 // b returns a pointer to a boolean
 func b(v bool) *bool {
 	return fi.Bool(v)

nodeup/pkg/model/kubeapiserver.go

@@ -223,6 +223,7 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 				HostPort:      8080,
 			},
 		},
+		Env: getProxyEnvVars(b.Cluster.Spec.EgressProxy),
 	}
 
 	for _, path := range b.SSLHostPaths() {

nodeup/pkg/model/kubecontrollermanager.go

@@ -186,6 +186,7 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) {
 			InitialDelaySeconds: 15,
 			TimeoutSeconds:      15,
 		},
+		Env: getProxyEnvVars(b.Cluster.Spec.EgressProxy),
 	}
 
 	for _, path := range b.SSLHostPaths() {

nodeup/pkg/model/kubescheduler.go

@@ -140,6 +140,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) {
 			InitialDelaySeconds: 15,
 			TimeoutSeconds:      15,
 		},
+		Env: getProxyEnvVars(b.Cluster.Spec.EgressProxy),
 	}
 
 	addHostPathMapping(pod, container, "varlibkubescheduler", "/var/lib/kube-scheduler")

nodeup/pkg/model/protokube.go

@@ -276,6 +276,9 @@ func (t *ProtokubeBuilder) ProtokubeFlags(k8sVersion semver.Version) *ProtokubeF
 func (t *ProtokubeBuilder) ProtokubeEnvironmentVariables() string {
 	var buffer bytes.Buffer
 
+	// TODO write out an environments file for this.  This is getting a tad long.
+
+	// Pass in required credentials when using user-defined s3 endpoint
 	if os.Getenv("AWS_REGION") != "" {
 		buffer.WriteString(" ")
 		buffer.WriteString("-e 'AWS_REGION=")
@@ -284,7 +287,6 @@ func (t *ProtokubeBuilder) ProtokubeEnvironmentVariables() string {
 		buffer.WriteString(" ")
 	}
 
-	// Pass in required credentials when using user-defined s3 endpoint
 	if os.Getenv("S3_ENDPOINT") != "" {
 		buffer.WriteString(" ")
 		buffer.WriteString("-e S3_ENDPOINT=")
@@ -306,9 +308,21 @@ func (t *ProtokubeBuilder) ProtokubeEnvironmentVariables() string {
 		buffer.WriteString(" ")
 	}
 
+	t.writeProxyEnvVars(&buffer)
+
 	return buffer.String()
 }
 
+func (t *ProtokubeBuilder) writeProxyEnvVars(buffer *bytes.Buffer) {
+	for _, envVar := range getProxyEnvVars(t.Cluster.Spec.EgressProxy) {
+		buffer.WriteString(" -e ")
+		buffer.WriteString(envVar.Name)
+		buffer.WriteString("=")
+		buffer.WriteString(envVar.Value)
+		buffer.WriteString(" ")
+	}
+}
+
 // buildCertificateTask is responsible for build a certificate request task
 func (t *ProtokubeBuilder) buildCeritificateTask(c *fi.ModelBuilderContext, name, filename string) error {
 	cert, err := t.KeyStore.Cert(name)

pkg/apis/kops/cluster.go

@@ -93,6 +93,9 @@ type ClusterSpec struct {
 	NonMasqueradeCIDR string `json:"nonMasqueradeCIDR,omitempty"`
 	// SSHAccess is a list of the CIDRs that can access SSH.
 	SSHAccess []string `json:"sshAccess,omitempty"`
+	// HTTPProxy defines connection information to support use of a private cluster behind an forward HTTP Proxy
+	EgressProxy *EgressProxySpec `json:"egressProxy,omitempty"`
+
 	// KubernetesAPIAccess is a list of the CIDRs that can access the Kubernetes API endpoint (master HTTPS)
 	KubernetesAPIAccess []string `json:"kubernetesApiAccess,omitempty"`
 	// IsolatesMasters determines whether we should lock down masters so that they are not on the pod network.
@@ -285,6 +288,19 @@ type ClusterSubnetSpec struct {
 	Type SubnetType `json:"type,omitempty"`
 }
 
+type EgressProxySpec struct {
+	HTTPProxy     HTTPProxy `json:"httpProxy,omitempty"`
+	ProxyExcludes string    `json:"excludes,omitempty"`
+}
+
+type HTTPProxy struct {
+	Host string `json:"host,omitempty"`
+	Port int    `json:"port,omitempty"`
+	// TODO #3070
+	// User     string `json:"user,omitempty"`
+	// Password string `json:"password,omitempty"`
+}
+
 // FillDefaults populates default values.
 // This is different from PerformAssignments, because these values are changeable, and thus we don't need to
 // store them (i.e. we don't need to 'lock them')

pkg/apis/kops/v1alpha1/cluster.go

@@ -212,6 +212,9 @@ type ClusterSpec struct {
 	//KubeProxyTestArgs             string `json:",omitempty"`
 	//KubeProxyTestLogLevel         string `json:",omitempty"`
 
+	// HTTPProxy defines connection information to support use of a private cluster behind an forward HTTP Proxy
+	EgressProxy *EgressProxySpec `json:"egressProxy,omitempty"`
+
 	// EtcdClusters stores the configuration for each cluster
 	EtcdClusters []*EtcdClusterSpec `json:"etcdClusters,omitempty"`
 
@@ -368,3 +371,16 @@ type ClusterZoneSpec struct {
 
 	Egress string `json:"egress,omitempty"`
 }
+
+type EgressProxySpec struct {
+	HTTPProxy     HTTPProxy `json:"httpProxy,omitempty"`
+	ProxyExcludes string    `json:"excludes,omitempty"`
+}
+
+type HTTPProxy struct {
+	Host string `json:"host,omitempty"`
+	Port int    `json:"port,omitempty"`
+	// TODO #3070
+	// User     string `json:"user,omitempty"`
+	// Password string `json:"password,omitempty"`
+}

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -67,6 +67,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_DNSSpec_To_v1alpha1_DNSSpec,
 		Convert_v1alpha1_DockerConfig_To_kops_DockerConfig,
 		Convert_kops_DockerConfig_To_v1alpha1_DockerConfig,
+		Convert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec,
+		Convert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec,
 		Convert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec,
 		Convert_kops_EtcdClusterSpec_To_v1alpha1_EtcdClusterSpec,
 		Convert_v1alpha1_EtcdMemberSpec_To_kops_EtcdMemberSpec,
@@ -83,6 +85,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_FederationSpec_To_v1alpha1_FederationSpec,
 		Convert_v1alpha1_FlannelNetworkingSpec_To_kops_FlannelNetworkingSpec,
 		Convert_kops_FlannelNetworkingSpec_To_v1alpha1_FlannelNetworkingSpec,
+		Convert_v1alpha1_HTTPProxy_To_kops_HTTPProxy,
+		Convert_kops_HTTPProxy_To_v1alpha1_HTTPProxy,
 		Convert_v1alpha1_HookSpec_To_kops_HookSpec,
 		Convert_kops_HookSpec_To_v1alpha1_HookSpec,
 		Convert_v1alpha1_InstanceGroup_To_kops_InstanceGroup,
@@ -524,6 +528,15 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
 	out.AdditionalPolicies = in.AdditionalPolicies
+	if in.EgressProxy != nil {
+		in, out := &in.EgressProxy, &out.EgressProxy
+		*out = new(kops.EgressProxySpec)
+		if err := Convert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.EgressProxy = nil
+	}
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters
 		*out = make([]*kops.EtcdClusterSpec, len(*in))
@@ -705,6 +718,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec,
 	out.ServiceClusterIPRange = in.ServiceClusterIPRange
 	out.NonMasqueradeCIDR = in.NonMasqueradeCIDR
 	// WARNING: in.SSHAccess requires manual conversion: does not exist in peer-type
+	if in.EgressProxy != nil {
+		in, out := &in.EgressProxy, &out.EgressProxy
+		*out = new(EgressProxySpec)
+		if err := Convert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.EgressProxy = nil
+	}
 	// WARNING: in.KubernetesAPIAccess requires manual conversion: does not exist in peer-type
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
@@ -946,6 +968,32 @@ func Convert_kops_DockerConfig_To_v1alpha1_DockerConfig(in *kops.DockerConfig, o
 	return autoConvert_kops_DockerConfig_To_v1alpha1_DockerConfig(in, out, s)
 }
 
+func autoConvert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec(in *EgressProxySpec, out *kops.EgressProxySpec, s conversion.Scope) error {
+	if err := Convert_v1alpha1_HTTPProxy_To_kops_HTTPProxy(&in.HTTPProxy, &out.HTTPProxy, s); err != nil {
+		return err
+	}
+	out.ProxyExcludes = in.ProxyExcludes
+	return nil
+}
+
+// Convert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec is an autogenerated conversion function.
+func Convert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec(in *EgressProxySpec, out *kops.EgressProxySpec, s conversion.Scope) error {
+	return autoConvert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec(in, out, s)
+}
+
+func autoConvert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec(in *kops.EgressProxySpec, out *EgressProxySpec, s conversion.Scope) error {
+	if err := Convert_kops_HTTPProxy_To_v1alpha1_HTTPProxy(&in.HTTPProxy, &out.HTTPProxy, s); err != nil {
+		return err
+	}
+	out.ProxyExcludes = in.ProxyExcludes
+	return nil
+}
+
+// Convert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec is an autogenerated conversion function.
+func Convert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec(in *kops.EgressProxySpec, out *EgressProxySpec, s conversion.Scope) error {
+	return autoConvert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec(in, out, s)
+}
+
 func autoConvert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdClusterSpec, out *kops.EtcdClusterSpec, s conversion.Scope) error {
 	out.Name = in.Name
 	out.EnableEtcdTLS = in.EnableEtcdTLS
@@ -1162,6 +1210,28 @@ func Convert_kops_FlannelNetworkingSpec_To_v1alpha1_FlannelNetworkingSpec(in *ko
 	return autoConvert_kops_FlannelNetworkingSpec_To_v1alpha1_FlannelNetworkingSpec(in, out, s)
 }
 
+func autoConvert_v1alpha1_HTTPProxy_To_kops_HTTPProxy(in *HTTPProxy, out *kops.HTTPProxy, s conversion.Scope) error {
+	out.Host = in.Host
+	out.Port = in.Port
+	return nil
+}
+
+// Convert_v1alpha1_HTTPProxy_To_kops_HTTPProxy is an autogenerated conversion function.
+func Convert_v1alpha1_HTTPProxy_To_kops_HTTPProxy(in *HTTPProxy, out *kops.HTTPProxy, s conversion.Scope) error {
+	return autoConvert_v1alpha1_HTTPProxy_To_kops_HTTPProxy(in, out, s)
+}
+
+func autoConvert_kops_HTTPProxy_To_v1alpha1_HTTPProxy(in *kops.HTTPProxy, out *HTTPProxy, s conversion.Scope) error {
+	out.Host = in.Host
+	out.Port = in.Port
+	return nil
+}
+
+// Convert_kops_HTTPProxy_To_v1alpha1_HTTPProxy is an autogenerated conversion function.
+func Convert_kops_HTTPProxy_To_v1alpha1_HTTPProxy(in *kops.HTTPProxy, out *HTTPProxy, s conversion.Scope) error {
+	return autoConvert_kops_HTTPProxy_To_v1alpha1_HTTPProxy(in, out, s)
+}
+
 func autoConvert_v1alpha1_HookSpec_To_kops_HookSpec(in *HookSpec, out *kops.HookSpec, s conversion.Scope) error {
 	if in.ExecContainer != nil {
 		in, out := &in.ExecContainer, &out.ExecContainer

pkg/apis/kops/v1alpha2/cluster.go

@@ -116,6 +116,9 @@ type ClusterSpec struct {
 	// Currently only a single CIDR is supported (though a richer grammar could be added in future)
 	SSHAccess []string `json:"sshAccess,omitempty"`
 
+	// HTTPProxy defines connection information to support use of a private cluster behind an forward HTTP Proxy
+	EgressProxy *EgressProxySpec `json:"egressProxy,omitempty"`
+
 	// KubernetesAPIAccess determines the permitted access to the API endpoints (master HTTPS)
 	// Currently only a single CIDR is supported (though a richer grammar could be added in future)
 	KubernetesAPIAccess []string `json:"kubernetesApiAccess,omitempty"`
@@ -293,3 +296,17 @@ type ClusterSubnetSpec struct {
 
 	Type SubnetType `json:"type,omitempty"`
 }
+
+type EgressProxySpec struct {
+	HTTPProxy     HTTPProxy `json:"httpProxy,omitempty"`
+	ProxyExcludes string    `json:"excludes,omitempty"`
+}
+
+type HTTPProxy struct {
+	Host string `json:"host,omitempty"`
+	Port int    `json:"port,omitempty"`
+
+	// TODO #3070
+	// User     string `json:"user,omitempty"`
+	// Password string `json:"password,omitempty"`
+}

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -71,6 +71,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_DNSSpec_To_v1alpha2_DNSSpec,
 		Convert_v1alpha2_DockerConfig_To_kops_DockerConfig,
 		Convert_kops_DockerConfig_To_v1alpha2_DockerConfig,
+		Convert_v1alpha2_EgressProxySpec_To_kops_EgressProxySpec,
+		Convert_kops_EgressProxySpec_To_v1alpha2_EgressProxySpec,
 		Convert_v1alpha2_EtcdClusterSpec_To_kops_EtcdClusterSpec,
 		Convert_kops_EtcdClusterSpec_To_v1alpha2_EtcdClusterSpec,
 		Convert_v1alpha2_EtcdMemberSpec_To_kops_EtcdMemberSpec,
@@ -87,6 +89,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_FederationSpec_To_v1alpha2_FederationSpec,
 		Convert_v1alpha2_FlannelNetworkingSpec_To_kops_FlannelNetworkingSpec,
 		Convert_kops_FlannelNetworkingSpec_To_v1alpha2_FlannelNetworkingSpec,
+		Convert_v1alpha2_HTTPProxy_To_kops_HTTPProxy,
+		Convert_kops_HTTPProxy_To_v1alpha2_HTTPProxy,
 		Convert_v1alpha2_HookSpec_To_kops_HookSpec,
 		Convert_kops_HookSpec_To_v1alpha2_HookSpec,
 		Convert_v1alpha2_InstanceGroup_To_kops_InstanceGroup,
@@ -558,6 +562,15 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	out.ServiceClusterIPRange = in.ServiceClusterIPRange
 	out.NonMasqueradeCIDR = in.NonMasqueradeCIDR
 	out.SSHAccess = in.SSHAccess
+	if in.EgressProxy != nil {
+		in, out := &in.EgressProxy, &out.EgressProxy
+		*out = new(kops.EgressProxySpec)
+		if err := Convert_v1alpha2_EgressProxySpec_To_kops_EgressProxySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.EgressProxy = nil
+	}
 	out.KubernetesAPIAccess = in.KubernetesAPIAccess
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
@@ -758,6 +771,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec,
 	out.ServiceClusterIPRange = in.ServiceClusterIPRange
 	out.NonMasqueradeCIDR = in.NonMasqueradeCIDR
 	out.SSHAccess = in.SSHAccess
+	if in.EgressProxy != nil {
+		in, out := &in.EgressProxy, &out.EgressProxy
+		*out = new(EgressProxySpec)
+		if err := Convert_kops_EgressProxySpec_To_v1alpha2_EgressProxySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.EgressProxy = nil
+	}
 	out.KubernetesAPIAccess = in.KubernetesAPIAccess
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
@@ -1034,6 +1056,32 @@ func Convert_kops_DockerConfig_To_v1alpha2_DockerConfig(in *kops.DockerConfig, o
 	return autoConvert_kops_DockerConfig_To_v1alpha2_DockerConfig(in, out, s)
 }
 
+func autoConvert_v1alpha2_EgressProxySpec_To_kops_EgressProxySpec(in *EgressProxySpec, out *kops.EgressProxySpec, s conversion.Scope) error {
+	if err := Convert_v1alpha2_HTTPProxy_To_kops_HTTPProxy(&in.HTTPProxy, &out.HTTPProxy, s); err != nil {
+		return err
+	}
+	out.ProxyExcludes = in.ProxyExcludes
+	return nil
+}
+
+// Convert_v1alpha2_EgressProxySpec_To_kops_EgressProxySpec is an autogenerated conversion function.
+func Convert_v1alpha2_EgressProxySpec_To_kops_EgressProxySpec(in *EgressProxySpec, out *kops.EgressProxySpec, s conversion.Scope) error {
+	return autoConvert_v1alpha2_EgressProxySpec_To_kops_EgressProxySpec(in, out, s)
+}
+
+func autoConvert_kops_EgressProxySpec_To_v1alpha2_EgressProxySpec(in *kops.EgressProxySpec, out *EgressProxySpec, s conversion.Scope) error {
+	if err := Convert_kops_HTTPProxy_To_v1alpha2_HTTPProxy(&in.HTTPProxy, &out.HTTPProxy, s); err != nil {
+		return err
+	}
+	out.ProxyExcludes = in.ProxyExcludes
+	return nil
+}
+
+// Convert_kops_EgressProxySpec_To_v1alpha2_EgressProxySpec is an autogenerated conversion function.
+func Convert_kops_EgressProxySpec_To_v1alpha2_EgressProxySpec(in *kops.EgressProxySpec, out *EgressProxySpec, s conversion.Scope) error {
+	return autoConvert_kops_EgressProxySpec_To_v1alpha2_EgressProxySpec(in, out, s)
+}
+
 func autoConvert_v1alpha2_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdClusterSpec, out *kops.EtcdClusterSpec, s conversion.Scope) error {
 	out.Name = in.Name
 	out.EnableEtcdTLS = in.EnableEtcdTLS
@@ -1260,6 +1308,28 @@ func Convert_kops_FlannelNetworkingSpec_To_v1alpha2_FlannelNetworkingSpec(in *ko
 	return autoConvert_kops_FlannelNetworkingSpec_To_v1alpha2_FlannelNetworkingSpec(in, out, s)
 }
 
+func autoConvert_v1alpha2_HTTPProxy_To_kops_HTTPProxy(in *HTTPProxy, out *kops.HTTPProxy, s conversion.Scope) error {
+	out.Host = in.Host
+	out.Port = in.Port
+	return nil
+}
+
+// Convert_v1alpha2_HTTPProxy_To_kops_HTTPProxy is an autogenerated conversion function.
+func Convert_v1alpha2_HTTPProxy_To_kops_HTTPProxy(in *HTTPProxy, out *kops.HTTPProxy, s conversion.Scope) error {
+	return autoConvert_v1alpha2_HTTPProxy_To_kops_HTTPProxy(in, out, s)
+}
+
+func autoConvert_kops_HTTPProxy_To_v1alpha2_HTTPProxy(in *kops.HTTPProxy, out *HTTPProxy, s conversion.Scope) error {
+	out.Host = in.Host
+	out.Port = in.Port
+	return nil
+}
+
+// Convert_kops_HTTPProxy_To_v1alpha2_HTTPProxy is an autogenerated conversion function.
+func Convert_kops_HTTPProxy_To_v1alpha2_HTTPProxy(in *kops.HTTPProxy, out *HTTPProxy, s conversion.Scope) error {
+	return autoConvert_kops_HTTPProxy_To_v1alpha2_HTTPProxy(in, out, s)
+}
+
 func autoConvert_v1alpha2_HookSpec_To_kops_HookSpec(in *HookSpec, out *kops.HookSpec, s conversion.Scope) error {
 	if in.ExecContainer != nil {
 		in, out := &in.ExecContainer, &out.ExecContainer

pkg/model/awsmodel/autoscalinggroup.go

@@ -117,7 +117,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 				return err
 			}
 
-			if t.UserData, err = b.BootstrapScript.ResourceNodeUp(ig); err != nil {
+			if t.UserData, err = b.BootstrapScript.ResourceNodeUp(ig, b.Cluster.Spec.EgressProxy); err != nil {
 				return err
 			}
 

pkg/model/bootstrapscript.go

@@ -17,8 +17,11 @@ limitations under the License.
 package model
 
 import (
+	"bytes"
 	"fmt"
 	"os"
+	"strconv"
+	"strings"
 	"text/template"
 
 	"k8s.io/kops/pkg/apis/kops"
@@ -34,7 +37,7 @@ type BootstrapScript struct {
 	NodeUpConfigBuilder func(ig *kops.InstanceGroup) (*nodeup.Config, error)
 }
 
-func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup) (*fi.ResourceHolder, error) {
+func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, ps *kops.EgressProxySpec) (*fi.ResourceHolder, error) {
 	if ig.Spec.Role == kops.InstanceGroupRoleBastion {
 		// Bastions are just bare machines (currently), used as SSH jump-hosts
 		return nil, nil
@@ -73,6 +76,9 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup) (*fi.ResourceHo
 			return ""
 		},
 
+		"ProxyEnv": func() string {
+			return b.createProxyEnv(ps)
+		},
 		"AWS_REGION": func() string {
 			if os.Getenv("AWS_REGION") != "" {
 				return fmt.Sprintf("export AWS_REGION=%s\n",
@@ -88,3 +94,66 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup) (*fi.ResourceHo
 	}
 	return fi.WrapResource(templateResource), nil
 }
+
+func (b *BootstrapScript) createProxyEnv(ps *kops.EgressProxySpec) string {
+	var buffer bytes.Buffer
+
+	if ps != nil && ps.HTTPProxy.Host != "" {
+		var httpProxyUrl string
+
+		// TODO double check that all the code does this
+		// TODO move this into a validate so we can enforce the string syntax
+		if !strings.HasPrefix(ps.HTTPProxy.Host, "http://") {
+			httpProxyUrl = "http://"
+		}
+
+		if ps.HTTPProxy.Port != 0 {
+			httpProxyUrl += ps.HTTPProxy.Host + ":" + strconv.Itoa(ps.HTTPProxy.Port)
+		} else {
+			httpProxyUrl += ps.HTTPProxy.Host
+		}
+
+		// Set base env variables
+		buffer.WriteString("export http_proxy=" + httpProxyUrl + "\n")
+		buffer.WriteString("export https_proxy=${http_proxy}\n")
+		buffer.WriteString("export no_proxy=" + ps.ProxyExcludes + "\n")
+		buffer.WriteString("export NO_PROXY=${no_proxy}\n")
+
+		// TODO move the rest of this configuration work to nodeup
+
+		// Set env variables for docker
+		buffer.WriteString("echo \"export http_proxy=${http_proxy}\" >> /etc/default/docker\n")
+		buffer.WriteString("echo \"export https_proxy=${http_proxy}\" >> /etc/default/docker\n")
+		buffer.WriteString("echo \"export no_proxy=${no_proxy}\" >> /etc/default/docker\n")
+		buffer.WriteString("echo \"export NO_PROXY=${no_proxy}\" >> /etc/default/docker\n")
+
+		// Set env variables for base environment
+		buffer.WriteString("echo \"export http_proxy=${http_proxy}\" >> /etc/environment\n")
+		buffer.WriteString("echo \"export https_proxy=${http_proxy}\" >> /etc/environment\n")
+		buffer.WriteString("echo \"export no_proxy=${no_proxy}\" >> /etc/environment\n")
+		buffer.WriteString("echo \"export NO_PROXY=${no_proxy}\" >> /etc/environment\n")
+
+		// Set env variables to systemd
+		buffer.WriteString("echo DefaultEnvironment=\\\"http_proxy=${http_proxy}\\\" \\\"https_proxy=${http_proxy}\\\"")
+		buffer.WriteString("echo DefaultEnvironment=\\\"http_proxy=${http_proxy}\\\" \\\"https_proxy=${http_proxy}\\\"")
+		buffer.WriteString(" \\\"NO_PROXY=${no_proxy}\\\" \\\"no_proxy=${no_proxy}\\\"")
+		buffer.WriteString(" >> /etc/systemd/system.conf\n")
+
+		// source in the environment this step ensures that environment file is correct
+		buffer.WriteString("source /etc/environment\n")
+
+		// Restart stuff
+		buffer.WriteString("systemctl daemon-reload\n")
+		buffer.WriteString("systemctl daemon-reexec\n")
+
+		// TODO do we need no_proxy in these as well??
+		// TODO handle CoreOS
+		// Depending on OS set package manager proxy settings
+		buffer.WriteString("if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ]; then\n")
+		buffer.WriteString("    echo \"Acquire::http::Proxy \\\"${http_proxy}\\\";\" > /etc/apt/apt.conf.d/30proxy\n")
+		buffer.WriteString("elif [ -f /etc/redhat-release ]; then\n")
+		buffer.WriteString("  echo \"http_proxy=${http_proxy}\" >> /etc/yum.conf\n")
+		buffer.WriteString("fi\n")
+	}
+	return buffer.String()
+}

pkg/model/bootstrapscript_test.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package model
+
+import (
+	"strings"
+	"testing"
+
+	"k8s.io/kops/pkg/apis/kops"
+)
+
+func Test_ProxyFunc(t *testing.T) {
+	b := &BootstrapScript{}
+	ps := &kops.EgressProxySpec{
+		HTTPProxy: kops.HTTPProxy{
+			Host: "example.com",
+			Port: 80,
+		},
+	}
+
+	script := b.createProxyEnv(ps)
+
+	if script == "" {
+		t.Fatalf("script cannot be empty")
+	}
+
+	if !strings.HasPrefix(script, "export http_proxy=http://example.com:80") {
+		t.Fatalf("script not setting http_proxy properly")
+	}
+
+	ps.ProxyExcludes = "www.google.com,www.kubernetes.io"
+
+	script = b.createProxyEnv(ps)
+	t.Logf(script)
+	if !strings.Contains(script, "export no_proxy="+ps.ProxyExcludes) {
+		t.Fatalf("script not setting no_proxy properly")
+	}
+
+}

pkg/model/gcemodel/autoscalinggroup.go

@@ -44,7 +44,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	for _, ig := range b.InstanceGroups {
 		name := b.SafeObjectName(ig.ObjectMeta.Name)
 
-		startupScript, err := b.BootstrapScript.ResourceNodeUp(ig)
+		startupScript, err := b.BootstrapScript.ResourceNodeUp(ig, b.Cluster.Spec.EgressProxy)
 		if err != nil {
 			return err
 		}

pkg/model/resources/nodeup.go

@@ -41,6 +41,8 @@ NODEUP_HASH={{ NodeUpSourceHash }}
 {{ S3Env }}
 {{ AWS_REGION }}
 
+{{ ProxyEnv }}
+
 function ensure-install-dir() {
   INSTALL_DIR="/var/cache/kubernetes-install"
   # On ContainerOS, we install to /var/lib/toolbox install (because of noexec)

upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template

@@ -38,6 +38,13 @@ spec:
 {{ range $arg := DnsControllerArgv }}
         - "{{ $arg }}"
 {{ end }}
+{{- if .EgressProxy }}
+        env:
+{{ range $name, $value := ProxyEnv }}
+        - name: {{ $name }}
+          value: {{ $value }}
+{{ end }}
+{{- end }}
         resources:
           requests:
             cpu: 50m

upup/pkg/fi/cloudup/defaults.go

@@ -18,10 +18,12 @@ package cloudup
 
 import (
 	"fmt"
+	"net"
+	"strings"
+
 	"github.com/golang/glog"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/util/pkg/vfs"
-	"strings"
 
 	kopsversion "k8s.io/kops"
 )
@@ -81,6 +83,11 @@ func PerformAssignments(c *kops.Cluster) error {
 		return err
 	}
 
+	c.Spec.EgressProxy, err = assignProxy(c)
+	if err != nil {
+		return err
+	}
+
 	return ensureKubernetesVersion(c)
 }
 
@@ -129,3 +136,84 @@ func FindLatestKubernetesVersion() (string, error) {
 	latestVersion := strings.TrimSpace(string(b))
 	return latestVersion, nil
 }
+
+func assignProxy(cluster *kops.Cluster) (*kops.EgressProxySpec, error) {
+
+	egressProxy := cluster.Spec.EgressProxy
+	// Add default no_proxy values if we are using a http proxy
+	if egressProxy != nil {
+
+		var egressSlice []string
+		if egressProxy.ProxyExcludes != "" {
+			egressSlice = strings.Split(egressProxy.ProxyExcludes, ",")
+		}
+
+		ip, _, err := net.ParseCIDR(cluster.Spec.NonMasqueradeCIDR)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse Non Masquerade CIDR")
+		}
+
+		firstIP, err := incrementIP(ip, cluster.Spec.NonMasqueradeCIDR)
+		if err != nil {
+			return nil, fmt.Errorf("unable to get first ip address in Non Masquerade CIDR")
+		}
+
+		// run through the basic list
+		for _, exclude := range []string{
+			"127.0.0.1",
+			"localhost",
+			cluster.Spec.ClusterDNSDomain, // TODO we may want this for public loadbalancers
+			cluster.Spec.MasterPublicName,
+			cluster.ObjectMeta.Name,
+			firstIP,
+			cluster.Spec.NonMasqueradeCIDR,
+		} {
+			if exclude == "" {
+				continue
+			}
+			if !strings.Contains(egressProxy.ProxyExcludes, exclude) {
+				egressSlice = append(egressSlice, exclude)
+			}
+		}
+
+		awsNoProxy := "169.254.169.254"
+
+		if cluster.Spec.CloudProvider == "aws" && !strings.Contains(cluster.Spec.EgressProxy.ProxyExcludes, awsNoProxy) {
+			egressSlice = append(egressSlice, awsNoProxy)
+		}
+
+		// the kube-apiserver will need to talk to kubelets on their node IP addresses port 10250
+		// for pod logs to be available via the api
+		if cluster.Spec.NetworkCIDR != "" {
+			if !strings.Contains(cluster.Spec.EgressProxy.ProxyExcludes, cluster.Spec.NetworkCIDR) {
+				egressSlice = append(egressSlice, cluster.Spec.NetworkCIDR)
+			}
+		} else {
+			glog.Warningf("No NetworkCIDR defined (yet), not adding to egressProxy.excludes")
+		}
+
+		egressProxy.ProxyExcludes = strings.Join(egressSlice, ",")
+		glog.V(8).Infof("Completed setting up Proxy excludes as follows: %q", egressProxy.ProxyExcludes)
+	} else {
+		glog.V(8).Info("Not setting up Proxy Excludes")
+	}
+
+	return egressProxy, nil
+}
+
+func incrementIP(ip net.IP, cidr string) (string, error) {
+	_, ipNet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return "", err
+	}
+	for i := len(ip) - 1; i >= 0; i-- {
+		ip[i]++
+		if ip[i] != 0 {
+			break
+		}
+	}
+	if !ipNet.Contains(ip) {
+		return "", fmt.Errorf("overflowed CIDR while incrementing IP")
+	}
+	return ip.String(), nil
+}

upup/pkg/fi/cloudup/defaults_test.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudup
+
+import (
+	"k8s.io/kops/pkg/apis/kops"
+	"testing"
+)
+
+func TestPopulateClusterSpec_Proxy(t *testing.T) {
+	c := buildMinimalCluster()
+
+	c.Spec.EgressProxy = &kops.EgressProxySpec{
+		ProxyExcludes: "google.com",
+		HTTPProxy: kops.HTTPProxy{
+			Host: "52.205.179.249",
+			Port: 3128,
+		},
+	}
+
+	c.Spec.NonMasqueradeCIDR = "100.64.0.1/10"
+	c.Spec.NetworkCIDR = "192.168.0.0/20"
+	var err error
+	c.Spec.EgressProxy, err = assignProxy(c)
+	if err != nil {
+		t.Fatalf("unable to assign proxy, %v", err)
+	}
+
+	expectedExcludes := "google.com,127.0.0.1,localhost,testcluster.test.com,100.64.0.2,100.64.0.1/10,169.254.169.254,192.168.0.0/20"
+	if c.Spec.EgressProxy.ProxyExcludes != expectedExcludes {
+		t.Fatalf("Incorrect proxy excludes set: %v, expected %v", c.Spec.EgressProxy.ProxyExcludes, expectedExcludes)
+	}
+
+	c.Spec.EgressProxy = &kops.EgressProxySpec{
+		HTTPProxy: kops.HTTPProxy{
+			Host: "52.205.179.249",
+			Port: 3128,
+		},
+	}
+
+	c.Spec.NonMasqueradeCIDR = "100.64.0.0/10"
+	c.Spec.NetworkCIDR = "192.168.0.0/20"
+	c.Spec.EgressProxy.ProxyExcludes = ""
+
+	c.Spec.EgressProxy, err = assignProxy(c)
+	if err != nil {
+		t.Fatalf("unable to assign proxy, %v", err)
+	}
+
+	expectedExcludes = "127.0.0.1,localhost,testcluster.test.com,100.64.0.1,100.64.0.0/10,169.254.169.254,192.168.0.0/20"
+	if c.Spec.EgressProxy.ProxyExcludes != expectedExcludes {
+		t.Fatalf("Incorrect proxy excludes set: %v, expected %v", c.Spec.EgressProxy.ProxyExcludes, expectedExcludes)
+	}
+
+	c.Spec.NonMasqueradeCIDR = "172.16.0.5/12"
+	c.Spec.NetworkCIDR = "192.168.0.0/20"
+	c.Spec.CloudProvider = "gce"
+	c.Spec.EgressProxy.ProxyExcludes = ""
+	c.Spec.EgressProxy, err = assignProxy(c)
+	if err != nil {
+		t.Fatalf("unable to assign proxy, %v", err)
+	}
+
+	expectedExcludes = "127.0.0.1,localhost,testcluster.test.com,172.16.0.6,172.16.0.5/12,192.168.0.0/20"
+	if c.Spec.EgressProxy.ProxyExcludes != expectedExcludes {
+		t.Fatalf("Incorrect proxy excludes set: %v", c.Spec.EgressProxy.ProxyExcludes)
+	}
+
+	// idempotency test
+	c.Spec.EgressProxy, err = assignProxy(c)
+	if err != nil {
+		t.Fatalf("unable to assign proxy, %v", err)
+	}
+
+	expectedExcludes = "127.0.0.1,localhost,testcluster.test.com,172.16.0.6,172.16.0.5/12,192.168.0.0/20"
+	if c.Spec.EgressProxy.ProxyExcludes != expectedExcludes {
+		t.Fatalf("Incorrect proxy excludes set during idempotency check: %v    should have been %v", c.Spec.EgressProxy.ProxyExcludes, expectedExcludes)
+	}
+
+}

upup/pkg/fi/cloudup/template_functions.go

@@ -31,6 +31,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"text/template"
 
@@ -102,6 +103,8 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap) {
 	dest["Region"] = func() string {
 		return tf.region
 	}
+
+	dest["ProxyEnv"] = tf.ProxyEnv
 }
 
 // SharedVPC is a simple helper function which makes the templates for a shared VPC clearer
@@ -197,3 +200,28 @@ func (tf *TemplateFunctions) ExternalDnsArgv() ([]string, error) {
 
 	return argv, nil
 }
+
+func (tf *TemplateFunctions) ProxyEnv() map[string]string {
+	envs := map[string]string{}
+	proxies := tf.cluster.Spec.EgressProxy
+	if proxies == nil {
+		return envs
+	}
+	httpProxy := proxies.HTTPProxy
+	if httpProxy.Host != "" {
+		var portSuffix string
+		if httpProxy.Port != 0 {
+			portSuffix = ":" + strconv.Itoa(httpProxy.Port)
+		} else {
+			portSuffix = ""
+		}
+		url := "http://" + httpProxy.Host + portSuffix
+		envs["http_proxy"] = url
+		envs["https_proxy"] = url
+	}
+	if proxies.ProxyExcludes != "" {
+		envs["no_proxy"] = proxies.ProxyExcludes
+		envs["NO_PROXY"] = proxies.ProxyExcludes
+	}
+	return envs
+}

upup/pkg/fi/cloudup/vspheretasks/attachiso.go

@@ -91,7 +91,8 @@ func (_ *AttachISO) CheckChanges(a, e, changes *AttachISO) error {
 
 // RenderVSphere executes the actual task logic, for vSphere cloud.
 func (_ *AttachISO) RenderVSphere(t *vsphere.VSphereAPITarget, a, e, changes *AttachISO) error {
-	startupScript, err := changes.BootstrapScript.ResourceNodeUp(changes.IG)
+	// TODO #3071 .. need to replace the nil for http proxy support
+	startupScript, err := changes.BootstrapScript.ResourceNodeUp(changes.IG, nil)
 	if err != nil {
 		return fmt.Errorf("error on resource nodeup: %v", err)
 	}