Simple mirror support

We recognize our primary location via string-matching, and we then
have a hard-coded list of mirrors for that location.

Didn't prove easy to make this much better, but we can hopefully do so
iteratively (e.g. fetch mirrors via URL)

upup/pkg/fi/assetstore.go

@@ -154,45 +154,67 @@ func hashFromHttpHeader(url string) (*hashing.Hash, error) {
 // Add an asset into the store, in one of the recognized formats (see Assets in types package)
 func (a *AssetStore) Add(id string) error {
 	if strings.HasPrefix(id, "http://") || strings.HasPrefix(id, "https://") {
-		return a.addURL(id, nil)
+		return a.addURLs(strings.Split(id, ","), nil)
 	}
 	i := strings.Index(id, "@http://")
 	if i == -1 {
 		i = strings.Index(id, "@https://")
 	}
 	if i != -1 {
-		url := id[i+1:]
+		urls := strings.Split(id[i+1:], ",")
 		hash, err := hashing.FromString(id[:i])
 		if err != nil {
 			return err
 		}
-		return a.addURL(url, hash)
+		return a.addURLs(urls, hash)
 	}
 	// TODO: local files!
 	return fmt.Errorf("unknown asset format: %q", id)
 }
 
-func (a *AssetStore) addURL(url string, hash *hashing.Hash) error {
+func (a *AssetStore) addURLs(urls []string, hash *hashing.Hash) error {
-	var err error
+	if len(urls) == 0 {
+		return fmt.Errorf("no urls were specified")
+	}
 
+	var err error
 	if hash == nil {
-		hash, err = hashFromHttpHeader(url)
+		for _, url := range urls {
+			hash, err = hashFromHttpHeader(url)
+			if err != nil {
+				glog.Warningf("unable to get hash from %q: %v", url, err)
+				continue
+			} else {
+				break
+			}
+		}
 		if err != nil {
 			return err
 		}
 	}
 
-	localFile := path.Join(a.cacheDir, hash.String()+"_"+utils.SanitizeString(url))
+	// We assume the first url is the "main" url, and download to that _name_, wherever we get it from
-	_, err = DownloadURL(url, localFile, hash)
+	primaryURL := urls[0]
+	localFile := path.Join(a.cacheDir, hash.String()+"_"+utils.SanitizeString(primaryURL))
+
+	for _, url := range urls {
+		_, err = DownloadURL(url, localFile, hash)
+		if err != nil {
+			glog.Warningf("error downloading url %q: %v", url, err)
+			continue
+		} else {
+			break
+		}
+	}
 	if err != nil {
 		return err
 	}
 
-	key := path.Base(url)
+	key := path.Base(primaryURL)
-	assetPath := url
+	assetPath := primaryURL
 	r := NewFileResource(localFile)
 
-	source := &Source{URL: url, Hash: hash}
+	source := &Source{URL: primaryURL, Hash: hash}
 
 	asset := &asset{
 		Key:       key,

upup/pkg/fi/cloudup/apply_cluster.go

@@ -70,6 +70,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/vsphere"
 	"k8s.io/kops/upup/pkg/fi/cloudup/vspheretasks"
 	"k8s.io/kops/upup/pkg/fi/fitasks"
+	"k8s.io/kops/util/pkg/hashing"
 	"k8s.io/kops/util/pkg/vfs"
 )
 
@@ -121,7 +122,7 @@ type ApplyClusterCmd struct {
 	// Formats:
 	//  raw url: http://... or https://...
 	//  url with hash: <hex>@http://... or <hex>@https://...
-	Assets []string
+	Assets []*MirroredAsset
 
 	Clientset simple.Clientset
 
@@ -1140,27 +1141,39 @@ func (c *ApplyClusterCmd) AddFileAssets(assetBuilder *assets.AssetBuilder) error
 		if err != nil {
 			return err
 		}
-		c.Assets = append(c.Assets, hash.Hex()+"@"+u.String())
+		c.Assets = append(c.Assets, BuildMirroredAsset(u, hash))
 	}
 
 	if usesCNI(c.Cluster) {
-		cniAsset, cniAssetHashString, err := findCNIAssets(c.Cluster, assetBuilder)
+		cniAsset, cniAssetHash, err := findCNIAssets(c.Cluster, assetBuilder)
 		if err != nil {
 			return err
 		}
 
-		c.Assets = append(c.Assets, cniAssetHashString+"@"+cniAsset.String())
+		c.Assets = append(c.Assets, BuildMirroredAsset(cniAsset, cniAssetHash))
 	}
 
 	if c.Cluster.Spec.Networking.LyftVPC != nil {
-		lyftVPCDownloadURL := os.Getenv("LYFT_VPC_DOWNLOAD_URL")
+		var hash *hashing.Hash
-		if lyftVPCDownloadURL == "" {
+
-			lyftVPCDownloadURL = "bfdc65028a3bf8ffe14388fca28ede3600e7e2dee4e781908b6a23f9e79f86ad@https://github.com/lyft/cni-ipvlan-vpc-k8s/releases/download/v0.4.2/cni-ipvlan-vpc-k8s-v0.4.2.tar.gz"
+		urlString := os.Getenv("LYFT_VPC_DOWNLOAD_URL")
+		if urlString == "" {
+			urlString = "https://github.com/lyft/cni-ipvlan-vpc-k8s/releases/download/v0.4.2/cni-ipvlan-vpc-k8s-v0.4.2.tar.gz"
+			hash, err = hashing.FromString("bfdc65028a3bf8ffe14388fca28ede3600e7e2dee4e781908b6a23f9e79f86ad")
+			if err != nil {
+				// Should be impossible
+				return fmt.Errorf("invalid hard-coded hash for lyft url")
+			}
 		} else {
-			glog.Warningf("Using url from LYFT_VPC_DOWNLOAD_URL env var: %q", lyftVPCDownloadURL)
+			glog.Warningf("Using url from LYFT_VPC_DOWNLOAD_URL env var: %q", urlString)
 		}
 
-		c.Assets = append(c.Assets, lyftVPCDownloadURL)
+		u, err := url.Parse(urlString)
+		if err != nil {
+			return fmt.Errorf("unable to parse lyft-vpc URL %q", urlString)
+		}
+
+		c.Assets = append(c.Assets, BuildMirroredAsset(u, hash))
 	}
 
 	// TODO figure out if we can only do this for CoreOS only and GCE Container OS
@@ -1173,7 +1186,7 @@ func (c *ApplyClusterCmd) AddFileAssets(assetBuilder *assets.AssetBuilder) error
 		if err != nil {
 			return err
 		}
-		c.Assets = append(c.Assets, hash.Hex()+"@"+utilsLocation.String())
+		c.Assets = append(c.Assets, BuildMirroredAsset(utilsLocation, hash))
 	}
 
 	n, hash, err := NodeUpLocation(assetBuilder)
@@ -1265,7 +1278,9 @@ func (c *ApplyClusterCmd) BuildNodeUpConfig(assetBuilder *assets.AssetBuilder, i
 		config.Tags = append(config.Tags, tag)
 	}
 
-	config.Assets = c.Assets
+	for _, a := range c.Assets {
+		config.Assets = append(config.Assets, a.CompactString())
+	}
 	config.ClusterName = cluster.ObjectMeta.Name
 	config.ConfigBase = fi.String(configBase.Path())
 	config.InstanceGroupName = ig.ObjectMeta.Name

upup/pkg/fi/cloudup/networking.go

@@ -26,6 +26,7 @@ import (
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/util/pkg/hashing"
 )
 
 func usesCNI(c *api.Cluster) bool {
@@ -128,12 +129,12 @@ const (
 	ENV_VAR_CNI_ASSET_HASH_STRING = "CNI_ASSET_HASH_STRING"
 )
 
-func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL, string, error) {
+func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL, *hashing.Hash, error) {
 
 	if cniVersionURL := os.Getenv(ENV_VAR_CNI_VERSION_URL); cniVersionURL != "" {
 		u, err := url.Parse(cniVersionURL)
 		if err != nil {
-			return nil, "", fmt.Errorf("unable to parse %q as a URL: %v", cniVersionURL, err)
+			return nil, nil, fmt.Errorf("unable to parse %q as a URL: %v", cniVersionURL, err)
 		}
 
 		glog.Infof("Using CNI asset version %q, as set in %s", cniVersionURL, ENV_VAR_CNI_VERSION_URL)
@@ -142,15 +143,19 @@ func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL,
 
 			glog.Infof("Using CNI asset hash %q, as set in %s", cniAssetHashString, ENV_VAR_CNI_ASSET_HASH_STRING)
 
-			return u, cniAssetHashString, nil
+			hash, err := hashing.FromString(cniAssetHashString)
+			if err != nil {
+				return nil, nil, fmt.Errorf("unable to parse CNI asset hash %q", cniAssetHashString)
+			}
+			return u, hash, nil
 		} else {
-			return u, "", nil
+			return u, nil, nil
 		}
 	}
 
 	sv, err := util.ParseKubernetesVersion(c.Spec.KubernetesVersion)
 	if err != nil {
-		return nil, "", fmt.Errorf("failed to lookup kubernetes version: %v", err)
+		return nil, nil, fmt.Errorf("failed to lookup kubernetes version: %v", err)
 	}
 
 	sv.Pre = nil
@@ -173,13 +178,18 @@ func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL,
 
 	u, err := url.Parse(cniAsset)
 	if err != nil {
-		return nil, "", nil
+		return nil, nil, nil
+	}
+
+	hash, err := hashing.FromString(cniAssetHash)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to parse CNI asset hash %q", cniAssetHash)
 	}
 
 	u, err = assetBuilder.RemapFileAndSHAValue(u, cniAssetHash)
 	if err != nil {
-		return nil, "", err
+		return nil, nil, err
 	}
 
-	return u, cniAssetHash, nil
+	return u, hash, nil
 }

upup/pkg/fi/cloudup/networking_test.go

@@ -36,7 +36,7 @@ func Test_FindCNIAssetFromEnvironmentVariable(t *testing.T) {
 	cluster.Spec.KubernetesVersion = "v1.9.0"
 
 	assetBuilder := assets.NewAssetBuilder(cluster, "")
-	cniAsset, cniAssetHashString, err := findCNIAssets(cluster, assetBuilder)
+	cniAsset, cniAssetHash, err := findCNIAssets(cluster, assetBuilder)
 
 	if err != nil {
 		t.Errorf("Unable to parse k8s version %s", err)
@@ -46,8 +46,8 @@ func Test_FindCNIAssetFromEnvironmentVariable(t *testing.T) {
 		t.Errorf("Expected CNI version from Environment variable %q, but got %q instead", desiredCNIVersion, cniAsset)
 	}
 
-	if cniAssetHashString != "" {
+	if cniAssetHash != nil {
-		t.Errorf("Expected Empty CNI Version Hash String, but got %q instead", cniAssetHashString)
+		t.Errorf("Expected Empty CNI Version Hash String, but got %v instead", cniAssetHash)
 	}
 }
 
@@ -56,7 +56,7 @@ func Test_FindCNIAssetDefaultValue1_6(t *testing.T) {
 	cluster := &api.Cluster{}
 	cluster.Spec.KubernetesVersion = "v1.7.0"
 	assetBuilder := assets.NewAssetBuilder(cluster, "")
-	cniAsset, cniAssetHashString, err := findCNIAssets(cluster, assetBuilder)
+	cniAsset, cniAssetHash, err := findCNIAssets(cluster, assetBuilder)
 
 	if err != nil {
 		t.Errorf("Unable to parse k8s version %s", err)
@@ -66,8 +66,8 @@ func Test_FindCNIAssetDefaultValue1_6(t *testing.T) {
 		t.Errorf("Expected default CNI version %q and got %q", defaultCNIAssetK8s1_5, cniAsset)
 	}
 
-	if cniAssetHashString != defaultCNIAssetHashStringK8s1_6 {
+	if cniAssetHash.Hex() != defaultCNIAssetHashStringK8s1_6 {
-		t.Errorf("Expected default CNI Version Hash String %q and got %q", defaultCNIAssetHashStringK8s1_5, cniAssetHashString)
+		t.Errorf("Expected default CNI Version Hash String %q and got %v", defaultCNIAssetHashStringK8s1_5, cniAssetHash)
 	}
 
 }
@@ -77,7 +77,7 @@ func Test_FindCNIAssetDefaultValue1_5(t *testing.T) {
 	cluster := &api.Cluster{}
 	cluster.Spec.KubernetesVersion = "v1.5.12"
 	assetBuilder := assets.NewAssetBuilder(cluster, "")
-	cniAsset, cniAssetHashString, err := findCNIAssets(cluster, assetBuilder)
+	cniAsset, cniAssetHash, err := findCNIAssets(cluster, assetBuilder)
 
 	if err != nil {
 		t.Errorf("Unable to parse k8s version %s", err)
@@ -87,8 +87,8 @@ func Test_FindCNIAssetDefaultValue1_5(t *testing.T) {
 		t.Errorf("Expected default CNI version %q and got %q", defaultCNIAssetK8s1_5, cniAsset)
 	}
 
-	if cniAssetHashString != defaultCNIAssetHashStringK8s1_5 {
+	if cniAssetHash.Hex() != defaultCNIAssetHashStringK8s1_5 {
-		t.Errorf("Expected default CNI Version Hash String %q and got %q", defaultCNIAssetHashStringK8s1_5, cniAssetHashString)
+		t.Errorf("Expected default CNI Version Hash String %q and got %v", defaultCNIAssetHashStringK8s1_5, cniAssetHash)
 	}
 
 }

upup/pkg/fi/cloudup/urls.go

@@ -20,8 +20,8 @@ import (
 	"fmt"
 	"net/url"
 	"os"
-
 	"path"
+	"strings"
 
 	"github.com/golang/glog"
 	"k8s.io/kops"
@@ -31,6 +31,16 @@ import (
 
 const defaultKopsBaseUrl = "https://kubeupv2.s3.amazonaws.com/kops/%s/"
 
+// defaultKopsMirrorBase will be detected and automatically set to pull from the defaultKopsMirrors
+const defaultKopsMirrorBase = "https://kubeupv2.s3.amazonaws.com/kops/"
+
+// defaultKopsMirrors is a list of our well-known mirrors
+var defaultKopsMirrors = []string{
+	"https://github.com/kubernetes/kops/releases/download/",
+	// We do need to include defaultKopsMirrorBase - the list replaces the base url
+	"https://kubeupv2.s3.amazonaws.com/kops/",
+}
+
 var kopsBaseUrl *url.URL
 
 // nodeUpLocation caches the nodeUpLocation url
@@ -182,3 +192,48 @@ func KopsFileUrl(file string, assetBuilder *assets.AssetBuilder) (*url.URL, *has
 
 	return fileUrl, hash, nil
 }
+
+type MirroredAsset struct {
+	Locations []string
+	Hash      *hashing.Hash
+}
+
+// BuildMirroredAsset checks to see if this is a file under the standard base location, and if so constructs some mirror locations
+func BuildMirroredAsset(u *url.URL, hash *hashing.Hash) *MirroredAsset {
+	baseUrlString := defaultKopsMirrorBase
+	if !strings.HasSuffix(baseUrlString, "/") {
+		baseUrlString += "/"
+	}
+
+	a := &MirroredAsset{
+		Hash: hash,
+	}
+
+	urlString := u.String()
+	a.Locations = []string{urlString}
+
+	// Look at mirrors
+	if strings.HasPrefix(urlString, baseUrlString) {
+		if hash == nil {
+			glog.Warningf("not using mirrors for asset %s as it does not have a known hash", u.String())
+		} else {
+			suffix := strings.TrimPrefix(urlString, baseUrlString)
+			// This is under our base url - add our well-known mirrors
+			a.Locations = []string{}
+			for _, m := range defaultKopsMirrors {
+				a.Locations = append(a.Locations, m+suffix)
+			}
+		}
+	}
+
+	return a
+}
+
+func (a *MirroredAsset) CompactString() string {
+	var s string
+	if a.Hash != nil {
+		s = a.Hash.Hex()
+	}
+	s += "@" + strings.Join(a.Locations, ",")
+	return s
+}