kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Simple mirror support 

We recognize our primary location via string-matching, and we then
have a hard-coded list of mirrors for that location.

Didn't prove easy to make this much better, but we can hopefully do so
iteratively (e.g. fetch mirrors via URL)
This commit is contained in:
Justin SB 2019-01-30 18:19:28 -05:00
parent 823f769a95
commit 8b9969e01c
No known key found for this signature in database
GPG Key ID: 8DEC5C8217494E37
5 changed files with 142 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
upup/pkg/fi/assetstore.go
View File

@ -154,45 +154,67 @@ func hashFromHttpHeader(url string) (*hashing.Hash, error) {
// Add an asset into the store, in one of the recognized formats (see Assets in types package)
func (a *AssetStore) Add(id string) error {
if strings.HasPrefix(id, "http://") || strings.HasPrefix(id, "https://") {
return a.addURL(id, nil)
return a.addURLs(strings.Split(id, ","), nil)
}
i := strings.Index(id, "@http://")
if i == -1 {
i = strings.Index(id, "@https://")
}
if i != -1 {
url := id[i+1:]
urls := strings.Split(id[i+1:], ",")
hash, err := hashing.FromString(id[:i])
if err != nil {
return err
}
return a.addURL(url, hash)
return a.addURLs(urls, hash)
}
// TODO: local files!
return fmt.Errorf("unknown asset format: %q", id)
}
func (a *AssetStore) addURL(url string, hash *hashing.Hash) error {
var err error
func (a *AssetStore) addURLs(urls []string, hash *hashing.Hash) error {
if len(urls) == 0 {
return fmt.Errorf("no urls were specified")
}
var err error
if hash == nil {
hash, err = hashFromHttpHeader(url)
for _, url := range urls {
hash, err = hashFromHttpHeader(url)
if err != nil {
glog.Warningf("unable to get hash from %q: %v", url, err)
continue
} else {
break
}
}
if err != nil {
return err
}
}
localFile := path.Join(a.cacheDir, hash.String()+"_"+utils.SanitizeString(url))
_, err = DownloadURL(url, localFile, hash)
// We assume the first url is the "main" url, and download to that _name_, wherever we get it from
primaryURL := urls[0]
localFile := path.Join(a.cacheDir, hash.String()+"_"+utils.SanitizeString(primaryURL))
for _, url := range urls {
_, err = DownloadURL(url, localFile, hash)
if err != nil {
glog.Warningf("error downloading url %q: %v", url, err)
continue
} else {
break
}
}
if err != nil {
return err
}
key := path.Base(url)
assetPath := url
key := path.Base(primaryURL)
assetPath := primaryURL
r := NewFileResource(localFile)
source := &Source{URL: url, Hash: hash}
source := &Source{URL: primaryURL, Hash: hash}
asset := &asset{
Key: key,

37
upup/pkg/fi/cloudup/apply_cluster.go
View File

@ -70,6 +70,7 @@ import (
"k8s.io/kops/upup/pkg/fi/cloudup/vsphere"
"k8s.io/kops/upup/pkg/fi/cloudup/vspheretasks"
"k8s.io/kops/upup/pkg/fi/fitasks"
"k8s.io/kops/util/pkg/hashing"
"k8s.io/kops/util/pkg/vfs"
)
@ -121,7 +122,7 @@ type ApplyClusterCmd struct {
// Formats:
// raw url: http://... or https://...
// url with hash: <hex>@http://... or <hex>@https://...
Assets []string
Assets []*MirroredAsset
Clientset simple.Clientset
@ -1140,27 +1141,39 @@ func (c *ApplyClusterCmd) AddFileAssets(assetBuilder *assets.AssetBuilder) error
if err != nil {
return err
}
c.Assets = append(c.Assets, hash.Hex()+"@"+u.String())
c.Assets = append(c.Assets, BuildMirroredAsset(u, hash))
}
if usesCNI(c.Cluster) {
cniAsset, cniAssetHashString, err := findCNIAssets(c.Cluster, assetBuilder)
cniAsset, cniAssetHash, err := findCNIAssets(c.Cluster, assetBuilder)
if err != nil {
return err
}
c.Assets = append(c.Assets, cniAssetHashString+"@"+cniAsset.String())
c.Assets = append(c.Assets, BuildMirroredAsset(cniAsset, cniAssetHash))
}
if c.Cluster.Spec.Networking.LyftVPC != nil {
lyftVPCDownloadURL := os.Getenv("LYFT_VPC_DOWNLOAD_URL")
if lyftVPCDownloadURL == "" {
lyftVPCDownloadURL = "bfdc65028a3bf8ffe14388fca28ede3600e7e2dee4e781908b6a23f9e79f86ad@https://github.com/lyft/cni-ipvlan-vpc-k8s/releases/download/v0.4.2/cni-ipvlan-vpc-k8s-v0.4.2.tar.gz"
var hash *hashing.Hash
urlString := os.Getenv("LYFT_VPC_DOWNLOAD_URL")
if urlString == "" {
urlString = "https://github.com/lyft/cni-ipvlan-vpc-k8s/releases/download/v0.4.2/cni-ipvlan-vpc-k8s-v0.4.2.tar.gz"
hash, err = hashing.FromString("bfdc65028a3bf8ffe14388fca28ede3600e7e2dee4e781908b6a23f9e79f86ad")
if err != nil {
// Should be impossible
return fmt.Errorf("invalid hard-coded hash for lyft url")
}
} else {
glog.Warningf("Using url from LYFT_VPC_DOWNLOAD_URL env var: %q", lyftVPCDownloadURL)
glog.Warningf("Using url from LYFT_VPC_DOWNLOAD_URL env var: %q", urlString)
}
c.Assets = append(c.Assets, lyftVPCDownloadURL)
u, err := url.Parse(urlString)
if err != nil {
return fmt.Errorf("unable to parse lyft-vpc URL %q", urlString)
}
c.Assets = append(c.Assets, BuildMirroredAsset(u, hash))
}
// TODO figure out if we can only do this for CoreOS only and GCE Container OS
@ -1173,7 +1186,7 @@ func (c *ApplyClusterCmd) AddFileAssets(assetBuilder *assets.AssetBuilder) error
if err != nil {
return err
}
c.Assets = append(c.Assets, hash.Hex()+"@"+utilsLocation.String())
c.Assets = append(c.Assets, BuildMirroredAsset(utilsLocation, hash))
}
n, hash, err := NodeUpLocation(assetBuilder)
@ -1265,7 +1278,9 @@ func (c *ApplyClusterCmd) BuildNodeUpConfig(assetBuilder *assets.AssetBuilder, i
config.Tags = append(config.Tags, tag)
}
config.Assets = c.Assets
for _, a := range c.Assets {
config.Assets = append(config.Assets, a.CompactString())
}
config.ClusterName = cluster.ObjectMeta.Name
config.ConfigBase = fi.String(configBase.Path())
config.InstanceGroupName = ig.ObjectMeta.Name

26
upup/pkg/fi/cloudup/networking.go
View File

@ -26,6 +26,7 @@ import (
api "k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/kops/util"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/util/pkg/hashing"
)
func usesCNI(c *api.Cluster) bool {
@ -128,12 +129,12 @@ const (
ENV_VAR_CNI_ASSET_HASH_STRING = "CNI_ASSET_HASH_STRING"
)
func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL, string, error) {
func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL, *hashing.Hash, error) {
if cniVersionURL := os.Getenv(ENV_VAR_CNI_VERSION_URL); cniVersionURL != "" {
u, err := url.Parse(cniVersionURL)
if err != nil {
return nil, "", fmt.Errorf("unable to parse %q as a URL: %v", cniVersionURL, err)
return nil, nil, fmt.Errorf("unable to parse %q as a URL: %v", cniVersionURL, err)
}
glog.Infof("Using CNI asset version %q, as set in %s", cniVersionURL, ENV_VAR_CNI_VERSION_URL)
@ -142,15 +143,19 @@ func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL,
glog.Infof("Using CNI asset hash %q, as set in %s", cniAssetHashString, ENV_VAR_CNI_ASSET_HASH_STRING)
return u, cniAssetHashString, nil
hash, err := hashing.FromString(cniAssetHashString)
if err != nil {
return nil, nil, fmt.Errorf("unable to parse CNI asset hash %q", cniAssetHashString)
}
return u, hash, nil
} else {
return u, "", nil
return u, nil, nil
}
}
sv, err := util.ParseKubernetesVersion(c.Spec.KubernetesVersion)
if err != nil {
return nil, "", fmt.Errorf("failed to lookup kubernetes version: %v", err)
return nil, nil, fmt.Errorf("failed to lookup kubernetes version: %v", err)
}
sv.Pre = nil
@ -173,13 +178,18 @@ func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL,
u, err := url.Parse(cniAsset)
if err != nil {
return nil, "", nil
return nil, nil, nil
}
hash, err := hashing.FromString(cniAssetHash)
if err != nil {
return nil, nil, fmt.Errorf("unable to parse CNI asset hash %q", cniAssetHash)
}
u, err = assetBuilder.RemapFileAndSHAValue(u, cniAssetHash)
if err != nil {
return nil, "", err
return nil, nil, err
}
return u, cniAssetHash, nil
return u, hash, nil
}

18
upup/pkg/fi/cloudup/networking_test.go
View File

@ -36,7 +36,7 @@ func Test_FindCNIAssetFromEnvironmentVariable(t *testing.T) {
cluster.Spec.KubernetesVersion = "v1.9.0"
assetBuilder := assets.NewAssetBuilder(cluster, "")
cniAsset, cniAssetHashString, err := findCNIAssets(cluster, assetBuilder)
cniAsset, cniAssetHash, err := findCNIAssets(cluster, assetBuilder)
if err != nil {
t.Errorf("Unable to parse k8s version %s", err)
@ -46,8 +46,8 @@ func Test_FindCNIAssetFromEnvironmentVariable(t *testing.T) {
t.Errorf("Expected CNI version from Environment variable %q, but got %q instead", desiredCNIVersion, cniAsset)
}
if cniAssetHashString != "" {
t.Errorf("Expected Empty CNI Version Hash String, but got %q instead", cniAssetHashString)
if cniAssetHash != nil {
t.Errorf("Expected Empty CNI Version Hash String, but got %v instead", cniAssetHash)
}
}
@ -56,7 +56,7 @@ func Test_FindCNIAssetDefaultValue1_6(t *testing.T) {
cluster := &api.Cluster{}
cluster.Spec.KubernetesVersion = "v1.7.0"
assetBuilder := assets.NewAssetBuilder(cluster, "")
cniAsset, cniAssetHashString, err := findCNIAssets(cluster, assetBuilder)
cniAsset, cniAssetHash, err := findCNIAssets(cluster, assetBuilder)
if err != nil {
t.Errorf("Unable to parse k8s version %s", err)
@ -66,8 +66,8 @@ func Test_FindCNIAssetDefaultValue1_6(t *testing.T) {
t.Errorf("Expected default CNI version %q and got %q", defaultCNIAssetK8s1_5, cniAsset)
}
if cniAssetHashString != defaultCNIAssetHashStringK8s1_6 {
t.Errorf("Expected default CNI Version Hash String %q and got %q", defaultCNIAssetHashStringK8s1_5, cniAssetHashString)
if cniAssetHash.Hex() != defaultCNIAssetHashStringK8s1_6 {
t.Errorf("Expected default CNI Version Hash String %q and got %v", defaultCNIAssetHashStringK8s1_5, cniAssetHash)
}
}
@ -77,7 +77,7 @@ func Test_FindCNIAssetDefaultValue1_5(t *testing.T) {
cluster := &api.Cluster{}
cluster.Spec.KubernetesVersion = "v1.5.12"
assetBuilder := assets.NewAssetBuilder(cluster, "")
cniAsset, cniAssetHashString, err := findCNIAssets(cluster, assetBuilder)
cniAsset, cniAssetHash, err := findCNIAssets(cluster, assetBuilder)
if err != nil {
t.Errorf("Unable to parse k8s version %s", err)
@ -87,8 +87,8 @@ func Test_FindCNIAssetDefaultValue1_5(t *testing.T) {
t.Errorf("Expected default CNI version %q and got %q", defaultCNIAssetK8s1_5, cniAsset)
}
if cniAssetHashString != defaultCNIAssetHashStringK8s1_5 {
t.Errorf("Expected default CNI Version Hash String %q and got %q", defaultCNIAssetHashStringK8s1_5, cniAssetHashString)
if cniAssetHash.Hex() != defaultCNIAssetHashStringK8s1_5 {
t.Errorf("Expected default CNI Version Hash String %q and got %v", defaultCNIAssetHashStringK8s1_5, cniAssetHash)
}
}

57
upup/pkg/fi/cloudup/urls.go
View File

@ -20,8 +20,8 @@ import (
"fmt"
"net/url"
"os"
"path"
"strings"
"github.com/golang/glog"
"k8s.io/kops"
@ -31,6 +31,16 @@ import (
const defaultKopsBaseUrl = "https://kubeupv2.s3.amazonaws.com/kops/%s/"
// defaultKopsMirrorBase will be detected and automatically set to pull from the defaultKopsMirrors
const defaultKopsMirrorBase = "https://kubeupv2.s3.amazonaws.com/kops/"
// defaultKopsMirrors is a list of our well-known mirrors
var defaultKopsMirrors = []string{
"https://github.com/kubernetes/kops/releases/download/",
// We do need to include defaultKopsMirrorBase - the list replaces the base url
"https://kubeupv2.s3.amazonaws.com/kops/",
}
var kopsBaseUrl *url.URL
// nodeUpLocation caches the nodeUpLocation url
@ -182,3 +192,48 @@ func KopsFileUrl(file string, assetBuilder *assets.AssetBuilder) (*url.URL, *has
return fileUrl, hash, nil
}
type MirroredAsset struct {
Locations []string
Hash *hashing.Hash
}
// BuildMirroredAsset checks to see if this is a file under the standard base location, and if so constructs some mirror locations
func BuildMirroredAsset(u *url.URL, hash *hashing.Hash) *MirroredAsset {
baseUrlString := defaultKopsMirrorBase
if !strings.HasSuffix(baseUrlString, "/") {
baseUrlString += "/"
}
a := &MirroredAsset{
Hash: hash,
}
urlString := u.String()
a.Locations = []string{urlString}
// Look at mirrors
if strings.HasPrefix(urlString, baseUrlString) {
if hash == nil {
glog.Warningf("not using mirrors for asset %s as it does not have a known hash", u.String())
} else {
suffix := strings.TrimPrefix(urlString, baseUrlString)
// This is under our base url - add our well-known mirrors
a.Locations = []string{}
for _, m := range defaultKopsMirrors {
a.Locations = append(a.Locations, m+suffix)
}
}
}
return a
}
func (a *MirroredAsset) CompactString() string {
var s string
if a.Hash != nil {
s = a.Hash.Hex()
}
s += "@" + strings.Join(a.Locations, ",")
return s
}