kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: refactor MirroredAsset into assets 

This seems logically coherent, and is part of the work to start using
compiled-in hashes for most well-known assets.
This commit is contained in:
justinsb 2024-03-30 10:34:27 -04:00
parent fdd3f0e5b7
commit 192b880434
13 changed files with 49 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
clusterapi/bootstrap/controllers/kopsconfig_controller.go
View File

@ -39,7 +39,6 @@ import (
"k8s.io/kops/pkg/wellknownservices"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/mirrors"
"k8s.io/kops/util/pkg/vfs"
"k8s.io/utils/pointer"
ctrl "sigs.k8s.io/controller-runtime"
@ -212,7 +211,7 @@ func (r *KopsConfigReconciler) buildBootstrapData(ctx context.Context) ([]byte,
// encryptionConfigSecretHash = base64.URLEncoding.EncodeToString(hashBytes[:])
// }
nodeUpAssets := make(map[architectures.Architecture]*mirrors.MirroredAsset)
nodeUpAssets := make(map[architectures.Architecture]*assets.MirroredAsset)
for _, arch := range architectures.GetSupported() {
asset, err := wellknownassets.NodeUpAsset(assetBuilder, arch)
if err != nil {
@ -221,7 +220,7 @@ func (r *KopsConfigReconciler) buildBootstrapData(ctx context.Context) ([]byte,
nodeUpAssets[arch] = asset
}
assets := make(map[architectures.Architecture][]*mirrors.MirroredAsset)
assets := make(map[architectures.Architecture][]*assets.MirroredAsset)
configBuilder, err := nodemodel.NewNodeUpConfigBuilder(cluster, assetBuilder, assets, encryptionConfigSecretHash)
if err != nil {
return nil, err

3
pkg/assets/builder.go
View File

@ -36,7 +36,6 @@ import (
"k8s.io/kops/pkg/kubemanifest"
"k8s.io/kops/pkg/values"
"k8s.io/kops/util/pkg/hashing"
"k8s.io/kops/util/pkg/mirrors"
"k8s.io/kops/util/pkg/vfs"
)
@ -337,7 +336,7 @@ func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {
}
for _, ext := range []string{".sha256", ".sha256sum"} {
for _, mirror := range mirrors.FindURLMirrors(u.String()) {
for _, mirror := range FindURLMirrors(u.String()) {
hashURL := mirror + ext
klog.V(3).Infof("Trying to read hash fie: %q", hashURL)
b, err := a.vfsContext.ReadFile(hashURL, vfs.WithBackoff(backoff))

2
util/pkg/mirrors/mirrored_asset.go → pkg/assets/mirrored_asset.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package mirrors
package assets
import (
"net/url"

2
util/pkg/mirrors/mirrors.go → pkg/assets/mirrors.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package mirrors
package assets
import (
"strings"

5
pkg/commands/toolbox_enroll.go
View File

@ -55,7 +55,6 @@ import (
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/mirrors"
"k8s.io/kops/util/pkg/vfs"
)
@ -424,7 +423,7 @@ func buildBootstrapData(ctx context.Context, clientset simple.Clientset, cluster
// encryptionConfigSecretHash = base64.URLEncoding.EncodeToString(hashBytes[:])
// }
nodeUpAssets := make(map[architectures.Architecture]*mirrors.MirroredAsset)
nodeUpAssets := make(map[architectures.Architecture]*assets.MirroredAsset)
for _, arch := range architectures.GetSupported() {
asset, err := wellknownassets.NodeUpAsset(assetBuilder, arch)
if err != nil {
@ -433,7 +432,7 @@ func buildBootstrapData(ctx context.Context, clientset simple.Clientset, cluster
nodeUpAssets[arch] = asset
}
assets := make(map[architectures.Architecture][]*mirrors.MirroredAsset)
assets := make(map[architectures.Architecture][]*assets.MirroredAsset)
configBuilder, err := nodemodel.NewNodeUpConfigBuilder(cluster, assetBuilder, assets, encryptionConfigSecretHash)
if err != nil {
return nil, err

4
pkg/model/bootstrapscript.go
View File

@ -29,6 +29,7 @@ import (
"k8s.io/klog/v2"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/nodeup"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/pkg/model/resources"
"k8s.io/kops/pkg/wellknownservices"
"k8s.io/kops/upup/pkg/fi"
@ -37,7 +38,6 @@ import (
"k8s.io/kops/upup/pkg/fi/fitasks"
"k8s.io/kops/upup/pkg/fi/utils"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/mirrors"
)
type NodeUpConfigBuilder interface {
@ -51,7 +51,7 @@ type WellKnownAddresses map[wellknownservices.WellKnownService][]string
type BootstrapScriptBuilder struct {
*KopsModelContext
Lifecycle fi.Lifecycle
NodeUpAssets map[architectures.Architecture]*mirrors.MirroredAsset
NodeUpAssets map[architectures.Architecture]*assets.MirroredAsset
NodeUpConfigBuilder NodeUpConfigBuilder
}

4
pkg/model/bootstrapscript_test.go
View File

@ -26,13 +26,13 @@ import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/nodeup"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/pkg/model/iam"
"k8s.io/kops/pkg/testutils/golden"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/fitasks"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/hashing"
"k8s.io/kops/util/pkg/mirrors"
)
func Test_ProxyFunc(t *testing.T) {
@ -170,7 +170,7 @@ func TestBootstrapUserData(t *testing.T) {
InstanceGroups: []*kops.InstanceGroup{group},
},
NodeUpConfigBuilder: &nodeupConfigBuilder{cluster: cluster},
NodeUpAssets: map[architectures.Architecture]*mirrors.MirroredAsset{
NodeUpAssets: map[architectures.Architecture]*assets.MirroredAsset{
architectures.ArchitectureAmd64: {
Locations: []string{"nodeup-amd64-1", "nodeup-amd64-2"},
Hash: hashing.MustFromString("833723369ad345a88dd85d61b1e77336d56e61b864557ded71b92b6e34158e6a"),

4
pkg/model/openstackmodel/servergroup_test.go
View File

@ -24,6 +24,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/nodeup"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/pkg/model"
"k8s.io/kops/pkg/model/iam"
"k8s.io/kops/pkg/testutils"
@ -31,7 +32,6 @@ import (
"k8s.io/kops/upup/pkg/fi/fitasks"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/hashing"
"k8s.io/kops/util/pkg/mirrors"
)
type serverGroupModelBuilderTestInput struct {
@ -1573,7 +1573,7 @@ func RunGoldenTest(t *testing.T, basedir string, testCase serverGroupModelBuilde
InstanceGroups: testCase.instanceGroups,
},
NodeUpConfigBuilder: &nodeupConfigBuilder{},
NodeUpAssets: map[architectures.Architecture]*mirrors.MirroredAsset{
NodeUpAssets: map[architectures.Architecture]*assets.MirroredAsset{
architectures.ArchitectureAmd64: {
Locations: []string{"nodeup-amd64-1", "nodeup-amd64-2"},
Hash: hashing.MustFromString("833723369ad345a88dd85d61b1e77336d56e61b864557ded71b92b6e34158e6a"),

4
pkg/model/resources/nodeup.go
View File

@ -29,10 +29,10 @@ import (
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/nodeup"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/utils"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/mirrors"
)
var nodeUpTemplate = `#!/bin/bash
@ -176,7 +176,7 @@ echo "== nodeup node config done =="
// NodeUpScript is responsible for creating the nodeup script
type NodeUpScript struct {
NodeUpAssets map[architectures.Architecture]*mirrors.MirroredAsset
NodeUpAssets map[architectures.Architecture]*assets.MirroredAsset
BootConfig *nodeup.BootConfig
CompressUserData bool
SetSysctls string

27
pkg/nodemodel/fileassets.go
View File

@ -30,7 +30,6 @@ import (
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/hashing"
"k8s.io/kops/util/pkg/mirrors"
)
type FileAssets struct {
@ -38,10 +37,10 @@ type FileAssets struct {
// Formats:
// raw url: http://... or https://...
// url with hash: <hex>@http://... or <hex>@https://...
Assets map[architectures.Architecture][]*mirrors.MirroredAsset
Assets map[architectures.Architecture][]*assets.MirroredAsset
// NodeUpAssets are the assets for downloading nodeup
NodeUpAssets map[architectures.Architecture]*mirrors.MirroredAsset
NodeUpAssets map[architectures.Architecture]*assets.MirroredAsset
Cluster *kops.Cluster
}
@ -55,10 +54,10 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
baseURL = "https://dl.k8s.io/release/v" + c.Cluster.Spec.KubernetesVersion
}
c.Assets = make(map[architectures.Architecture][]*mirrors.MirroredAsset)
c.NodeUpAssets = make(map[architectures.Architecture]*mirrors.MirroredAsset)
c.Assets = make(map[architectures.Architecture][]*assets.MirroredAsset)
c.NodeUpAssets = make(map[architectures.Architecture]*assets.MirroredAsset)
for _, arch := range architectures.GetSupported() {
c.Assets[arch] = []*mirrors.MirroredAsset{}
c.Assets[arch] = []*assets.MirroredAsset{}
k8sAssetsNames := []string{
fmt.Sprintf("/bin/linux/%s/kubelet", arch),
@ -80,7 +79,7 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(u, hash))
}
kubernetesVersion, _ := util.ParseKubernetesVersion(c.Cluster.Spec.KubernetesVersion)
@ -112,7 +111,7 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(u, hash))
case kops.CloudProviderAWS:
binaryLocation := c.Cluster.Spec.CloudProvider.AWS.BinariesLocation
if binaryLocation == nil {
@ -128,7 +127,7 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(u, hash))
}
}
@ -137,7 +136,7 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(cniAsset, cniAssetHash))
c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(cniAsset, cniAssetHash))
}
if c.Cluster.Spec.Containerd == nil || !c.Cluster.Spec.Containerd.SkipInstall {
@ -146,7 +145,7 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
return err
}
if containerdAssetUrl != nil && containerdAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(containerdAssetUrl, containerdAssetHash))
c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(containerdAssetUrl, containerdAssetHash))
}
runcAssetUrl, runcAssetHash, err := wellknownassets.FindRuncAsset(c.Cluster, assetBuilder, arch)
@ -154,14 +153,14 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
return err
}
if runcAssetUrl != nil && runcAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(runcAssetUrl, runcAssetHash))
c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(runcAssetUrl, runcAssetHash))
}
nerdctlAssetUrl, nerdctlAssetHash, err := wellknownassets.FindNerdctlAsset(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
if nerdctlAssetUrl != nil && nerdctlAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(nerdctlAssetUrl, nerdctlAssetHash))
c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(nerdctlAssetUrl, nerdctlAssetHash))
}
}
@ -170,7 +169,7 @@ func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
return err
}
if crictlAssetUrl != nil && crictlAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(crictlAssetUrl, crictlAssetHash))
c.Assets[arch] = append(c.Assets[arch], assets.BuildMirroredAsset(crictlAssetUrl, crictlAssetHash))
}
asset, err := wellknownassets.NodeUpAsset(assetBuilder, arch)

15
pkg/nodemodel/nodeupconfigbuilder.go
View File

@ -37,7 +37,6 @@ import (
"k8s.io/kops/pkg/wellknownservices"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/mirrors"
"k8s.io/kops/util/pkg/vfs"
)
@ -46,7 +45,7 @@ type nodeUpConfigBuilder struct {
// Formats:
// raw url: http://... or https://...
// url with hash: <hex>@http://... or <hex>@https://...
assets map[architectures.Architecture][]*mirrors.MirroredAsset
assets map[architectures.Architecture][]*assets.MirroredAsset
assetBuilder *assets.AssetBuilder
channels []string
@ -54,12 +53,12 @@ type nodeUpConfigBuilder struct {
cluster *kops.Cluster
etcdManifests map[string][]string
images map[kops.InstanceGroupRole]map[architectures.Architecture][]*nodeup.Image
protokubeAsset map[architectures.Architecture][]*mirrors.MirroredAsset
channelsAsset map[architectures.Architecture][]*mirrors.MirroredAsset
protokubeAsset map[architectures.Architecture][]*assets.MirroredAsset
channelsAsset map[architectures.Architecture][]*assets.MirroredAsset
encryptionConfigSecretHash string
}
func NewNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBuilder, assets map[architectures.Architecture][]*mirrors.MirroredAsset, encryptionConfigSecretHash string) (model.NodeUpConfigBuilder, error) {
func NewNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBuilder, nodeAssets map[architectures.Architecture][]*assets.MirroredAsset, encryptionConfigSecretHash string) (model.NodeUpConfigBuilder, error) {
configBase, err := vfs.Context.BuildVfsPath(cluster.Spec.ConfigStore.Base)
if err != nil {
return nil, fmt.Errorf("error parsing configStore.base %q: %v", cluster.Spec.ConfigStore.Base, err)
@ -75,8 +74,8 @@ func NewNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui
etcdManifests := map[string][]string{}
images := map[kops.InstanceGroupRole]map[architectures.Architecture][]*nodeup.Image{}
protokubeAsset := map[architectures.Architecture][]*mirrors.MirroredAsset{}
channelsAsset := map[architectures.Architecture][]*mirrors.MirroredAsset{}
protokubeAsset := map[architectures.Architecture][]*assets.MirroredAsset{}
channelsAsset := map[architectures.Architecture][]*assets.MirroredAsset{}
for _, arch := range architectures.GetSupported() {
asset, err := wellknownassets.ProtokubeAsset(assetBuilder, arch)
@ -194,7 +193,7 @@ func NewNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui
configBuilder := nodeUpConfigBuilder{
assetBuilder: assetBuilder,
assets: assets,
assets: nodeAssets,
channels: channels,
configBase: configBase,
cluster: cluster,

25
pkg/nodemodel/wellknownassets/kopsassets.go
View File

@ -27,7 +27,6 @@ import (
"k8s.io/kops/pkg/assets"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/hashing"
"k8s.io/kops/util/pkg/mirrors"
)
const (
@ -37,13 +36,13 @@ const (
var kopsBaseURL *url.URL
// nodeUpAsset caches the nodeup binary download url/hash
var nodeUpAsset map[architectures.Architecture]*mirrors.MirroredAsset
var nodeUpAsset map[architectures.Architecture]*assets.MirroredAsset
// protokubeAsset caches the protokube binary download url/hash
var protokubeAsset map[architectures.Architecture]*mirrors.MirroredAsset
var protokubeAsset map[architectures.Architecture]*assets.MirroredAsset
// channelsAsset caches the channels binary download url/hash
var channelsAsset map[architectures.Architecture]*mirrors.MirroredAsset
var channelsAsset map[architectures.Architecture]*assets.MirroredAsset
// BaseURL returns the base url for the distribution of kops - in particular for nodeup & docker images
func BaseURL() (*url.URL, error) {
@ -84,9 +83,9 @@ func copyBaseURL(base *url.URL) (*url.URL, error) {
}
// NodeUpAsset returns the asset for where nodeup should be downloaded
func NodeUpAsset(assetsBuilder *assets.AssetBuilder, arch architectures.Architecture) (*mirrors.MirroredAsset, error) {
func NodeUpAsset(assetsBuilder *assets.AssetBuilder, arch architectures.Architecture) (*assets.MirroredAsset, error) {
if nodeUpAsset == nil {
nodeUpAsset = make(map[architectures.Architecture]*mirrors.MirroredAsset)
nodeUpAsset = make(map[architectures.Architecture]*assets.MirroredAsset)
}
if nodeUpAsset[arch] != nil {
// Avoid repeated logging
@ -98,16 +97,16 @@ func NodeUpAsset(assetsBuilder *assets.AssetBuilder, arch architectures.Architec
if err != nil {
return nil, err
}
nodeUpAsset[arch] = mirrors.BuildMirroredAsset(u, hash)
nodeUpAsset[arch] = assets.BuildMirroredAsset(u, hash)
klog.V(8).Infof("Using default nodeup location for %s: %q", arch, u.String())
return nodeUpAsset[arch], nil
}
// ProtokubeAsset returns the url and hash of the protokube binary
func ProtokubeAsset(assetsBuilder *assets.AssetBuilder, arch architectures.Architecture) (*mirrors.MirroredAsset, error) {
func ProtokubeAsset(assetsBuilder *assets.AssetBuilder, arch architectures.Architecture) (*assets.MirroredAsset, error) {
if protokubeAsset == nil {
protokubeAsset = make(map[architectures.Architecture]*mirrors.MirroredAsset)
protokubeAsset = make(map[architectures.Architecture]*assets.MirroredAsset)
}
if protokubeAsset[arch] != nil {
klog.V(8).Infof("Using cached protokube binary location for %s: %v", arch, protokubeAsset[arch].Locations)
@ -118,16 +117,16 @@ func ProtokubeAsset(assetsBuilder *assets.AssetBuilder, arch architectures.Archi
if err != nil {
return nil, err
}
protokubeAsset[arch] = mirrors.BuildMirroredAsset(u, hash)
protokubeAsset[arch] = assets.BuildMirroredAsset(u, hash)
klog.V(8).Infof("Using default protokube location for %s: %q", arch, u.String())
return protokubeAsset[arch], nil
}
// ChannelsAsset returns the url and hash of the channels binary
func ChannelsAsset(assetsBuilder *assets.AssetBuilder, arch architectures.Architecture) (*mirrors.MirroredAsset, error) {
func ChannelsAsset(assetsBuilder *assets.AssetBuilder, arch architectures.Architecture) (*assets.MirroredAsset, error) {
if channelsAsset == nil {
channelsAsset = make(map[architectures.Architecture]*mirrors.MirroredAsset)
channelsAsset = make(map[architectures.Architecture]*assets.MirroredAsset)
}
if channelsAsset[arch] != nil {
klog.V(8).Infof("Using cached channels binary location for %s: %v", arch, channelsAsset[arch].Locations)
@ -138,7 +137,7 @@ func ChannelsAsset(assetsBuilder *assets.AssetBuilder, arch architectures.Archit
if err != nil {
return nil, err
}
channelsAsset[arch] = mirrors.BuildMirroredAsset(u, hash)
channelsAsset[arch] = assets.BuildMirroredAsset(u, hash)
klog.V(8).Infof("Using default channels location for %s: %q", arch, u.String())
return channelsAsset[arch], nil

4
pkg/nodemodel/wellknownassets/kopsassets_test.go
View File

@ -23,8 +23,8 @@ import (
"testing"
"k8s.io/kops"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/util/pkg/hashing"
"k8s.io/kops/util/pkg/mirrors"
)
func Test_BuildMirroredAsset(t *testing.T) {
@ -56,7 +56,7 @@ func Test_BuildMirroredAsset(t *testing.T) {
t.Errorf("cannot parse URL: %s", fmt.Sprintf(tc.url, kops.Version))
return
}
actual := mirrors.BuildMirroredAsset(u, h)
actual := assets.BuildMirroredAsset(u, h)
if !reflect.DeepEqual(actual.Locations, tc.expected) {
t.Errorf("Locations differ:\nActual: %+v\nExpect: %+v", actual.Locations, tc.expected)