kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

metal: stub out functions to enable cluster creation 

Start adding the minimal implementation such that we can `kops create cluster`
This commit is contained in:
justinsb 2024-08-29 22:32:50 -04:00
parent d269ac2994
commit 7f58570a04
15 changed files with 289 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nodeup/pkg/model/bootstrap_client.go
View File

@ -94,7 +94,7 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
}
authenticator = a
case "metal":
case kops.CloudProviderMetal:
a, err := pkibootstrap.NewAuthenticatorFromFile("/etc/kubernetes/kops/pki/machine/private.pem")
if err != nil {
return err

2
pkg/model/components/apiserver.go
View File

@ -112,6 +112,8 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(cluster *kops.Cluster) error
c.CloudProvider = "azure"
case kops.CloudProviderScaleway:
c.CloudProvider = "external"
case kops.CloudProviderMetal:
c.CloudProvider = "external"
default:
return fmt.Errorf("unknown cloudprovider %q", cluster.GetCloudProvider())
}

5
pkg/model/components/etcdmanager/model.go
View File

@ -525,6 +525,11 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
fmt.Sprintf("%s=%s", scaleway.TagNameRolePrefix, scaleway.TagRoleControlPlane),
}
config.VolumeNameTag = fmt.Sprintf("%s=%s", scaleway.TagInstanceGroup, instanceGroupName)
case kops.CloudProviderMetal:
config.VolumeProvider = "external"
// TODO: Use static configuration here?
default:
return nil, fmt.Errorf("CloudProvider %q not supported with etcd-manager", b.Cluster.GetCloudProvider())
}

4
pkg/model/master_volumes.go
View File

@ -122,6 +122,10 @@ func (b *MasterVolumeBuilder) Build(c *fi.CloudupModelBuilderContext) error {
}
case kops.CloudProviderScaleway:
b.addScalewayVolume(c, name, volumeSize, zone, etcd, m, allMembers)
case kops.CloudProviderMetal:
// Nothing special to do for Metal (yet)
default:
return fmt.Errorf("unknown cloudprovider %q", b.Cluster.GetCloudProvider())
}

12
tests/e2e/scenarios/bare-metal/run-test
View File

@ -61,7 +61,17 @@ export S3_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
# Create the state-store bucket in our mock s3 server
export KOPS_STATE_STORE=s3://kops-state-store/
aws --version
aws --endpoint-url=${S3_ENDPOINT} --debug s3 mb s3://kops-state-store
aws --endpoint-url=${S3_ENDPOINT} s3 mb s3://kops-state-store
# List clusters (there should not be any yet)
go run ./cmd/kops get cluster || true
# Create a cluster
go run ./cmd/kops create cluster --cloud=metal metal.k8s.local --zones main
# List clusters
go run ./cmd/kops get cluster
# List instance groups
go run ./cmd/kops get ig --name metal.k8s.local
go run ./cmd/kops get ig --name metal.k8s.local -oyaml

70
tools/metal/storage/main.go
View File

@ -22,9 +22,9 @@ import (
"flag"
"fmt"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/kubernetes/kops/tools/metal/dhcp/pkg/objectstore"
"github.com/kubernetes/kops/tools/metal/dhcp/pkg/objectstore/testobjectstore"
@ -90,7 +90,7 @@ func (s *S3Server) ListAllMyBuckets(ctx context.Context, req *s3Request, r *List
for _, bucket := range s.store.ListBuckets(ctx) {
output.Buckets = append(output.Buckets, s3model.Bucket{
CreationDate: bucket.CreationDate.Format(time.RFC3339),
CreationDate: bucket.CreationDate.Format(s3TimeFormat),
Name: bucket.Name,
})
}
@ -107,6 +107,11 @@ func (s *S3Server) ServeRequest(ctx context.Context, w http.ResponseWriter, r *h
tokens := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/")
values, err := url.ParseQuery(r.URL.RawQuery)
if err != nil {
return fmt.Errorf("failed to parse query: %w", err)
}
req := &s3Request{
w: w,
r: r,
@ -121,7 +126,9 @@ func (s *S3Server) ServeRequest(ctx context.Context, w http.ResponseWriter, r *h
switch r.Method {
case http.MethodGet:
return s.ListObjectsV2(ctx, req, &ListObjectsV2Input{
Bucket: bucket,
Bucket: bucket,
Delimiter: values.Get("delimiter"),
Prefix: values.Get("prefix"),
})
case http.MethodPut:
return s.CreateBucket(ctx, req, &CreateBucketInput{
@ -136,10 +143,22 @@ func (s *S3Server) ServeRequest(ctx context.Context, w http.ResponseWriter, r *h
if len(tokens) > 1 {
bucket := tokens[0]
return s.GetObject(ctx, req, &GetObjectInput{
Bucket: bucket,
Key: strings.TrimPrefix(r.URL.Path, "/"+bucket+"/"),
})
key := strings.TrimPrefix(r.URL.Path, "/"+bucket+"/")
switch r.Method {
case http.MethodGet:
return s.GetObject(ctx, req, &GetObjectInput{
Bucket: bucket,
Key: key,
})
case http.MethodPut:
return s.PutObject(ctx, req, &PutObjectInput{
Bucket: bucket,
Key: key,
})
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return nil
}
}
return fmt.Errorf("unhandled path %q", r.URL.Path)
@ -147,8 +166,13 @@ func (s *S3Server) ServeRequest(ctx context.Context, w http.ResponseWriter, r *h
type ListObjectsV2Input struct {
Bucket string
Delimiter string
Prefix string
}
const s3TimeFormat = "2006-01-02T15:04:05.000Z"
func (s *S3Server) ListObjectsV2(ctx context.Context, req *s3Request, input *ListObjectsV2Input) error {
bucket, err := s.store.GetBucket(ctx, input.Bucket)
if err != nil {
@ -168,12 +192,17 @@ func (s *S3Server) ListObjectsV2(ctx context.Context, req *s3Request, input *Lis
}
for _, object := range objects {
if input.Prefix != "" && !strings.HasPrefix(object.Key, input.Prefix) {
continue
}
// TODO: support delimiter
output.Contents = append(output.Contents, s3model.Object{
Key: object.Key,
LastModified: object.LastModified.Format(time.RFC3339),
LastModified: object.LastModified.Format(s3TimeFormat),
Size: object.Size,
})
}
output.KeyCount = len(output.Contents)
return req.writeXML(ctx, output)
}
@ -232,6 +261,31 @@ func (s *S3Server) GetObject(ctx context.Context, req *s3Request, input *GetObje
return object.WriteTo(req.w)
}
type PutObjectInput struct {
Bucket string
Key string
}
func (s *S3Server) PutObject(ctx context.Context, req *s3Request, input *PutObjectInput) error {
log := klog.FromContext(ctx)
bucket, err := s.store.GetBucket(ctx, input.Bucket)
if err != nil {
return fmt.Errorf("failed to get bucket %q: %w", input.Bucket, err)
}
if bucket == nil {
return req.writeError(ctx, http.StatusNotFound, nil)
}
objectInfo, err := bucket.PutObject(ctx, input.Key, req.r.Body)
if err != nil {
return fmt.Errorf("failed to create object %q in bucket %q: %w", input.Key, input.Bucket, err)
}
log.Info("object created", "object", objectInfo)
return nil
}
type s3Request struct {
Action string
Version string

4
tools/metal/storage/pkg/objectstore/interfaces.go
View File

@ -18,6 +18,7 @@ package objectstore
import (
"context"
"io"
"net/http"
"time"
)
@ -44,6 +45,9 @@ type Bucket interface {
// If the object does not exist, it returns (nil, nil).
GetObject(ctx context.Context, key string) (Object, error)
// PutObject creates the object with the given key.
PutObject(ctx context.Context, key string, r io.Reader) (*ObjectInfo, error)
// ListObjects returns the list of objects in the bucket.
ListObjects(ctx context.Context) ([]ObjectInfo, error)
}

24
tools/metal/storage/pkg/objectstore/testobjectstore/testobjectstore.go
View File

@ -18,6 +18,8 @@ package testobjectstore
import (
"context"
"fmt"
"io"
"net/http"
"sync"
"time"
@ -119,6 +121,28 @@ func (m *TestBucket) GetObject(ctx context.Context, key string) (objectstore.Obj
return obj, nil
}
func (m *TestBucket) PutObject(ctx context.Context, key string, r io.Reader) (*objectstore.ObjectInfo, error) {
m.mutex.Lock()
defer m.mutex.Unlock()
b, err := io.ReadAll(r)
if err != nil {
return nil, fmt.Errorf("reading data: %w", err)
}
info := objectstore.ObjectInfo{
Key: key,
LastModified: time.Now().UTC(),
Size: int64(len(b)),
}
m.objects[key] = &TestObject{
data: b,
info: info,
}
return &info, nil
}
type TestObject struct {
data []byte
info objectstore.ObjectInfo

20
tools/metal/storage/pkg/s3model/api.go
View File

@ -49,14 +49,14 @@ type ListBucketResult struct {
}
type Object struct {
ChecksumAlgorithm string `xml:"ChecksumAlgorithm"`
ETag string `xml:"ETag"`
Key string `xml:"Key"`
LastModified string `xml:"LastModified"`
Owner Owner `xml:"Owner"`
RestoreStatus RestoreStatus `xml:"RestoreStatus"`
Size int64 `xml:"Size"`
StorageClass string `xml:"StorageClass"`
ChecksumAlgorithm string `xml:"ChecksumAlgorithm"`
ETag string `xml:"ETag"`
Key string `xml:"Key"`
LastModified string `xml:"LastModified"`
Owner *Owner `xml:"Owner"`
RestoreStatus *RestoreStatus `xml:"RestoreStatus"`
Size int64 `xml:"Size"`
StorageClass string `xml:"StorageClass"`
}
type Owner struct {
DisplayName string `xml:"DisplayName"`
@ -64,6 +64,6 @@ type Owner struct {
}
type RestoreStatus struct {
IsRestoreInProgress bool `xml:"IsRestoreInProgress"`
RestoreExpiryDate string `xml:"RestoreExpiryDate"`
IsRestoreInProgress bool `xml:"IsRestoreInProgress"`
RestoreExpiryDate *string `xml:"RestoreExpiryDate"`
}

6
upup/pkg/fi/cloudup/apply_cluster.go
View File

@ -482,6 +482,9 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) {
scwZone = scwCloud.Zone()
}
case kops.CloudProviderMetal:
// Metal is a special case, we don't need to do anything here (yet)
default:
return nil, fmt.Errorf("unknown CloudProvider %q", cluster.GetCloudProvider())
}
@ -686,6 +689,9 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) {
&scalewaymodel.SSHKeyModelBuilder{ScwModelContext: scwModelContext, Lifecycle: securityLifecycle},
)
case kops.CloudProviderMetal:
// No special builders for bare metal (yet)
default:
return nil, fmt.Errorf("unknown cloudprovider %q", cluster.GetCloudProvider())
}

50
upup/pkg/fi/cloudup/metal/api_target.go Normal file
View File

@ -0,0 +1,50 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metal
import (
"k8s.io/klog/v2"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
)
type APITarget struct {
Cloud *Cloud
OtherClouds []fi.Cloud
}
var _ fi.CloudupTarget = &APITarget{}
func NewAPITarget(cloud *Cloud, otherClouds []fi.Cloud) *APITarget {
return &APITarget{
Cloud: cloud,
OtherClouds: otherClouds,
}
}
func (t *APITarget) GetAWSCloud() awsup.AWSCloud {
klog.Fatalf("cannot find instance of AWSCloud in context")
return nil
}
func (t *APITarget) Finish(taskMap map[string]fi.CloudupTask) error {
return nil
}
func (t *APITarget) DefaultCheckExisting() bool {
return true
}

92
upup/pkg/fi/cloudup/metal/cloud.go Normal file
View File

@ -0,0 +1,92 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metal
import (
"fmt"
v1 "k8s.io/api/core/v1"
"k8s.io/kops/dnsprovider/pkg/dnsprovider"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/cloudinstances"
"k8s.io/kops/upup/pkg/fi"
)
var _ fi.Cloud = &Cloud{}
// Cloud holds the fi.Cloud implementation for metal resources.
type Cloud struct {
}
// NewCloud returns a Cloud for metal resources.
func NewCloud() (*Cloud, error) {
cloud := &Cloud{}
return cloud, nil
}
func (c *Cloud) ProviderID() kops.CloudProviderID {
return kops.CloudProviderMetal
}
func (c *Cloud) DNS() (dnsprovider.Interface, error) {
return nil, fmt.Errorf("method not implemented")
}
// FindVPCInfo looks up the specified VPC by id, returning info if found, otherwise (nil, nil).
func (c *Cloud) FindVPCInfo(id string) (*fi.VPCInfo, error) {
return nil, fmt.Errorf("method not implemented")
}
// DeleteInstance deletes a cloud instance.
func (c *Cloud) DeleteInstance(instance *cloudinstances.CloudInstance) error {
return fmt.Errorf("method not implemented")
}
// // DeregisterInstance drains a cloud instance and loadbalancers.
func (c *Cloud) DeregisterInstance(instance *cloudinstances.CloudInstance) error {
return fmt.Errorf("method not implemented")
}
// DeleteGroup deletes the cloud resources that make up a CloudInstanceGroup, including the instances.
func (c *Cloud) DeleteGroup(group *cloudinstances.CloudInstanceGroup) error {
return fmt.Errorf("method not implemented")
}
// DetachInstance causes a cloud instance to no longer be counted against the group's size limits.
func (c *Cloud) DetachInstance(instance *cloudinstances.CloudInstance) error {
return fmt.Errorf("method not implemented")
}
// GetCloudGroups returns a map of cloud instances that back a kops cluster.
// Detached instances must be returned in the NeedUpdate slice.
func (c *Cloud) GetCloudGroups(cluster *kops.Cluster, instancegroups []*kops.InstanceGroup, warnUnmatched bool, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) {
return nil, fmt.Errorf("method not implemented")
}
// Region returns the cloud region bound to the cloud instance.
// If the region concept does not apply, returns "".
func (c *Cloud) Region() string {
return ""
}
// FindClusterStatus discovers the status of the cluster, by inspecting the cloud objects
func (c *Cloud) FindClusterStatus(cluster *kops.Cluster) (*kops.ClusterStatus, error) {
return nil, fmt.Errorf("method metal.Cloud::FindClusterStatus not implemented")
}
func (c *Cloud) GetApiIngressStatus(cluster *kops.Cluster) ([]fi.ApiIngressStatus, error) {
return nil, fmt.Errorf("method metal.Cloud::GetApiIngressStatus not implemented")
}

8
upup/pkg/fi/cloudup/new_cluster.go
View File

@ -359,7 +359,11 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
cloud = osCloud
case api.CloudProviderScaleway:
cluster.Spec.CloudProvider.Scaleway = &api.ScalewaySpec{}
case api.CloudProviderMetal:
if cluster.Labels == nil {
cluster.Labels = make(map[string]string)
}
cluster.Labels[api.AlphaLabelCloudProvider] = string(api.CloudProviderMetal)
default:
return nil, fmt.Errorf("unsupported cloud provider %s", opt.CloudProvider)
}
@ -1652,6 +1656,8 @@ func defaultImage(cluster *api.Cluster, channel *api.Channel, architecture archi
return defaultHetznerImageJammy, nil
case api.CloudProviderScaleway:
return defaultScalewayImageJammy, nil
case api.CloudProviderMetal:
return "dummy-metal-image", nil
}
}

4
upup/pkg/fi/cloudup/template_functions.go
View File

@ -762,6 +762,10 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) {
ClusterName: tf.ClusterName(),
}
case kops.CloudProviderMetal:
// Use crypto public/private keys for Metal
config.Server.PKI = &pkibootstrap.Options{}
default:
return "", fmt.Errorf("unsupported cloud provider %s", cluster.GetCloudProvider())
}

7
upup/pkg/fi/cloudup/utils.go
View File

@ -31,6 +31,7 @@ import (
"k8s.io/kops/upup/pkg/fi/cloudup/do"
"k8s.io/kops/upup/pkg/fi/cloudup/gce"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
"k8s.io/kops/upup/pkg/fi/cloudup/metal"
"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
)
@ -193,6 +194,12 @@ func BuildCloud(cluster *kops.Cluster) (fi.Cloud, error) {
cloud = scwCloud
}
case kops.CloudProviderMetal:
metalCloud, err := metal.NewCloud()
if err != nil {
return nil, fmt.Errorf("error initializing Metal cloud: %w", err)
}
cloud = metalCloud
default:
return nil, fmt.Errorf("unknown CloudProvider %q", cluster.GetCloudProvider())
}