Merge pull request #13060 from srikiz/DO-Add-New-VPC

[DigitalOcean] Implement new VPC if network-cidr flag is specified
This commit is contained in:
Kubernetes Prow Robot 2022-02-18 12:44:23 -08:00 committed by GitHub
commit e29591e21e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 430 additions and 27 deletions

View File

@ -73,6 +73,21 @@ kops create cluster --cloud=digitalocean --name=dev5.k8s.local --networking=cili
kops delete cluster dev5.k8s.local --yes
```
## VPC Support
If you already have a VPC created and want to run kops cluster in this vpc, specify the vpc uuid as below.
```bash
/kops create cluster --cloud=digitalocean --name=dev1.example.com --vpc=af287488-862e-46c7-a783-5e5fa89cb200 --networking=cilium --zones=tor1 --ssh-public-key=~/.ssh/id_rsa.pub
```
If you want to create a new VPC for running your kops cluster, specify the network-cidr as below.
```bash
./kops create cluster --cloud=digitalocean --name=dev1.example.com --networking=calico --network-cidr=192.168.11.0/24 --zones=nyc1 --ssh-public-key=~/.ssh/id_rsa.pub --yes
```
## Features Still in Development
kOps for DigitalOcean currently does not support these features:

View File

@ -68,6 +68,8 @@ It is recommended to keep using the `v1alpha2` API version.
* Fix inconsistent output of `kops get clusters -ojson`. This will now always return a list (irrespective of a single or multiple clusters) to keep the format consistent. However, note that `kops get cluster dev.example.com -ojson` will continue to work as previously, and will return a single object.
* Digital Ocean kops now has vpc support. You can specify a `network-cidr` range while creating the kops cluster. kops resources will be created in the new vpc range. Also supports shared vpc; you can specify the vpc uuid while creating kops cluster.
# Full change list since 1.22.0 release

View File

@ -76,9 +76,10 @@ type ClusterSpec struct {
MasterPublicName string `json:"masterPublicName,omitempty"`
// MasterInternalName is the internal DNS name for the master nodes
MasterInternalName string `json:"masterInternalName,omitempty"`
// NetworkCIDR is the CIDR used for the AWS VPC / GCE Network, or otherwise allocated to k8s
// NetworkCIDR is the CIDR used for the AWS VPC / DO/ GCE Network, or otherwise allocated to k8s
// This is a real CIDR, not the internal k8s network
// On AWS, it maps to the VPC CIDR. It is not required on GCE.
// On DO, it maps to the VPC CIDR.
NetworkCIDR string `json:"networkCIDR,omitempty"`
// AdditionalNetworkCIDRs is a list of additional CIDR used for the AWS VPC
// or otherwise allocated to k8s. This is a real CIDR, not the internal k8s network

View File

@ -73,9 +73,7 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
requiresSubnets = false
requiresSubnetCIDR = false
requiresNetworkCIDR = false
if c.Spec.NetworkCIDR != "" {
allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("networkCIDR"), "networkCIDR should not be set on DigitalOcean"))
}
case kops.CloudProviderAWS:
case kops.CloudProviderAzure:
case kops.CloudProviderOpenstack:
@ -135,6 +133,16 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
if err != nil {
allErrs = append(allErrs, field.Invalid(fieldSpec.Child("networkCIDR"), c.Spec.NetworkCIDR, "Cluster had an invalid networkCIDR"))
}
if kops.CloudProviderID(c.Spec.CloudProvider) == kops.CloudProviderDO {
// verify if the NetworkCIDR is in a private range as per RFC1918
if !networkCIDR.IP.IsPrivate() {
allErrs = append(allErrs, field.Invalid(fieldSpec.Child("networkCIDR"), c.Spec.NetworkCIDR, "Cluster had a networkCIDR outside the private IP range"))
}
// verify if networkID is not specified. In case of DO, this is mutually exclusive.
if c.Spec.NetworkID != "" {
allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("networkCIDR"), "DO doesn't support specifying both NetworkID and NetworkCIDR together"))
}
}
}
}

View File

@ -6,6 +6,7 @@ go_library(
"api_loadbalancer.go",
"context.go",
"droplets.go",
"network.go",
],
importpath = "k8s.io/kops/pkg/model/domodel",
visibility = ["//visibility:public"],

View File

@ -18,7 +18,6 @@ package domodel
import (
"fmt"
"strings"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/dns"
@ -56,7 +55,7 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.ModelBuilderContext) error {
return fmt.Errorf("unhandled LoadBalancer type %q", lbSpec.Type)
}
clusterName := strings.Replace(b.ClusterName(), ".", "-", -1)
clusterName := do.SafeClusterName(b.ClusterName())
loadbalancerName := "api-" + clusterName
clusterMasterTag := do.TagKubernetesClusterMasterPrefix + ":" + clusterName
@ -67,6 +66,15 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.ModelBuilderContext) error {
DropletTag: fi.String(clusterMasterTag),
Lifecycle: b.Lifecycle,
}
if b.Cluster.Spec.NetworkID != "" {
loadbalancer.VPCUUID = fi.String(b.Cluster.Spec.NetworkID)
} else if b.Cluster.Spec.NetworkCIDR != "" {
vpcName := "vpc-" + clusterName
loadbalancer.VPCName = fi.String(vpcName)
loadbalancer.NetworkCIDR = fi.String(b.Cluster.Spec.NetworkCIDR)
}
c.AddTask(loadbalancer)
// Temporarily do not know the role of the following function

View File

@ -46,8 +46,9 @@ func (d *DropletBuilder) Build(c *fi.ModelBuilderContext) error {
sshKeyFingerPrint := splitSSHKeyName[len(splitSSHKeyName)-1]
// replace "." with "-" since DO API does not accept "."
clusterTag := do.TagKubernetesClusterNamePrefix + ":" + strings.Replace(d.ClusterName(), ".", "-", -1)
clusterMasterTag := do.TagKubernetesClusterMasterPrefix + ":" + strings.Replace(d.ClusterName(), ".", "-", -1)
clusterName := do.SafeClusterName(d.ClusterName())
clusterTag := do.TagKubernetesClusterNamePrefix + ":" + clusterName
clusterMasterTag := do.TagKubernetesClusterMasterPrefix + ":" + clusterName
masterIndexCount := 0
// In the future, DigitalOcean will use Machine API to manage groups,
@ -81,7 +82,13 @@ func (d *DropletBuilder) Build(c *fi.ModelBuilderContext) error {
}
if d.Cluster.Spec.NetworkID != "" {
droplet.VPC = fi.String(d.Cluster.Spec.NetworkID)
droplet.VPCUUID = fi.String(d.Cluster.Spec.NetworkID)
} else if d.Cluster.Spec.NetworkCIDR != "" {
// since networkCIDR specified as part of the request, it is made sure that vpc with this cidr exist before
// creating the droplet, so you can associate with vpc uuid for this droplet.
vpcName := "vpc-" + clusterName
droplet.VPCName = fi.String(vpcName)
droplet.NetworkCIDR = fi.String(d.Cluster.Spec.NetworkCIDR)
}
userData, err := d.BootstrapScriptBuilder.ResourceNodeUp(c, ig)

View File

@ -0,0 +1,55 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package domodel
import (
"strings"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/dotasks"
)
// NetworkModelBuilder configures network objects
type NetworkModelBuilder struct {
*DOModelContext
Lifecycle fi.Lifecycle
}
var _ fi.ModelBuilder = &NetworkModelBuilder{}
func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
ipRange := b.Cluster.Spec.NetworkCIDR
if ipRange == "" {
// no cidr specified, use the default vpc in DO that's always available
return nil
}
clusterName := strings.Replace(b.ClusterName(), ".", "-", -1)
vpcName := "vpc-" + clusterName
// Create a separate vpc for this cluster.
vpc := &dotasks.VPC{
Name: fi.String(vpcName),
Region: fi.String(b.Cluster.Spec.Subnets[0].Region),
Lifecycle: b.Lifecycle,
IPRange: fi.String(ipRange),
}
c.AddTask(vpc)
return nil
}

View File

@ -40,6 +40,7 @@ const (
resourceTypeVolume = "volume"
resourceTypeDNSRecord = "dns-record"
resourceTypeLoadBalancer = "loadbalancer"
resourceTypeVPC = "vpc"
)
type listFn func(fi.Cloud, string) ([]*resources.Resource, error)
@ -52,6 +53,7 @@ func ListResources(cloud do.DOCloud, clusterName string) (map[string]*resources.
listDroplets,
listDNS,
listLoadBalancers,
listVPCs,
}
for _, fn := range listFunctions {
@ -261,6 +263,16 @@ func deleteDroplet(cloud fi.Cloud, t *resources.Resource) error {
return nil
}
func deleteVPC(cloud fi.Cloud, t *resources.Resource) error {
c := cloud.(do.DOCloud)
_, err := c.VPCsService().Delete(context.TODO(), t.ID)
if err != nil {
return fmt.Errorf("failed to delete VPC %s (ID %s): %s", t.Name, t.ID, err)
}
return nil
}
func deleteVolume(cloud fi.Cloud, t *resources.Resource) error {
c := cloud.(do.DOCloud)
volume := t.Obj.(godo.Volume)
@ -370,3 +382,32 @@ func dumpDroplet(op *resources.DumpOperation, r *resources.Resource) error {
return nil
}
func listVPCs(cloud fi.Cloud, clusterName string) ([]*resources.Resource, error) {
c := cloud.(do.DOCloud)
var resourceTrackers []*resources.Resource
clusterName = do.SafeClusterName(clusterName)
vpcName := "vpc-" + clusterName
vpcs, err := c.GetAllVPCs()
if err != nil {
return nil, fmt.Errorf("failed to list vpcs: %v", err)
}
for _, vpc := range vpcs {
if vpc.Name == vpcName {
resourceTracker := &resources.Resource{
Name: vpc.Name,
ID: vpc.ID,
Type: resourceTypeVPC,
Deleter: deleteVPC,
Obj: vpc,
}
resourceTrackers = append(resourceTrackers, resourceTracker)
}
}
return resourceTrackers, nil
}

View File

@ -585,6 +585,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
l.Builders = append(l.Builders,
&domodel.APILoadBalancerModelBuilder{DOModelContext: doModelContext, Lifecycle: securityLifecycle},
&domodel.DropletBuilder{DOModelContext: doModelContext, BootstrapScriptBuilder: bootstrapScriptBuilder, Lifecycle: clusterLifecycle},
&domodel.NetworkModelBuilder{DOModelContext: doModelContext, Lifecycle: networkLifecycle},
)
case kops.CloudProviderGCE:
gceModelContext := &gcemodel.GCEModelContext{

View File

@ -70,10 +70,13 @@ type DOCloud interface {
LoadBalancersService() godo.LoadBalancersService
DomainService() godo.DomainsService
ActionsService() godo.ActionsService
VPCsService() godo.VPCsService
FindClusterStatus(cluster *kops.Cluster) (*kops.ClusterStatus, error)
GetAllLoadBalancers() ([]godo.LoadBalancer, error)
GetAllDropletsByTag(tag string) ([]godo.Droplet, error)
GetAllVolumesByRegion() ([]godo.Volume, error)
GetVPCUUID(networkCIDR string, vpcName string) (string, error)
GetAllVPCs() ([]*godo.VPC, error)
}
var readBackoff = wait.Backoff{
@ -238,11 +241,44 @@ func (c *doCloudImplementation) ActionsService() godo.ActionsService {
return c.Client.Actions
}
func (c *doCloudImplementation) VPCsService() godo.VPCsService {
return c.Client.VPCs
}
// FindVPCInfo is not implemented, it's only here to satisfy the fi.Cloud interface
func (c *doCloudImplementation) FindVPCInfo(id string) (*fi.VPCInfo, error) {
return nil, errors.New("not implemented")
}
func (c *doCloudImplementation) GetVPCUUID(networkCIDR string, vpcName string) (string, error) {
vpcUUID := ""
done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) {
vpcs, err := c.GetAllVPCs()
if err != nil {
return false, err
}
for _, vpc := range vpcs {
if vpc.IPRange == networkCIDR && vpc.Name == vpcName {
vpcUUID = vpc.ID
return true, nil
}
}
return false, fmt.Errorf("vpc not yet created..")
})
if err != nil {
return "", err
}
if done {
return vpcUUID, nil
} else {
return "", wait.ErrWaitTimeout
}
}
func (c *doCloudImplementation) GetApiIngressStatus(cluster *kops.Cluster) ([]fi.ApiIngressStatus, error) {
var ingresses []fi.ApiIngressStatus
done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) {
@ -532,6 +568,33 @@ func (c *doCloudImplementation) GetAllLoadBalancers() ([]godo.LoadBalancer, erro
return allLoadBalancers, nil
}
func (c *doCloudImplementation) GetAllVPCs() ([]*godo.VPC, error) {
allVPCs := []*godo.VPC{}
opt := &godo.ListOptions{}
for {
vpcs, resp, err := c.VPCsService().List(context.TODO(), opt)
if err != nil {
return nil, err
}
allVPCs = append(allVPCs, vpcs...)
if resp.Links == nil || resp.Links.IsLastPage() {
break
}
page, err := resp.Links.CurrentPage()
if err != nil {
return nil, err
}
opt.Page = page + 1
}
return allVPCs, nil
}
func (c *doCloudImplementation) GetAllDropletsByTag(tag string) ([]godo.Droplet, error) {
allDroplets := []godo.Droplet{}

View File

@ -130,3 +130,15 @@ func (c *doCloudMockImplementation) GetAllDropletsByTag(tag string) ([]godo.Drop
func (c *doCloudMockImplementation) GetAllVolumesByRegion() ([]godo.Volume, error) {
return nil, nil
}
func (c *doCloudMockImplementation) GetVPCUUID(networkCIDR string, vpcName string) (string, error) {
return "", nil
}
func (c *doCloudMockImplementation) GetAllVPCs() ([]*godo.VPC, error) {
return nil, nil
}
func (c *doCloudMockImplementation) VPCsService() godo.VPCsService {
return c.Client.VPCs
}

View File

@ -9,6 +9,8 @@ go_library(
"loadbalancer_fitask.go",
"volume.go",
"volume_fitask.go",
"vpc.go",
"vpc_fitask.go",
],
importpath = "k8s.io/kops/upup/pkg/fi/cloudup/dotasks",
visibility = ["//visibility:public"],

View File

@ -19,10 +19,10 @@ package dotasks
import (
"context"
"errors"
"fmt"
"github.com/digitalocean/godo"
"k8s.io/klog/v2"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/do"
_ "k8s.io/kops/upup/pkg/fi/cloudup/terraform"
@ -39,7 +39,9 @@ type Droplet struct {
Size *string
Image *string
SSHKey *string
VPC *string
VPCUUID *string
NetworkCIDR *string
VPCName *string
Tags []string
Count int
UserData fi.Resource
@ -86,7 +88,7 @@ func (d *Droplet) Find(c *fi.Context) (*Droplet, error) {
Tags: foundDroplet.Tags,
SSHKey: d.SSHKey, // TODO: get from droplet or ignore change
UserData: d.UserData, // TODO: get from droplet or ignore change
VPC: fi.String(foundDroplet.VPCUUID),
VPCUUID: fi.String(foundDroplet.VPCUUID),
Lifecycle: d.Lifecycle,
}, nil
}
@ -147,6 +149,17 @@ func (_ *Droplet) RenderDO(t *do.DOAPITarget, a, e, changes *Droplet) error {
newDropletCount = expectedCount - actualCount
}
// associate vpcuuid to the droplet if set.
vpcUUID := ""
if fi.StringValue(e.NetworkCIDR) != "" {
vpcUUID, err = t.Cloud.GetVPCUUID(fi.StringValue(e.NetworkCIDR), fi.StringValue(e.VPCName))
if err != nil {
return fmt.Errorf("Error fetching vpcUUID from network cidr=%s", fi.StringValue(e.NetworkCIDR))
}
} else if fi.StringValue(e.VPCUUID) != "" {
vpcUUID = fi.StringValue(e.VPCUUID)
}
for i := 0; i < newDropletCount; i++ {
_, _, err = t.Cloud.DropletsService().Create(context.TODO(), &godo.DropletCreateRequest{
Name: fi.StringValue(e.Name),
@ -154,14 +167,13 @@ func (_ *Droplet) RenderDO(t *do.DOAPITarget, a, e, changes *Droplet) error {
Size: fi.StringValue(e.Size),
Image: godo.DropletCreateImage{Slug: fi.StringValue(e.Image)},
Tags: e.Tags,
VPCUUID: fi.StringValue(e.VPC),
VPCUUID: vpcUUID,
UserData: userData,
SSHKeys: []godo.DropletCreateSSHKey{{Fingerprint: fi.StringValue(e.SSHKey)}},
})
if err != nil {
klog.Errorf("Error creating droplet with Name=%s", fi.StringValue(e.Name))
return err
return fmt.Errorf("Error creating droplet with Name=%s", fi.StringValue(e.Name))
}
}

View File

@ -41,6 +41,9 @@ type LoadBalancer struct {
Region *string
DropletTag *string
IPAddress *string
VPCUUID *string
VPCName *string
NetworkCIDR *string
ForAPIServer bool
}
@ -78,6 +81,7 @@ func (lb *LoadBalancer) Find(c *fi.Context) (*LoadBalancer, error) {
Name: fi.String(loadbalancer.Name),
ID: fi.String(loadbalancer.ID),
Region: fi.String(loadbalancer.Region.Slug),
VPCUUID: fi.String(loadbalancer.VPCUUID),
// Ignore system fields
Lifecycle: lb.Lifecycle,
@ -154,17 +158,28 @@ func (_ *LoadBalancer) RenderDO(t *do.DOAPITarget, a, e, changes *LoadBalancer)
}
}
// associate vpcuuid to the loadbalancer if set
vpcUUID := ""
if fi.StringValue(e.NetworkCIDR) != "" {
vpcUUID, err = t.Cloud.GetVPCUUID(fi.StringValue(e.NetworkCIDR), fi.StringValue(e.VPCName))
if err != nil {
return fmt.Errorf("Error fetching vpcUUID from network cidr=%s", fi.StringValue(e.NetworkCIDR))
}
} else if fi.StringValue(e.VPCUUID) != "" {
vpcUUID = fi.StringValue(e.VPCUUID)
}
loadBalancerService := t.Cloud.LoadBalancersService()
loadbalancer, _, err := loadBalancerService.Create(context.TODO(), &godo.LoadBalancerRequest{
Name: fi.StringValue(e.Name),
Region: fi.StringValue(e.Region),
Tag: fi.StringValue(e.DropletTag),
VPCUUID: vpcUUID,
ForwardingRules: Rules,
HealthCheck: HealthCheck,
})
if err != nil {
klog.Errorf("Error creating load balancer with Name=%s, Error=%v", fi.StringValue(e.Name), err)
return err
return fmt.Errorf("Error creating load balancer with Name=%s, Error=%v", fi.StringValue(e.Name), err)
}
e.ID = fi.String(loadbalancer.ID)

View File

@ -0,0 +1,108 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package dotasks
import (
"context"
"github.com/digitalocean/godo"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/do"
)
// +kops:fitask
type VPC struct {
Name *string
ID *string
Lifecycle fi.Lifecycle
IPRange *string
Region *string
}
var _ fi.CompareWithID = &VPC{}
func (v *VPC) CompareWithID() *string {
return v.ID
}
func (v *VPC) Find(c *fi.Context) (*VPC, error) {
cloud := c.Cloud.(do.DOCloud)
vpcService := cloud.VPCsService()
opt := &godo.ListOptions{}
vpcs, _, err := vpcService.List(context.TODO(), opt)
if err != nil {
return nil, err
}
for _, vpc := range vpcs {
if vpc.Name == fi.StringValue(v.Name) {
return &VPC{
Name: fi.String(vpc.Name),
ID: fi.String(vpc.ID),
Lifecycle: v.Lifecycle,
IPRange: fi.String(vpc.IPRange),
Region: fi.String(vpc.RegionSlug),
}, nil
}
}
// VPC = nil if not found
return nil, nil
}
func (v *VPC) Run(c *fi.Context) error {
return fi.DefaultDeltaRunMethod(v, c)
}
func (_ *VPC) CheckChanges(a, e, changes *VPC) error {
if a != nil {
if changes.Name != nil {
return fi.CannotChangeField("Name")
}
if changes.ID != nil {
return fi.CannotChangeField("ID")
}
if changes.Region != nil {
return fi.CannotChangeField("Region")
}
} else {
if e.Name == nil {
return fi.RequiredField("Name")
}
if e.Region == nil {
return fi.RequiredField("Region")
}
}
return nil
}
func (_ *VPC) RenderDO(t *do.DOAPITarget, a, e, changes *VPC) error {
if a != nil {
return nil
}
vpcService := t.Cloud.VPCsService()
_, _, err := vpcService.Create(context.TODO(), &godo.VPCCreateRequest{
Name: fi.StringValue(e.Name),
RegionSlug: fi.StringValue(e.Region),
IPRange: fi.StringValue(e.IPRange),
})
return err
}

View File

@ -0,0 +1,52 @@
//go:build !ignore_autogenerated
// +build !ignore_autogenerated
/*
Copyright The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by fitask. DO NOT EDIT.
package dotasks
import (
"k8s.io/kops/upup/pkg/fi"
)
// VPC
var _ fi.HasLifecycle = &VPC{}
// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
func (o *VPC) GetLifecycle() fi.Lifecycle {
return o.Lifecycle
}
// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
func (o *VPC) SetLifecycle(lifecycle fi.Lifecycle) {
o.Lifecycle = lifecycle
}
var _ fi.HasName = &VPC{}
// GetName returns the Name of the object, implementing fi.HasName
func (o *VPC) GetName() *string {
return o.Name
}
// String is the stringer function for the task, producing readable output using fi.TaskAsString
func (o *VPC) String() string {
return fi.TaskAsString(o)
}