kops/upup/pkg/fi/cloudup/openstack/cloud_test.go

779 lines
20 KiB
Go

/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"slices"
"sort"
"testing"
"github.com/gophercloud/gophercloud/v2"
"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers"
"github.com/gophercloud/gophercloud/v2/openstack/loadbalancer/v2/loadbalancers"
l3floatingips "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/floatingips"
"github.com/gophercloud/gophercloud/v2/openstack/networking/v2/ports"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kops/cloudmock/openstack/mocknetworking"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/util/pkg/vfs"
)
func Test_OpenstackCloud_MakeCloud(t *testing.T) {
baseCloudConfigWithBlockStorage := []string{
"auth-url=\"\"",
"username=\"\"",
"password=\"\"",
"region=\"\"",
"tenant-id=\"\"",
"tenant-name=\"\"",
"domain-name=\"\"",
"domain-id=\"\"",
"application-credential-id=\"\"",
"application-credential-secret=\"\"",
"",
"[BlockStorage]",
"bs-version=",
"ignore-volume-az=false",
}
tests := []struct {
desc string
cluster *kops.Cluster
expectedCloudConfig []string
}{
{
desc: "Ignore volume microversion is set to false when not configured",
cluster: &kops.Cluster{
Spec: kops.ClusterSpec{
CloudProvider: kops.CloudProviderSpec{
Openstack: &kops.OpenstackSpec{
BlockStorage: &kops.OpenstackBlockStorageConfig{},
},
},
},
},
expectedCloudConfig: append(baseCloudConfigWithBlockStorage,
"ignore-volume-microversion=false",
"",
),
},
{
desc: "Ignore volume microversion is set to configured value",
cluster: &kops.Cluster{
Spec: kops.ClusterSpec{
CloudProvider: kops.CloudProviderSpec{
Openstack: &kops.OpenstackSpec{
BlockStorage: &kops.OpenstackBlockStorageConfig{
IgnoreVolumeMicroVersion: fi.PtrTo(true),
},
},
},
},
},
expectedCloudConfig: append(baseCloudConfigWithBlockStorage,
"ignore-volume-microversion=true",
"",
),
},
}
for _, testCase := range tests {
t.Run(testCase.desc, func(t *testing.T) {
actualCloudConfig := MakeCloudConfig(testCase.cluster.Spec.CloudProvider.Openstack)
if !reflect.DeepEqual(actualCloudConfig, testCase.expectedCloudConfig) {
t.Errorf("Ingress status differ: expected\n%+#v\n\tgot:\n%+#v\n", testCase.expectedCloudConfig, actualCloudConfig)
}
})
}
}
func Test_OpenstackCloud_GetApiIngressStatus(t *testing.T) {
tests := []struct {
desc string
cluster *kops.Cluster
loadbalancers []loadbalancers.LoadBalancer
l3FloatingIPs []l3floatingips.FloatingIP
instances serverList
cloudFloatingEnabled bool
expectedAPIIngress []fi.ApiIngressStatus
expectedError error
}{
{
desc: "Loadbalancer configured master public name set",
cluster: &kops.Cluster{
Spec: kops.ClusterSpec{
API: kops.APISpec{
PublicName: "master.k8s.local",
},
CloudProvider: kops.CloudProviderSpec{
Openstack: &kops.OpenstackSpec{
Loadbalancer: &kops.OpenstackLoadbalancerConfig{},
},
},
},
},
loadbalancers: []loadbalancers.LoadBalancer{
{
ID: "lb_id",
Name: "name",
VipAddress: "10.1.2.3",
VipPortID: "vip_port_id",
VipSubnetID: "vip_subnet_id",
VipNetworkID: "vip_network_id",
},
},
l3FloatingIPs: []l3floatingips.FloatingIP{
{
ID: "id",
FixedIP: "10.1.2.3",
PortID: "vip_port_id",
FloatingIP: "8.8.8.8",
},
},
expectedAPIIngress: []fi.ApiIngressStatus{
{
IP: "8.8.8.8",
},
},
},
{
desc: "Loadbalancer configured master public name set multiple IPs match",
cluster: &kops.Cluster{
Spec: kops.ClusterSpec{
API: kops.APISpec{
PublicName: "master.k8s.local",
},
CloudProvider: kops.CloudProviderSpec{
Openstack: &kops.OpenstackSpec{
Loadbalancer: &kops.OpenstackLoadbalancerConfig{},
},
},
},
},
loadbalancers: []loadbalancers.LoadBalancer{
{
ID: "lb_id",
Name: "master.k8s.local",
VipAddress: "10.1.2.3",
VipPortID: "vip_port_id",
VipSubnetID: "vip_subnet_id",
VipNetworkID: "vip_network_id",
},
},
l3FloatingIPs: []l3floatingips.FloatingIP{
{
ID: "cluster",
FixedIP: "10.1.2.3",
PortID: "vip_port_id",
FloatingIP: "8.8.8.8",
},
{
ID: "something_else",
FixedIP: "192.168.2.3",
PortID: "xx_id",
FloatingIP: "2.2.2.2",
},
{
ID: "yet_another",
FixedIP: "10.1.2.3",
PortID: "yy_id",
FloatingIP: "9.9.9.9",
},
},
expectedAPIIngress: []fi.ApiIngressStatus{
{IP: "8.8.8.8"},
},
},
{
desc: "Loadbalancer configured master public name not set",
cluster: &kops.Cluster{
Spec: kops.ClusterSpec{
CloudProvider: kops.CloudProviderSpec{
Openstack: &kops.OpenstackSpec{
Loadbalancer: &kops.OpenstackLoadbalancerConfig{},
},
},
},
},
expectedAPIIngress: nil,
},
{
desc: "No Loadbalancer configured no floating enabled",
cluster: &kops.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster.k8s.local",
},
Spec: kops.ClusterSpec{
CloudProvider: kops.CloudProviderSpec{
Openstack: &kops.OpenstackSpec{},
},
},
},
instances: []servers.Server{
{
ID: "master1_no_lb_no_floating",
Metadata: map[string]string{
"k8s": "cluster.k8s.local",
"KopsRole": "ControlPlane",
},
Addresses: map[string]interface{}{
"1": []Address{
{Addr: "1.2.3.4"},
{Addr: "2.3.4.5"},
},
"2": []Address{
{Addr: "3.4.5.6"},
{Addr: "4.5.6.7"},
},
},
},
{
ID: "master2_no_lb_no_floating",
Metadata: map[string]string{
"k8s": "cluster.k8s.local",
"KopsRole": "ControlPlane",
},
Addresses: map[string]interface{}{
"1": []Address{
{Addr: "10.20.30.40"},
{Addr: "20.30.40.50"},
},
"2": []Address{
{Addr: "30.40.50.60"},
{Addr: "40.50.60.70"},
},
},
},
{
ID: "node_no_lb_no_floating",
Metadata: map[string]string{
"k8s": "cluster.k8s.local",
"KopsRole": "Node",
},
Addresses: map[string]interface{}{
"1": []Address{
{Addr: "110.120.130.140", IPType: "floating"},
{Addr: "120.130.140.150", IPType: "private"},
},
},
},
},
expectedAPIIngress: []fi.ApiIngressStatus{
{IP: "1.2.3.4"},
{IP: "2.3.4.5"},
{IP: "3.4.5.6"},
{IP: "4.5.6.7"},
{IP: "10.20.30.40"},
{IP: "20.30.40.50"},
{IP: "30.40.50.60"},
{IP: "40.50.60.70"},
},
},
{
desc: "No Loadbalancer configured with floating enabled",
cluster: &kops.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster.k8s.local",
},
Spec: kops.ClusterSpec{
CloudProvider: kops.CloudProviderSpec{
Openstack: &kops.OpenstackSpec{},
},
},
},
instances: []servers.Server{
{
ID: "master1_no_lb_floating",
Metadata: map[string]string{
"k8s": "cluster.k8s.local",
"KopsRole": "ControlPlane",
},
Addresses: map[string]interface{}{
"1": []map[string]interface{}{
{"Addr": "1.2.3.4", "OS-EXT-IPS:type": "floating"},
{"Addr": "2.3.4.5", "OS-EXT-IPS:type": "private"},
},
"2": []map[string]string{
{"Addr": "3.4.5.6", "OS-EXT-IPS:type": "private"},
{"Addr": "4.5.6.7", "OS-EXT-IPS:type": "floating"},
},
},
},
{
ID: "master2_no_lb_floating",
Metadata: map[string]string{
"k8s": "cluster.k8s.local",
"KopsRole": "ControlPlane",
},
Addresses: map[string]interface{}{
"1": []map[string]string{
{"Addr": "10.20.30.40", "OS-EXT-IPS:type": "private"},
{"Addr": "20.30.40.50", "OS-EXT-IPS:type": "private"},
},
"2": []map[string]string{
{"Addr": "30.40.50.60", "OS-EXT-IPS:type": "private"},
{"Addr": "40.50.60.70", "OS-EXT-IPS:type": "floating"},
},
},
},
{
ID: "node_no_lb_floating",
Metadata: map[string]string{
"k8s": "cluster.k8s.local",
"KopsRole": "Node",
},
Addresses: map[string]interface{}{
"1": []map[string]string{
{"Addr": "110.120.130.140", "OS-EXT-IPS:type": "floating"},
{"Addr": "120.130.140.150", "OS-EXT-IPS:type": "private"},
},
},
},
},
cloudFloatingEnabled: true,
expectedAPIIngress: []fi.ApiIngressStatus{
{IP: "1.2.3.4"},
{IP: "4.5.6.7"},
{IP: "40.50.60.70"},
},
},
}
for _, testCase := range tests {
t.Run(testCase.desc, func(t *testing.T) {
mux := http.NewServeMux()
fixture(
mux,
"/servers/detail",
http.MethodGet,
string(mustJSONMarshal(json.Marshal(
struct {
Servers []servers.Server `json:"servers"`
}{
Servers: testCase.instances,
},
))),
http.StatusOK,
)
for _, server := range testCase.instances {
fixture(
mux,
fmt.Sprintf("/servers/%s", server.ID),
http.MethodGet,
string(mustJSONMarshal(json.Marshal(
struct {
Server servers.Server `json:"server"`
}{
Server: server,
},
))),
http.StatusOK,
)
}
fixture(
mux,
"/lbaas/loadbalancers",
http.MethodGet,
string(mustJSONMarshal(json.Marshal(
struct{ LoadBalancers []loadbalancers.LoadBalancer }{
LoadBalancers: testCase.loadbalancers,
},
))),
http.StatusOK,
)
mux.HandleFunc("/floatingips", func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
params := r.URL.Query()
portID := params.Get("port_id")
if portID == "" {
fmt.Fprint(w, string(mustJSONMarshal(json.Marshal(
struct {
FloatingIPs []l3floatingips.FloatingIP `json:"floatingips"`
}{
FloatingIPs: testCase.l3FloatingIPs,
},
))))
return
}
for _, fip := range testCase.l3FloatingIPs {
if fip.PortID == portID {
json := string(mustJSONMarshal(json.Marshal(
struct {
FloatingIPs []l3floatingips.FloatingIP `json:"floatingips"`
}{
FloatingIPs: []l3floatingips.FloatingIP{fip},
},
)))
fmt.Fprint(w, json)
return
}
}
fmt.Fprint(w, string(mustJSONMarshal(json.Marshal(
struct {
FloatingIPs []l3floatingips.FloatingIP `json:"floatingips"`
}{
FloatingIPs: []l3floatingips.FloatingIP{},
},
))))
})
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
t.Errorf("Unexpected request for `%v`", r.URL)
http.Error(w, "Unexpected request", http.StatusInternalServerError)
})
testServer := httptest.NewServer(mux)
defer testServer.Close()
cloud := &openstackCloud{
floatingEnabled: testCase.cloudFloatingEnabled,
lbClient: serviceClient(testServer.URL),
novaClient: serviceClient(testServer.URL),
neutronClient: serviceClient(testServer.URL),
}
ingress, err := cloud.GetApiIngressStatus(testCase.cluster)
compareErrors(t, testCase.expectedError, err)
sortedExpected := sortByIP(testCase.expectedAPIIngress)
sortedActual := sortByIP(ingress)
sort.Sort(sortedExpected)
sort.Sort(sortedActual)
if !reflect.DeepEqual(sortedActual, sortedExpected) {
t.Errorf("Ingress status differ: expected\n%+#v\n\tgot:\n%+#v\n", testCase.expectedAPIIngress, ingress)
}
})
}
}
type sortByIP []fi.ApiIngressStatus
// Len is the number of elements in the collection.
func (s sortByIP) Len() int {
return len(s)
}
// Less reports whether the element with
// index i should sort before the element with index j.
func (s sortByIP) Less(i int, j int) bool {
return s[i].IP < s[j].IP
}
// Swap swaps the elements with indexes i and j.
func (s sortByIP) Swap(i int, j int) {
s[i], s[j] = s[j], s[i]
}
type serverList []servers.Server
func (s serverList) Get(id string) *servers.Server {
for _, server := range s {
if server.ID == id {
return &server
}
}
return nil
}
func serviceClient(url string) *gophercloud.ServiceClient {
return &gophercloud.ServiceClient{
ProviderClient: &gophercloud.ProviderClient{},
Endpoint: url + "/",
}
}
func fixture(mux *http.ServeMux, url string, method string, responseBody string, status int) {
mux.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) {
if r.Method != method {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(status)
fmt.Fprint(w, responseBody)
})
}
func mustJSONMarshal(data []byte, err error) []byte {
if err != nil {
panic(err)
}
return data
}
func compareErrors(t *testing.T, actual, expected error) {
t.Helper()
if pointersAreBothNil(t, "errors", actual, expected) {
return
}
a := fmt.Sprintf("%v", actual)
e := fmt.Sprintf("%v", expected)
if a != e {
t.Errorf("error differs: %+v instead of %+v", actual, expected)
}
}
func pointersAreBothNil(t *testing.T, name string, actual, expected interface{}) bool {
t.Helper()
if actual == nil && expected == nil {
return true
}
if !reflect.ValueOf(actual).IsValid() {
return false
}
if reflect.ValueOf(actual).IsNil() && reflect.ValueOf(expected).IsNil() {
return true
}
if actual == nil && expected != nil {
t.Fatalf("%s differ: actual is nil, expected is not", name)
}
if actual != nil && expected == nil {
t.Fatalf("%s differ: expected is nil, actual is not", name)
}
return false
}
func Test_BuildClients(t *testing.T) {
tags := map[string]string{
TagClusterName: "test.k8s.local",
}
provider := &gophercloud.ProviderClient{
EndpointLocator: func(eo gophercloud.EndpointOpts) (string, error) { return "", nil },
}
grid := []struct {
name string
spec *kops.OpenstackSpec
expectLB bool
expectedType string
expectFloatingEnabled bool
expectError bool
expectedExtNetName *string
}{
{
name: "Empty openstack spec means no load balancer",
spec: &kops.OpenstackSpec{},
expectLB: false,
expectedType: "",
},
{
name: "When octavia is set, but no router, an error should be returned",
spec: &kops.OpenstackSpec{
Loadbalancer: &kops.OpenstackLoadbalancerConfig{
UseOctavia: fi.PtrTo(true),
},
},
expectLB: true,
expectedType: "",
expectError: true,
},
{
name: "When octavia is set, and there is a router, a load-balancer should be returned",
spec: &kops.OpenstackSpec{
Loadbalancer: &kops.OpenstackLoadbalancerConfig{
UseOctavia: fi.PtrTo(true),
},
Router: &kops.OpenstackRouter{},
},
expectLB: true,
expectedType: "load-balancer",
expectFloatingEnabled: true,
expectError: false,
},
{
name: "When octavia is not set, network should be returned",
spec: &kops.OpenstackSpec{
Loadbalancer: &kops.OpenstackLoadbalancerConfig{},
},
expectLB: true,
expectedType: "network",
expectError: false,
},
{
name: "When router is set, but no LB, FIP support should be enabled",
spec: &kops.OpenstackSpec{
Router: &kops.OpenstackRouter{
ExternalNetwork: fi.PtrTo("some-ext-net"),
},
},
expectLB: false,
expectedType: "",
expectFloatingEnabled: true,
expectedExtNetName: fi.PtrTo("some-ext-net"),
}}
for _, g := range grid {
t.Run(g.name, func(t *testing.T) {
cloud, err := buildClients(provider, tags, g.spec, vfs.OpenstackConfig{}, "", false)
if g.expectError {
if err != nil {
return
} else {
t.Fatalf("expected error, but got nil")
}
}
if err != nil {
t.Fatalf("failed to build cloud clients: %v", err)
}
lbClient := cloud.LoadBalancerClient()
hasLB := cloud.LoadBalancerClient() != nil
if hasLB != g.expectLB {
t.Fatalf("did not match expectation. Expected: %v, actual: %v", g.expectLB, hasLB)
}
if g.expectLB {
if lbClient.Type != g.expectedType {
t.Fatalf("did not match expectation. Expected: %v, actual: %v", g.expectedType, lbClient.Type)
}
}
actualFloatingEnabled := cloud.(*openstackCloud).floatingEnabled
if g.expectFloatingEnabled != actualFloatingEnabled {
t.Fatalf("did not match expectation. Expected: %v, actual: %v", g.expectFloatingEnabled, actualFloatingEnabled)
}
actualExtNetName := fi.ValueOf(cloud.(*openstackCloud).extNetworkName)
expectedExtNetName := fi.ValueOf(g.expectedExtNetName)
if expectedExtNetName != actualExtNetName {
t.Fatalf("did not match expectation. Expected: %v, actual: %v", expectedExtNetName, actualExtNetName)
}
})
}
}
func setupMockCloudForDeletePortsTest(portDefinitions map[string]map[string]int) (*MockCloud, error) {
cloud := InstallMockOpenstackCloud("mock-central-1")
cloud.MockNeutronClient = mocknetworking.CreateClient()
for clusterName, instanceGroups := range portDefinitions {
for instanceGroup, n := range instanceGroups {
for i := 0; i < n; i++ {
port, err := cloud.CreatePort(ports.CreateOpts{
Name: fmt.Sprintf("port-%s-%d-%s", instanceGroup, i+1, clusterName),
NetworkID: "mock-network-id",
})
if err != nil {
return nil, fmt.Errorf("error creating port: %v", err)
}
err = cloud.AppendTag(ResourceTypePort, port.ID, fmt.Sprintf("%s=%s", TagClusterName, clusterName))
if err != nil {
return nil, fmt.Errorf("error appending tag: %v", err)
}
err = cloud.AppendTag(ResourceTypePort, port.ID, fmt.Sprintf("%s=%s", TagKopsInstanceGroup, instanceGroup))
if err != nil {
return nil, fmt.Errorf("error appending tag: %v", err)
}
}
}
}
return cloud, nil
}
func Test_deletePorts(t *testing.T) {
testCases := []struct {
description string
clusterName string
instanceGroup string
expectedPorts []string
}{
{
description: "Only delete ports of worker IG of my-cluster",
clusterName: "my-cluster",
instanceGroup: "worker",
expectedPorts: []string{
"port-control-plane-0-1-my-cluster",
"port-worker-2-1-my-cluster",
"port-worker-2-2-my-cluster",
"port-control-plane-0-1-my-cluster-2",
"port-worker-1-my-cluster-2",
"port-worker-2-my-cluster-2",
"port-worker-2-1-my-cluster-2",
"port-worker-2-2-my-cluster-2",
},
},
{
description: "Only delete ports of worker-2 IG of my-cluster",
clusterName: "my-cluster",
instanceGroup: "worker-2",
expectedPorts: []string{
"port-control-plane-0-1-my-cluster",
"port-worker-1-my-cluster",
"port-worker-2-my-cluster",
"port-control-plane-0-1-my-cluster-2",
"port-worker-1-my-cluster-2",
"port-worker-2-my-cluster-2",
"port-worker-2-1-my-cluster-2",
"port-worker-2-2-my-cluster-2",
},
},
}
portDefinitions := map[string]map[string]int{
"my-cluster": {
"control-plane-0": 1,
"worker": 2,
"worker-2": 2,
},
"my-cluster-2": {
"control-plane-0": 1,
"worker": 2,
"worker-2": 2,
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
cloud, err := setupMockCloudForDeletePortsTest(portDefinitions)
if err != nil {
t.Errorf("error while setting up test: %v", err)
}
deletePorts(cloud, testCase.instanceGroup, testCase.clusterName)
allPorts, err := cloud.ListPorts(ports.ListOpts{})
if err != nil {
t.Errorf("error while listing ports: %v", err)
}
actualPorts := []string{}
for _, port := range allPorts {
actualPorts = append(actualPorts, port.Name)
}
slices.Sort(actualPorts)
slices.Sort(testCase.expectedPorts)
if !reflect.DeepEqual(actualPorts, testCase.expectedPorts) {
t.Errorf("ports differ: expected\n%+#v\n\tgot:\n%+#v\n", testCase.expectedPorts, actualPorts)
}
})
}
}