mirror of https://github.com/kubernetes/kops.git
789 lines
27 KiB
Go
789 lines
27 KiB
Go
/*
|
|
Copyright 2019 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package openstackmodel
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"k8s.io/kops/pkg/apis/kops"
|
|
"k8s.io/kops/upup/pkg/fi"
|
|
"k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks"
|
|
|
|
"k8s.io/klog/v2"
|
|
"k8s.io/kops/pkg/wellknownports"
|
|
"k8s.io/utils/net"
|
|
|
|
sg "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/security/groups"
|
|
"github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/security/rules"
|
|
)
|
|
|
|
const (
|
|
IPProtocolTCP = string(rules.ProtocolTCP)
|
|
IPProtocolUDP = string(rules.ProtocolUDP)
|
|
IPV4 = string(rules.EtherType4)
|
|
IPV6 = string(rules.EtherType6)
|
|
ProtocolIPEncap = "4" // IP in IPv4/IPv6
|
|
)
|
|
|
|
// FirewallModelBuilder configures firewall network objects
|
|
type FirewallModelBuilder struct {
|
|
*OpenstackModelContext
|
|
Lifecycle fi.Lifecycle
|
|
Rules map[string]*openstacktasks.SecurityGroupRule
|
|
}
|
|
|
|
var _ fi.CloudupModelBuilder = &FirewallModelBuilder{}
|
|
|
|
func (b *FirewallModelBuilder) usesOctavia() bool {
|
|
if b.Cluster.Spec.CloudProvider.Openstack.Loadbalancer != nil {
|
|
return fi.ValueOf(b.Cluster.Spec.CloudProvider.Openstack.Loadbalancer.UseOctavia)
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (b *FirewallModelBuilder) getOctaviaProvider() string {
|
|
if b.Cluster.Spec.CloudProvider.Openstack.Loadbalancer != nil {
|
|
return fi.ValueOf(b.Cluster.Spec.CloudProvider.Openstack.Loadbalancer.Provider)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// addDirectionalGroupRule - create a rule on the source group to the dest group provided a securityGroupRuleTask
|
|
//
|
|
// Example
|
|
// Create an Ingress rule on source allowing traffic from dest with the options in the SecurityGroupRule
|
|
// Create an Egress rule on source allowing traffic to dest with the options in the SecurityGroupRule
|
|
func (b *FirewallModelBuilder) addDirectionalGroupRule(c *fi.CloudupModelBuilderContext, source, dest *openstacktasks.SecurityGroup, sgr *openstacktasks.SecurityGroupRule) {
|
|
t := &openstacktasks.SecurityGroupRule{
|
|
Direction: sgr.Direction,
|
|
EtherType: sgr.EtherType,
|
|
Lifecycle: sgr.Lifecycle,
|
|
PortRangeMin: sgr.PortRangeMin,
|
|
PortRangeMax: sgr.PortRangeMax,
|
|
Protocol: sgr.Protocol,
|
|
RemoteGroup: dest,
|
|
RemoteIPPrefix: sgr.RemoteIPPrefix,
|
|
SecGroup: source,
|
|
Delete: fi.PtrTo(false),
|
|
}
|
|
|
|
klog.V(8).Infof("Adding rule %v", fi.ValueOf(t.GetName()))
|
|
b.Rules[fi.ValueOf(t.GetName())] = t
|
|
}
|
|
|
|
// addSSHRules - sets the ssh rules based on the presence of a bastion
|
|
func (b *FirewallModelBuilder) addSSHRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
bastionName := b.SecurityGroupName(kops.InstanceGroupRoleBastion)
|
|
masterSG := sgMap[masterName]
|
|
nodeSG := sgMap[nodeName]
|
|
bastionSG := sgMap[bastionName]
|
|
|
|
sshIngress := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(string(rules.EtherType4)),
|
|
PortRangeMin: i(22),
|
|
PortRangeMax: i(22),
|
|
}
|
|
|
|
if b.UsesSSHBastion() {
|
|
for _, sshAccess := range b.Cluster.Spec.SSHAccess {
|
|
etherType := IPV4
|
|
if !net.IsIPv4CIDRString(sshAccess) {
|
|
etherType = IPV6
|
|
}
|
|
sshRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(string(etherType)),
|
|
PortRangeMin: i(22),
|
|
PortRangeMax: i(22),
|
|
RemoteIPPrefix: s(sshAccess),
|
|
}
|
|
b.addDirectionalGroupRule(c, bastionSG, nil, sshRule)
|
|
}
|
|
// Allow ingress ssh from the bastion on the masters and nodes
|
|
b.addDirectionalGroupRule(c, masterSG, bastionSG, sshIngress)
|
|
b.addDirectionalGroupRule(c, nodeSG, bastionSG, sshIngress)
|
|
} else {
|
|
for _, sshAccess := range b.Cluster.Spec.SSHAccess {
|
|
etherType := IPV4
|
|
if !net.IsIPv4CIDRString(sshAccess) {
|
|
etherType = IPV6
|
|
}
|
|
sshRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(string(etherType)),
|
|
PortRangeMin: i(22),
|
|
PortRangeMax: i(22),
|
|
RemoteIPPrefix: s(sshAccess),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, nil, sshRule)
|
|
b.addDirectionalGroupRule(c, nodeSG, nil, sshRule)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// addETCDRules - Add ETCD access rules based on which CNI might need to access __ETCD_ENDPOINTS__
|
|
func (b *FirewallModelBuilder) addETCDRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
masterSG := sgMap[masterName]
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
nodeSG := sgMap[nodeName]
|
|
|
|
// ETCD Peer Discovery
|
|
etcdRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(4001),
|
|
PortRangeMax: i(4002),
|
|
}
|
|
etcdPeerRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(2380),
|
|
PortRangeMax: i(2381),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, etcdRule)
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, etcdPeerRule)
|
|
|
|
if b.Cluster.Spec.Networking.Cilium != nil && b.Cluster.Spec.Networking.Cilium.EtcdManaged {
|
|
etcdCiliumPeerRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(2382),
|
|
PortRangeMax: i(2382),
|
|
}
|
|
etcdCiliumGRPCRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(wellknownports.EtcdCiliumGRPC),
|
|
PortRangeMax: i(wellknownports.EtcdCiliumGRPC),
|
|
}
|
|
etcdCiliumQuarantinedClient := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(wellknownports.EtcdCiliumQuarantinedClientPort),
|
|
PortRangeMax: i(wellknownports.EtcdCiliumQuarantinedClientPort),
|
|
}
|
|
etcdCiliumClientRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(wellknownports.EtcdCiliumClientPort),
|
|
PortRangeMax: i(wellknownports.EtcdCiliumClientPort),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, etcdCiliumPeerRule)
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, etcdCiliumGRPCRule)
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, etcdCiliumClientRule)
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, etcdCiliumQuarantinedClient)
|
|
b.addDirectionalGroupRule(c, nodeSG, masterSG, etcdCiliumClientRule)
|
|
}
|
|
|
|
for _, portRange := range wellknownports.ETCDPortRanges() {
|
|
etcdMgmrRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(string(rules.EtherType4)),
|
|
PortRangeMin: i(portRange.Min),
|
|
PortRangeMax: i(portRange.Max),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, etcdMgmrRule)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// addNodePortRules - Add node port rules to nodes give the NodePortRange
|
|
func (b *FirewallModelBuilder) addNodePortRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
nodeSG := sgMap[nodeName]
|
|
|
|
for _, nodePortAccess := range b.Cluster.Spec.NodePortAccess {
|
|
|
|
nodePortRange, err := b.NodePortRange()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, protocol := range []string{IPProtocolTCP, IPProtocolUDP} {
|
|
nodePortRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(protocol),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(nodePortRange.Base),
|
|
PortRangeMax: i(nodePortRange.Base + nodePortRange.Size - 1),
|
|
RemoteIPPrefix: s(nodePortAccess),
|
|
}
|
|
b.addDirectionalGroupRule(c, nodeSG, nil, nodePortRule)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// addHTTPSRules - Add rules to 443 access given the presence of a loadbalancer or not
|
|
func (b *FirewallModelBuilder) addHTTPSRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup, useVIPACL bool) error {
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
lbSGName := b.APIResourceName()
|
|
lbSG := sgMap[lbSGName]
|
|
masterSG := sgMap[masterName]
|
|
nodeSG := sgMap[nodeName]
|
|
|
|
httpsIngress := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(443),
|
|
PortRangeMax: i(443),
|
|
}
|
|
|
|
// Allow all local communication for kubernetes.svc and to the api.internal lb/gossip for kubelet's
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, httpsIngress)
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, httpsIngress)
|
|
|
|
if b.UseLoadBalancerForAPI() {
|
|
if !useVIPACL {
|
|
// Allow API Access to the lb sg
|
|
for _, apiAccess := range b.Cluster.Spec.API.Access {
|
|
etherType := IPV4
|
|
if !net.IsIPv4CIDRString(apiAccess) {
|
|
etherType = IPV6
|
|
}
|
|
b.addDirectionalGroupRule(c, lbSG, nil, &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(etherType),
|
|
PortRangeMin: i(443),
|
|
PortRangeMax: i(443),
|
|
RemoteIPPrefix: s(apiAccess),
|
|
})
|
|
}
|
|
// Allow masters ingress from the sg
|
|
b.addDirectionalGroupRule(c, masterSG, lbSG, httpsIngress)
|
|
}
|
|
|
|
// FIXME: Octavia port traffic appears to be denied though its port is in lbSG
|
|
if b.usesOctavia() {
|
|
if b.getOctaviaProvider() == "ovn" {
|
|
for _, apiAccess := range b.Cluster.Spec.API.Access {
|
|
etherType := IPV4
|
|
if !net.IsIPv4CIDRString(apiAccess) {
|
|
etherType = IPV6
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, nil, &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(etherType),
|
|
PortRangeMin: i(443),
|
|
PortRangeMax: i(443),
|
|
RemoteIPPrefix: s(apiAccess),
|
|
})
|
|
}
|
|
} else {
|
|
b.addDirectionalGroupRule(c, masterSG, nil, &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(443),
|
|
PortRangeMax: i(443),
|
|
RemoteIPPrefix: s(b.Cluster.Spec.Networking.NetworkCIDR),
|
|
})
|
|
}
|
|
}
|
|
|
|
} else {
|
|
// Allow the masters to receive connections from KubernetesAPIAccess
|
|
for _, apiAccess := range b.Cluster.Spec.API.Access {
|
|
etherType := IPV4
|
|
if !net.IsIPv4CIDRString(apiAccess) {
|
|
etherType = IPV6
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, nil, &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(etherType),
|
|
PortRangeMin: i(443),
|
|
PortRangeMax: i(443),
|
|
RemoteIPPrefix: s(apiAccess),
|
|
})
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// addKubeletRules - Add rules to 10250 port
|
|
func (b *FirewallModelBuilder) addKubeletRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
// TODO: This is the default port for kubelet and may be overridden
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
masterSG := sgMap[masterName]
|
|
nodeSG := sgMap[nodeName]
|
|
|
|
kubeletRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(10250),
|
|
PortRangeMax: i(10250),
|
|
}
|
|
|
|
// allow node-node, node-master and master-master and master-node
|
|
for _, sgName := range []*openstacktasks.SecurityGroup{masterSG, nodeSG} {
|
|
b.addDirectionalGroupRule(c, masterSG, sgName, kubeletRule)
|
|
b.addDirectionalGroupRule(c, nodeSG, sgName, kubeletRule)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// addNodeExporterAndOccmRules - Allow 9100 TCP port from nodesg, allow 10258 from nodes to master - expose occm metrics
|
|
func (b *FirewallModelBuilder) addNodeExporterAndOccmRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
masterSG := sgMap[masterName]
|
|
nodeSG := sgMap[nodeName]
|
|
nodeExporterIngress := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(9100),
|
|
PortRangeMax: i(9100),
|
|
}
|
|
// allow 9100 port from nodeSG
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, nodeExporterIngress)
|
|
b.addDirectionalGroupRule(c, nodeSG, nodeSG, nodeExporterIngress)
|
|
|
|
occmMetrics := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(10258),
|
|
PortRangeMax: i(10258),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, occmMetrics)
|
|
|
|
if fi.ValueOf(b.Cluster.Spec.CloudProvider.Openstack.BlockStorage.MetricsEnabled) {
|
|
csiMetrics := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(9809),
|
|
PortRangeMax: i(9809),
|
|
}
|
|
// allow 9809 port from nodeSG & masterSG
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, csiMetrics)
|
|
b.addDirectionalGroupRule(c, nodeSG, nodeSG, csiMetrics)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// addKubeControllerManagerMetricsRules - Add rules to 10257 port
|
|
func (b *FirewallModelBuilder) addKubeControllerManagerMetricsRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
// TODO: This is the default port for kube-controller-manager metrics and may be overridden
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
masterSG := sgMap[masterName]
|
|
nodeSG := sgMap[nodeName]
|
|
|
|
kubeControllerManagerMetricsRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(10257),
|
|
PortRangeMax: i(10257),
|
|
}
|
|
|
|
// allow port 10257 from nodeSG to masterSG
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, kubeControllerManagerMetricsRule)
|
|
return nil
|
|
}
|
|
|
|
// addKubeSchedulerMetricsRules - Add rules to 10259 port
|
|
func (b *FirewallModelBuilder) addKubeSchedulerMetricsRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
// TODO: This is the default port for kube-scheduler metrics and may be overridden
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
masterSG := sgMap[masterName]
|
|
nodeSG := sgMap[nodeName]
|
|
|
|
kubeSchedulerMetricsRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(IPProtocolTCP),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(10259),
|
|
PortRangeMax: i(10259),
|
|
}
|
|
|
|
// allow port 10259 from nodeSG to masterSG
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, kubeSchedulerMetricsRule)
|
|
return nil
|
|
}
|
|
|
|
// addDNSRules - Add DNS rules for internal DNS queries
|
|
func (b *FirewallModelBuilder) addDNSRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
masterSG := sgMap[masterName]
|
|
nodeSG := sgMap[nodeName]
|
|
for _, protocol := range []string{IPProtocolTCP, IPProtocolUDP} {
|
|
dnsRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(protocol),
|
|
EtherType: s(IPV4),
|
|
PortRangeMin: i(53),
|
|
PortRangeMax: i(53),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, dnsRule)
|
|
b.addDirectionalGroupRule(c, nodeSG, masterSG, dnsRule)
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, dnsRule)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// addCNIRules - Add ports required for different CNI implementations
|
|
func (b *FirewallModelBuilder) addCNIRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
udpPorts := []int{}
|
|
tcpPorts := []int{}
|
|
protocols := []string{}
|
|
|
|
if b.Cluster.Spec.Networking.Kopeio != nil {
|
|
// VXLAN over UDP
|
|
// https://tools.ietf.org/html/rfc7348
|
|
udpPorts = append(udpPorts, 4789)
|
|
}
|
|
|
|
if b.Cluster.Spec.Networking.Cilium != nil {
|
|
udpPorts = append(udpPorts, 8472)
|
|
tcpPorts = append(tcpPorts, 4240)
|
|
if b.Cluster.Spec.Networking.Cilium.Hubble != nil && fi.ValueOf(b.Cluster.Spec.Networking.Cilium.Hubble.Enabled) {
|
|
tcpPorts = append(tcpPorts, 4244)
|
|
}
|
|
}
|
|
|
|
if b.Cluster.Spec.Networking.Flannel != nil {
|
|
switch b.Cluster.Spec.Networking.Flannel.Backend {
|
|
case "", "udp":
|
|
udpPorts = append(udpPorts, 8285)
|
|
case "vxlan":
|
|
udpPorts = append(udpPorts, 8472)
|
|
default:
|
|
klog.Warningf("unknown flannel networking backend %q", b.Cluster.Spec.Networking.Flannel.Backend)
|
|
}
|
|
}
|
|
|
|
if b.Cluster.Spec.Networking.Calico != nil {
|
|
tcpPorts = append(tcpPorts, 179)
|
|
protocols = append(protocols, ProtocolIPEncap)
|
|
}
|
|
|
|
if b.Cluster.Spec.Networking.KubeRouter != nil {
|
|
protocols = append(protocols, ProtocolIPEncap)
|
|
}
|
|
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
masterSG := sgMap[masterName]
|
|
nodeSG := sgMap[nodeName]
|
|
|
|
for _, udpPort := range udpPorts {
|
|
udpRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolUDP)),
|
|
EtherType: s(string(rules.EtherType4)),
|
|
PortRangeMin: i(udpPort),
|
|
PortRangeMax: i(udpPort),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, udpRule)
|
|
b.addDirectionalGroupRule(c, nodeSG, masterSG, udpRule)
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, udpRule)
|
|
b.addDirectionalGroupRule(c, nodeSG, nodeSG, udpRule)
|
|
}
|
|
for _, tcpPort := range tcpPorts {
|
|
tcpRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(string(rules.EtherType4)),
|
|
PortRangeMin: i(tcpPort),
|
|
PortRangeMax: i(tcpPort),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, tcpRule)
|
|
b.addDirectionalGroupRule(c, nodeSG, masterSG, tcpRule)
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, tcpRule)
|
|
b.addDirectionalGroupRule(c, nodeSG, nodeSG, tcpRule)
|
|
}
|
|
|
|
for _, protocol := range protocols {
|
|
protocolRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(protocol),
|
|
EtherType: s(string(rules.EtherType4)),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, nil, protocolRule)
|
|
b.addDirectionalGroupRule(c, nodeSG, nil, protocolRule)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// addKopsControllerRules - Add rules for kops-controller for node bootstrap
|
|
func (b *FirewallModelBuilder) addKopsControllerRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
masterSG := sgMap[masterName]
|
|
nodeSG := sgMap[nodeName]
|
|
kopsControllerRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(string(rules.EtherType4)),
|
|
PortRangeMin: i(wellknownports.KopsControllerPort),
|
|
PortRangeMax: i(wellknownports.KopsControllerPort),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, kopsControllerRule)
|
|
return nil
|
|
}
|
|
|
|
// addProtokubeRules - Add rules for protokube if gossip DNS is enabled
|
|
func (b *FirewallModelBuilder) addProtokubeRules(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
if b.Cluster.UsesLegacyGossip() {
|
|
masterName := b.SecurityGroupName(kops.InstanceGroupRoleControlPlane)
|
|
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
|
|
masterSG := sgMap[masterName]
|
|
nodeSG := sgMap[nodeName]
|
|
for _, portRange := range wellknownports.DNSGossipPortRanges() {
|
|
protokubeRule := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirIngress)),
|
|
Protocol: s(string(rules.ProtocolTCP)),
|
|
EtherType: s(string(rules.EtherType4)),
|
|
PortRangeMin: i(portRange.Min),
|
|
PortRangeMax: i(portRange.Max),
|
|
}
|
|
b.addDirectionalGroupRule(c, masterSG, nodeSG, protokubeRule)
|
|
b.addDirectionalGroupRule(c, nodeSG, masterSG, protokubeRule)
|
|
b.addDirectionalGroupRule(c, masterSG, masterSG, protokubeRule)
|
|
b.addDirectionalGroupRule(c, nodeSG, nodeSG, protokubeRule)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *FirewallModelBuilder) getExistingRules(sgMap map[string]*openstacktasks.SecurityGroup) error {
|
|
osCloud, err := b.createCloud()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sgIdMap := make(map[string]*openstacktasks.SecurityGroup)
|
|
for sgName, sgt := range sgMap {
|
|
|
|
sgs, err := osCloud.ListSecurityGroups(sg.ListOpts{
|
|
Name: sgName,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(sgs) == 0 {
|
|
continue
|
|
}
|
|
if len(sgs) > 1 {
|
|
return fmt.Errorf("Found multiple security groups with the same name: %v", sgName)
|
|
}
|
|
sg := sgs[0]
|
|
sgt.Name = fi.PtrTo(sg.Name)
|
|
sgIdMap[sg.ID] = sgt
|
|
}
|
|
|
|
for sgid := range sgIdMap {
|
|
|
|
sgRules, err := osCloud.ListSecurityGroupRules(rules.ListOpts{
|
|
SecGroupID: sgid,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, rule := range sgRules {
|
|
|
|
t := &openstacktasks.SecurityGroupRule{
|
|
ID: fi.PtrTo(rule.ID),
|
|
Direction: fi.PtrTo(rule.Direction),
|
|
EtherType: fi.PtrTo(rule.EtherType),
|
|
PortRangeMax: fi.PtrTo(rule.PortRangeMax),
|
|
PortRangeMin: fi.PtrTo(rule.PortRangeMin),
|
|
Protocol: fi.PtrTo(rule.Protocol),
|
|
RemoteIPPrefix: fi.PtrTo(rule.RemoteIPPrefix),
|
|
RemoteGroup: sgIdMap[rule.RemoteGroupID],
|
|
Lifecycle: b.Lifecycle,
|
|
SecGroup: sgIdMap[rule.SecGroupID],
|
|
Delete: fi.PtrTo(true),
|
|
}
|
|
klog.V(8).Infof("Adding existing rule %v", t)
|
|
b.Rules[fi.ValueOf(t.GetName())] = t
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *FirewallModelBuilder) addDefaultEgress(c *fi.CloudupModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup, useVIPACL bool) {
|
|
for name, sg := range sgMap {
|
|
if useVIPACL && name == b.APIResourceName() {
|
|
continue
|
|
}
|
|
t := &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirEgress)),
|
|
EtherType: s(string(rules.EtherType4)),
|
|
PortRangeMin: i(0),
|
|
PortRangeMax: i(0),
|
|
}
|
|
b.addDirectionalGroupRule(c, sg, nil, t)
|
|
|
|
t = &openstacktasks.SecurityGroupRule{
|
|
Lifecycle: b.Lifecycle,
|
|
Direction: s(string(rules.DirEgress)),
|
|
EtherType: s(string(rules.EtherType6)),
|
|
PortRangeMin: i(0),
|
|
PortRangeMax: i(0),
|
|
}
|
|
b.addDirectionalGroupRule(c, sg, nil, t)
|
|
}
|
|
}
|
|
|
|
// Build - schedule security groups and security group rule tasks for Openstack
|
|
func (b *FirewallModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
|
|
roles := []kops.InstanceGroupRole{kops.InstanceGroupRoleControlPlane, kops.InstanceGroupRoleNode}
|
|
if b.UsesSSHBastion() {
|
|
roles = append(roles, kops.InstanceGroupRoleBastion)
|
|
}
|
|
|
|
sgMap := make(map[string]*openstacktasks.SecurityGroup)
|
|
|
|
useVIPACL := false
|
|
if b.UseLoadBalancerForAPI() && b.UseVIPACL() {
|
|
useVIPACL = true
|
|
}
|
|
sg := &openstacktasks.SecurityGroup{
|
|
Name: s(b.APIResourceName()),
|
|
Lifecycle: b.Lifecycle,
|
|
RemoveExtraRules: []string{"port=443"},
|
|
}
|
|
if useVIPACL {
|
|
sg.RemoveGroup = true
|
|
}
|
|
c.AddTask(sg)
|
|
sgMap[b.APIResourceName()] = sg
|
|
for _, role := range roles {
|
|
|
|
// Create Security Group for Role
|
|
groupName := b.SecurityGroupName(role)
|
|
sg := &openstacktasks.SecurityGroup{
|
|
Name: s(groupName),
|
|
Lifecycle: b.Lifecycle,
|
|
RemoveGroup: false,
|
|
}
|
|
switch role {
|
|
case kops.InstanceGroupRoleBastion:
|
|
sg.RemoveExtraRules = []string{"port=22"}
|
|
case kops.InstanceGroupRoleNode:
|
|
sg.RemoveExtraRules = []string{"port=22", "port=10250"}
|
|
case kops.InstanceGroupRoleControlPlane:
|
|
sg.RemoveExtraRules = []string{"port=22", "port=443", "port=10250"}
|
|
}
|
|
c.AddTask(sg)
|
|
sgMap[groupName] = sg
|
|
}
|
|
|
|
b.Rules = make(map[string]*openstacktasks.SecurityGroupRule)
|
|
|
|
err := b.getExistingRules(sgMap)
|
|
// if this fails, it just means that existing rules won't be cleaned up
|
|
if err != nil {
|
|
klog.Warningf("Failed to list existing security groups: %v", err)
|
|
}
|
|
|
|
b.addDefaultEgress(c, sgMap, useVIPACL)
|
|
|
|
// Add API Server Rules
|
|
b.addHTTPSRules(c, sgMap, useVIPACL)
|
|
|
|
// Add SSH
|
|
b.addSSHRules(c, sgMap)
|
|
// Allow overlay DNS
|
|
b.addDNSRules(c, sgMap)
|
|
// Add Kubelet Rules
|
|
b.addKubeletRules(c, sgMap)
|
|
// Add Node exporter and occm metrics Rules
|
|
b.addNodeExporterAndOccmRules(c, sgMap)
|
|
// Add kube controller manager metrics Rules
|
|
b.addKubeControllerManagerMetricsRules(c, sgMap)
|
|
// Add kube scheduler metrics Rules
|
|
b.addKubeSchedulerMetricsRules(c, sgMap)
|
|
// Protokube Rules
|
|
b.addProtokubeRules(c, sgMap)
|
|
// Kops-controller Rules
|
|
b.addKopsControllerRules(c, sgMap)
|
|
// Allow necessary local traffic
|
|
b.addCNIRules(c, sgMap)
|
|
// ETCD Leader Election
|
|
b.addETCDRules(c, sgMap)
|
|
// Add NodePort Rules:
|
|
err = b.addNodePortRules(c, sgMap)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to add node port rules: %v", err)
|
|
}
|
|
|
|
for _, r := range b.Rules {
|
|
c.AddTask(r)
|
|
}
|
|
|
|
return nil
|
|
}
|