kops/pkg/model/openstackmodel/firewall.go

519 lines
17 KiB
Go

/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstackmodel
import (
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks"
//TODO: Replace with klog
"k8s.io/klog"
"k8s.io/kops/pkg/dns"
)
const (
IPProtocolTCP = string(rules.ProtocolTCP)
IPProtocolUDP = string(rules.ProtocolUDP)
IPV4 = string(rules.EtherType4)
ProtocolIPEncap = "4" // IP in IPv4/IPv6
)
// FirewallModelBuilder configures firewall network objects
type FirewallModelBuilder struct {
*OpenstackModelContext
Lifecycle *fi.Lifecycle
}
var _ fi.ModelBuilder = &FirewallModelBuilder{}
func (b *FirewallModelBuilder) usesOctavia() bool {
if b.Cluster.Spec.CloudConfig != nil &&
b.Cluster.Spec.CloudConfig.Openstack != nil &&
b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer != nil {
return fi.BoolValue(b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer.UseOctavia)
}
return false
}
// addDirectionalGroupRule - create a rule on the source group to the dest group provided a securityGroupRuleTask
// Example
// Create an Ingress rule on source allowing traffic from dest with the options in the SecurityGroupRule
// Create an Egress rule on source allowing traffic to dest with the options in the SecurityGroupRule
func addDirectionalGroupRule(c *fi.ModelBuilderContext, source, dest *openstacktasks.SecurityGroup, sgr *openstacktasks.SecurityGroupRule) {
t := &openstacktasks.SecurityGroupRule{
Direction: sgr.Direction,
EtherType: sgr.EtherType,
Lifecycle: sgr.Lifecycle,
PortRangeMin: sgr.PortRangeMin,
PortRangeMax: sgr.PortRangeMax,
Protocol: sgr.Protocol,
RemoteGroup: dest,
RemoteIPPrefix: sgr.RemoteIPPrefix,
SecGroup: source,
}
c.AddTask(t)
}
// addSSHRules - sets the ssh rules based on the presence of a bastion
func (b *FirewallModelBuilder) addSSHRules(c *fi.ModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
masterName := b.SecurityGroupName(kops.InstanceGroupRoleMaster)
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
bastionName := b.SecurityGroupName(kops.InstanceGroupRoleBastion)
masterSG := sgMap[masterName]
nodeSG := sgMap[nodeName]
bastionSG := sgMap[bastionName]
sshIngress := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(string(rules.EtherType4)),
PortRangeMin: i(22),
PortRangeMax: i(22),
}
if b.UsesSSHBastion() {
for _, sshAccess := range b.Cluster.Spec.SSHAccess {
sshRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(string(rules.EtherType4)),
PortRangeMin: i(22),
PortRangeMax: i(22),
RemoteIPPrefix: s(sshAccess),
}
addDirectionalGroupRule(c, bastionSG, nil, sshRule)
}
//Allow ingress ssh from the bastion on the masters and nodes
addDirectionalGroupRule(c, masterSG, bastionSG, sshIngress)
addDirectionalGroupRule(c, nodeSG, bastionSG, sshIngress)
} else {
for _, sshAccess := range b.Cluster.Spec.SSHAccess {
sshRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(string(rules.EtherType4)),
PortRangeMin: i(22),
PortRangeMax: i(22),
RemoteIPPrefix: s(sshAccess),
}
addDirectionalGroupRule(c, masterSG, nil, sshRule)
addDirectionalGroupRule(c, nodeSG, nil, sshRule)
}
}
return nil
}
// addETCDRules - Add ETCD access rules based on which CNI might need to access __ETCD_ENDPOINTS__
func (b *FirewallModelBuilder) addETCDRules(c *fi.ModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
masterName := b.SecurityGroupName(kops.InstanceGroupRoleMaster)
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
masterSG := sgMap[masterName]
nodeSG := sgMap[nodeName]
// ETCD Peer Discovery
etcdRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(IPV4),
PortRangeMin: i(4001),
PortRangeMax: i(4002),
}
etcdPeerRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(IPV4),
PortRangeMin: i(2380),
PortRangeMax: i(2381),
}
addDirectionalGroupRule(c, masterSG, masterSG, etcdRule)
addDirectionalGroupRule(c, masterSG, masterSG, etcdPeerRule)
if b.Cluster.Spec.Networking.Romana != nil ||
b.Cluster.Spec.Networking.Calico != nil {
etcdCNIRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(IPV4),
PortRangeMin: i(4001),
PortRangeMax: i(4001),
}
// Master access from other masters covered above
// Allow nodes to reach ETCD endpoints
addDirectionalGroupRule(c, masterSG, nodeSG, etcdCNIRule)
}
return nil
}
// addNodePortRules - Add node port rules to nodes give the NodePortRange
func (b *FirewallModelBuilder) addNodePortRules(c *fi.ModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
nodeSG := sgMap[nodeName]
for _, nodePortAccess := range b.Cluster.Spec.NodePortAccess {
nodePortRange, err := b.NodePortRange()
if err != nil {
return err
}
for _, protocol := range []string{IPProtocolTCP, IPProtocolUDP} {
nodePortRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(protocol),
EtherType: s(IPV4),
PortRangeMin: i(nodePortRange.Base),
PortRangeMax: i(nodePortRange.Base + nodePortRange.Size - 1),
RemoteIPPrefix: s(nodePortAccess),
}
addDirectionalGroupRule(c, nodeSG, nil, nodePortRule)
}
}
return nil
}
// addHTTPSRules - Add rules to 443 access given the presence of a loadbalancer or not
func (b *FirewallModelBuilder) addHTTPSRules(c *fi.ModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
masterName := b.SecurityGroupName(kops.InstanceGroupRoleMaster)
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
lbSGName := b.Cluster.Spec.MasterPublicName
lbSG := sgMap[lbSGName]
masterSG := sgMap[masterName]
nodeSG := sgMap[nodeName]
httpsIngress := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(IPProtocolTCP),
EtherType: s(IPV4),
PortRangeMin: i(443),
PortRangeMax: i(443),
}
//Allow all local communication for kubernetes.svc and to the api.internal lb/gossip for kubelet's
addDirectionalGroupRule(c, masterSG, nodeSG, httpsIngress)
addDirectionalGroupRule(c, masterSG, masterSG, httpsIngress)
if b.UseLoadBalancerForAPI() {
//Allow API Access to the lb sg
for _, apiAccess := range b.Cluster.Spec.KubernetesAPIAccess {
addDirectionalGroupRule(c, lbSG, nil, &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(IPProtocolTCP),
EtherType: s(IPV4),
PortRangeMin: i(443),
PortRangeMax: i(443),
RemoteIPPrefix: s(apiAccess),
})
}
//Allow masters ingress from the sg
addDirectionalGroupRule(c, masterSG, lbSG, httpsIngress)
//FIXME: Octavia port traffic appears to be denied though its port is in lbSG
if b.usesOctavia() {
addDirectionalGroupRule(c, masterSG, nil, &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(IPProtocolTCP),
EtherType: s(IPV4),
PortRangeMin: i(443),
PortRangeMax: i(443),
RemoteIPPrefix: s(b.Cluster.Spec.NetworkCIDR),
})
}
} else {
// Allow the masters to receive connections from KubernetesAPIAccess
for _, apiAccess := range b.Cluster.Spec.KubernetesAPIAccess {
addDirectionalGroupRule(c, masterSG, nil, &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(IPProtocolTCP),
EtherType: s(IPV4),
PortRangeMin: i(443),
PortRangeMax: i(443),
RemoteIPPrefix: s(apiAccess),
})
}
}
return nil
}
// addKubeletRules - Add rules to 10250 port
func (b *FirewallModelBuilder) addKubeletRules(c *fi.ModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
//TODO: This is the default port for kubelet and may be overwridden
masterName := b.SecurityGroupName(kops.InstanceGroupRoleMaster)
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
masterSG := sgMap[masterName]
nodeSG := sgMap[nodeName]
kubeletRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(IPProtocolTCP),
EtherType: s(IPV4),
PortRangeMin: i(10250),
PortRangeMax: i(10250),
}
// allow node-node, node-master and master-master and master-node
for _, sgName := range []*openstacktasks.SecurityGroup{masterSG, nodeSG} {
addDirectionalGroupRule(c, masterSG, sgName, kubeletRule)
addDirectionalGroupRule(c, nodeSG, sgName, kubeletRule)
}
return nil
}
// addNodeExporterRules - Allow 9100 TCP port from nodesg
func (b *FirewallModelBuilder) addNodeExporterRules(c *fi.ModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
masterName := b.SecurityGroupName(kops.InstanceGroupRoleMaster)
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
masterSG := sgMap[masterName]
nodeSG := sgMap[nodeName]
nodeExporterIngress := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(IPProtocolTCP),
EtherType: s(IPV4),
PortRangeMin: i(9100),
PortRangeMax: i(9100),
}
// allow 9100 port from nodeSG
addDirectionalGroupRule(c, masterSG, nodeSG, nodeExporterIngress)
addDirectionalGroupRule(c, nodeSG, nodeSG, nodeExporterIngress)
return nil
}
// addDNSRules - Add DNS rules for internal DNS queries
func (b *FirewallModelBuilder) addDNSRules(c *fi.ModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
masterName := b.SecurityGroupName(kops.InstanceGroupRoleMaster)
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
masterSG := sgMap[masterName]
nodeSG := sgMap[nodeName]
for _, protocol := range []string{IPProtocolTCP, IPProtocolUDP} {
dnsRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(protocol),
EtherType: s(IPV4),
PortRangeMin: i(53),
PortRangeMax: i(53),
}
addDirectionalGroupRule(c, masterSG, nodeSG, dnsRule)
addDirectionalGroupRule(c, nodeSG, masterSG, dnsRule)
addDirectionalGroupRule(c, masterSG, masterSG, dnsRule)
}
return nil
}
// addCNIRules - Add ports required for different CNI implementations
func (b *FirewallModelBuilder) addCNIRules(c *fi.ModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
udpPorts := []int{}
tcpPorts := []int{}
protocols := []string{}
// allow cadvisor
tcpPorts = append(tcpPorts, 4194)
if b.Cluster.Spec.Networking != nil {
if b.Cluster.Spec.Networking.Kopeio != nil {
// VXLAN over UDP
// https://tools.ietf.org/html/rfc7348
udpPorts = append(udpPorts, 4789)
}
if b.Cluster.Spec.Networking.Weave != nil {
udpPorts = append(udpPorts, 6783)
tcpPorts = append(tcpPorts, 6783)
udpPorts = append(udpPorts, 6784)
}
if b.Cluster.Spec.Networking.Flannel != nil {
switch b.Cluster.Spec.Networking.Flannel.Backend {
case "", "udp":
udpPorts = append(udpPorts, 8285)
case "vxlan":
udpPorts = append(udpPorts, 8472)
default:
klog.Warningf("unknown flannel networking backend %q", b.Cluster.Spec.Networking.Flannel.Backend)
}
}
if b.Cluster.Spec.Networking.Calico != nil {
tcpPorts = append(tcpPorts, 179)
protocols = append(protocols, ProtocolIPEncap)
}
if b.Cluster.Spec.Networking.Romana != nil {
tcpPorts = append(tcpPorts, 9600)
}
if b.Cluster.Spec.Networking.Kuberouter != nil {
protocols = append(protocols, ProtocolIPEncap)
}
}
masterName := b.SecurityGroupName(kops.InstanceGroupRoleMaster)
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
masterSG := sgMap[masterName]
nodeSG := sgMap[nodeName]
for _, udpPort := range udpPorts {
udpRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolUDP)),
EtherType: s(string(rules.EtherType4)),
PortRangeMin: i(udpPort),
PortRangeMax: i(udpPort),
RemoteIPPrefix: s(b.Cluster.Spec.NetworkCIDR),
}
addDirectionalGroupRule(c, masterSG, nil, udpRule)
addDirectionalGroupRule(c, nodeSG, nil, udpRule)
}
for _, tcpPort := range tcpPorts {
tcpRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(string(rules.EtherType4)),
PortRangeMin: i(tcpPort),
PortRangeMax: i(tcpPort),
RemoteIPPrefix: s(b.Cluster.Spec.NetworkCIDR),
}
addDirectionalGroupRule(c, masterSG, nil, tcpRule)
addDirectionalGroupRule(c, nodeSG, nil, tcpRule)
}
for _, protocol := range protocols {
protocolRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(protocol),
EtherType: s(string(rules.EtherType4)),
}
addDirectionalGroupRule(c, nodeSG, masterSG, protocolRule)
}
return nil
}
// addProtokubeRules - Add rules for protokube if gossip DNS is enabled
func (b *FirewallModelBuilder) addProtokubeRules(c *fi.ModelBuilderContext, sgMap map[string]*openstacktasks.SecurityGroup) error {
if dns.IsGossipHostname(b.ClusterName()) {
masterName := b.SecurityGroupName(kops.InstanceGroupRoleMaster)
nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
masterSG := sgMap[masterName]
nodeSG := sgMap[nodeName]
protokubeRule := &openstacktasks.SecurityGroupRule{
Lifecycle: b.Lifecycle,
Direction: s(string(rules.DirIngress)),
Protocol: s(string(rules.ProtocolTCP)),
EtherType: s(string(rules.EtherType4)),
PortRangeMin: i(3994),
PortRangeMax: i(3999),
}
addDirectionalGroupRule(c, masterSG, nodeSG, protokubeRule)
addDirectionalGroupRule(c, nodeSG, masterSG, protokubeRule)
addDirectionalGroupRule(c, masterSG, masterSG, protokubeRule)
addDirectionalGroupRule(c, nodeSG, nodeSG, protokubeRule)
}
return nil
}
// Build - schedule security groups and security group rule tasks for Openstack
func (b *FirewallModelBuilder) Build(c *fi.ModelBuilderContext) error {
roles := []kops.InstanceGroupRole{kops.InstanceGroupRoleMaster, kops.InstanceGroupRoleNode}
if b.UsesSSHBastion() {
roles = append(roles, kops.InstanceGroupRoleBastion)
}
sgMap := make(map[string]*openstacktasks.SecurityGroup)
if b.UseLoadBalancerForAPI() {
sg := &openstacktasks.SecurityGroup{
Name: s(b.Cluster.Spec.MasterPublicName),
Lifecycle: b.Lifecycle,
RemoveExtraRules: []string{"port=443"},
}
c.AddTask(sg)
sgMap[b.Cluster.Spec.MasterPublicName] = sg
}
for _, role := range roles {
// Create Security Group for Role
groupName := b.SecurityGroupName(role)
sg := &openstacktasks.SecurityGroup{
Name: s(groupName),
Lifecycle: b.Lifecycle,
}
if role == kops.InstanceGroupRoleBastion {
sg.RemoveExtraRules = []string{"port=22"}
} else if role == kops.InstanceGroupRoleNode {
sg.RemoveExtraRules = []string{"port=22", "port=10250"}
} else if role == kops.InstanceGroupRoleMaster {
sg.RemoveExtraRules = []string{"port=22", "port=443", "port=10250"}
}
c.AddTask(sg)
sgMap[groupName] = sg
}
//Add API Server Rules
b.addHTTPSRules(c, sgMap)
//Add SSH
b.addSSHRules(c, sgMap)
//Allow overlay DNS
b.addDNSRules(c, sgMap)
//Add Kubelet Rules
b.addKubeletRules(c, sgMap)
//Add Node exporter Rules
b.addNodeExporterRules(c, sgMap)
// Protokube Rules
b.addProtokubeRules(c, sgMap)
//Allow necessary local traffic
b.addCNIRules(c, sgMap)
//ETCD Leader Election
b.addETCDRules(c, sgMap)
// Add NodePort Rules:
err := b.addNodePortRules(c, sgMap)
if err != nil {
return err
}
return nil
}