Merge pull request #3923 from justinsb/lockdown_etcd_peer_ports

Automatic merge from submit-queue.

Block etcd peer port from nodes

pkg/model/firewall.go

@@ -52,11 +52,9 @@ func (b *FirewallModelBuilder) Build(c *fi.ModelBuilderContext) error {
 }
 
 func (b *FirewallModelBuilder) buildNodeRules(c *fi.ModelBuilderContext) error {
-	name := "nodes." + b.ClusterName()
-
 	{
 		t := &awstasks.SecurityGroup{
-			Name:             s(name),
+			Name:             s(b.SecurityGroupName(kops.InstanceGroupRoleNode)),
 			Lifecycle:        b.Lifecycle,
 			VPC:              b.LinkToVPC(),
 			Description:      s("Security group for nodes"),
@@ -211,7 +209,16 @@ func (b *FirewallModelBuilder) applyNodeToMasterBlockSpecificPorts(c *fi.ModelBu
 
 	// TODO: Make less hacky
 	// TODO: Fix management - we need a wildcard matcher now
-	tcpRanges := []portRange{{From: 1, To: 4000}, {From: 4003, To: 65535}}
+	tcpBlocked := make(map[int]bool)
+
+	// Don't allow nodes to access etcd client port
+	tcpBlocked[4001] = true
+	tcpBlocked[4002] = true
+
+	// Don't allow nodes to access etcd peer port
+	tcpBlocked[2380] = true
+	tcpBlocked[2381] = true
+
 	udpRanges := []portRange{{From: 1, To: 65535}}
 	protocols := []Protocol{}
 
@@ -219,14 +226,14 @@ func (b *FirewallModelBuilder) applyNodeToMasterBlockSpecificPorts(c *fi.ModelBu
 		// Calico needs to access etcd
 		// TODO: Remove, replace with etcd in calico manifest
 		glog.Warningf("Opening etcd port on masters for access from the nodes, for calico.  This is unsafe in untrusted environments.")
-		tcpRanges = []portRange{{From: 1, To: 4001}, {From: 4003, To: 65535}}
+		tcpBlocked[4001] = false
 		protocols = append(protocols, ProtocolIPIP)
 	}
 
 	if b.Cluster.Spec.Networking.Romana != nil {
 		// Romana needs to access etcd
 		glog.Warningf("Opening etcd port on masters for access from the nodes, for romana.  This is unsafe in untrusted environments.")
-		tcpRanges = []portRange{{From: 1, To: 4001}, {From: 4003, To: 65535}}
+		tcpBlocked[4001] = false
 		protocols = append(protocols, ProtocolIPIP)
 	}
 
@@ -245,6 +252,21 @@ func (b *FirewallModelBuilder) applyNodeToMasterBlockSpecificPorts(c *fi.ModelBu
 			Protocol:      s("udp"),
 		})
 	}
+
+	tcpRanges := []portRange{
+		{From: 1, To: 0},
+	}
+	for port := 1; port < 65536; port++ {
+		previous := &tcpRanges[len(tcpRanges)-1]
+		if !tcpBlocked[port] {
+			if (previous.To + 1) == port {
+				previous.To = port
+			} else {
+				tcpRanges = append(tcpRanges, portRange{From: port, To: port})
+			}
+		}
+	}
+
 	for _, r := range tcpRanges {
 		c.AddTask(&awstasks.SecurityGroupRule{
 			Name:          s(fmt.Sprintf("node-to-master-tcp-%d-%d", r.From, r.To)),
@@ -277,18 +299,19 @@ func (b *FirewallModelBuilder) applyNodeToMasterBlockSpecificPorts(c *fi.ModelBu
 }
 
 func (b *FirewallModelBuilder) buildMasterRules(c *fi.ModelBuilderContext) error {
-	name := "masters." + b.ClusterName()
-
 	{
 		t := &awstasks.SecurityGroup{
-			Name:        s(name),
+			Name:        s(b.SecurityGroupName(kops.InstanceGroupRoleMaster)),
 			Lifecycle:   b.Lifecycle,
 			VPC:         b.LinkToVPC(),
 			Description: s("Security group for masters"),
 			RemoveExtraRules: []string{
 				"port=22",   // SSH
 				"port=443",  // k8s api
-				"port=4001", // etcd main (etcd events is 4002)
+				"port=2380", // etcd main peer
+				"port=2381", // etcd events peer
+				"port=4001", // etcd main
+				"port=4002", // etcd events
 				"port=4789", // VXLAN
 				"port=179",  // Calico
 

tests/integration/update_cluster/additional_user-data/cloudformation.json

@@ -266,7 +266,7 @@
         "CidrIp": "0.0.0.0/0"
       }
     },
-    "AWSEC2SecurityGroupIngressnodetomastertcp14000": {
+    "AWSEC2SecurityGroupIngressnodetomastertcp12379": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
@@ -276,6 +276,20 @@
           "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom"
         },
         "FromPort": 1,
+        "ToPort": 2379,
+        "IpProtocol": "tcp"
+      }
+    },
+    "AWSEC2SecurityGroupIngressnodetomastertcp23824000": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom"
+        },
+        "SourceSecurityGroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom"
+        },
+        "FromPort": 2382,
         "ToPort": 4000,
         "IpProtocol": "tcp"
       }

tests/integration/update_cluster/complex/kubernetes.tf

@@ -339,11 +339,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-complex-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-complex-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-complex-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-complex-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/ha/kubernetes.tf

@@ -481,11 +481,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-ha-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-ha-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-ha-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-ha-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/lifecycle_phases/security-kubernetes.tf

@@ -250,11 +250,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-privateweave-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-privateweave-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/minimal-141/kubernetes.tf

@@ -311,11 +311,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-minimal-141-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-minimal-141-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-minimal-141-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-minimal-141-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/minimal-cloudformation/cloudformation.json

@@ -266,7 +266,7 @@
         "CidrIp": "0.0.0.0/0"
       }
     },
-    "AWSEC2SecurityGroupIngressnodetomastertcp14000": {
+    "AWSEC2SecurityGroupIngressnodetomastertcp12379": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
@@ -276,6 +276,20 @@
           "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom"
         },
         "FromPort": 1,
+        "ToPort": 2379,
+        "IpProtocol": "tcp"
+      }
+    },
+    "AWSEC2SecurityGroupIngressnodetomastertcp23824000": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupId": {
+          "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom"
+        },
+        "SourceSecurityGroupId": {
+          "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom"
+        },
+        "FromPort": 2382,
         "ToPort": 4000,
         "IpProtocol": "tcp"
       }

tests/integration/update_cluster/minimal/kubernetes.tf

@@ -311,11 +311,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-minimal-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-minimal-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-minimal-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-minimal-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/privatecalico/kubernetes.tf

@@ -591,11 +591,20 @@ resource "aws_security_group_rule" "node-to-master-protocol-ipip" {
   protocol                 = "4"
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4001" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-privatecalico-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-privatecalico-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4001" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-privatecalico-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-privatecalico-example-com.id}"
+  from_port                = 2382
   to_port                  = 4001
   protocol                 = "tcp"
 }

tests/integration/update_cluster/privatecanal/kubernetes.tf

@@ -582,11 +582,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-privatecanal-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-privatecanal-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-privatecanal-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-privatecanal-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/privatedns1/kubernetes.tf

@@ -587,11 +587,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-privatedns1-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-privatedns1-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-privatedns1-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-privatedns1-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/privatedns2/kubernetes.tf

@@ -573,11 +573,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-privatedns2-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-privatedns2-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-privatedns2-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-privatedns2-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/privateflannel/kubernetes.tf

@@ -582,11 +582,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-privateflannel-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-privateflannel-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-privateflannel-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-privateflannel-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/privatekopeio/kubernetes.tf

@@ -573,11 +573,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-privatekopeio-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-privatekopeio-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-privatekopeio-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-privatekopeio-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/privateweave/kubernetes.tf

@@ -582,11 +582,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-privateweave-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-privateweave-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/shared_subnet/kubernetes.tf

@@ -286,11 +286,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-sharedsubnet-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-sharedsubnet-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-sharedsubnet-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-sharedsubnet-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }

tests/integration/update_cluster/shared_vpc/kubernetes.tf

@@ -302,11 +302,20 @@ resource "aws_security_group_rule" "node-egress" {
   cidr_blocks       = ["0.0.0.0/0"]
 }
 
-resource "aws_security_group_rule" "node-to-master-tcp-1-4000" {
+resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
   type                     = "ingress"
   security_group_id        = "${aws_security_group.masters-sharedvpc-example-com.id}"
   source_security_group_id = "${aws_security_group.nodes-sharedvpc-example-com.id}"
   from_port                = 1
+  to_port                  = 2379
+  protocol                 = "tcp"
+}
+
+resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
+  type                     = "ingress"
+  security_group_id        = "${aws_security_group.masters-sharedvpc-example-com.id}"
+  source_security_group_id = "${aws_security_group.nodes-sharedvpc-example-com.id}"
+  from_port                = 2382
   to_port                  = 4000
   protocol                 = "tcp"
 }