diff --git a/Makefile b/Makefile index 212f769568..504dc5bc28 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ GOVERSION=1.8.1 MAKEDIR:=$(strip $(shell dirname "$(realpath $(lastword $(MAKEFILE_LIST)))")) # Keep in sync with upup/models/cloudup/resources/addons/dns-controller/ -DNS_CONTROLLER_TAG=1.6.1 +DNS_CONTROLLER_TAG=1.6.2 KOPS_RELEASE_VERSION = 1.6.1 KOPS_CI_VERSION = 1.6.2-beta.1 diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 267be585ff..9c7285b263 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -34,6 +34,7 @@ import ( "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/apis/kops/validation" "k8s.io/kops/pkg/client/simple/vfsclientset" + "k8s.io/kops/pkg/dns" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" @@ -802,7 +803,12 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e } else { switch cluster.Spec.Topology.Masters { case api.TopologyPublic: - cluster.Spec.API.DNS = &api.DNSAccessSpec{} + if dns.IsGossipHostname(cluster.Name) { + // gossip DNS names don't work outside the cluster, so we use a LoadBalancer instead + cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} + } else { + cluster.Spec.API.DNS = &api.DNSAccessSpec{} + } case api.TopologyPrivate: cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} diff --git a/hack/.packages b/hack/.packages index 15fb11cb13..2aec75856a 100644 --- a/hack/.packages +++ b/hack/.packages @@ -56,6 +56,7 @@ k8s.io/kops/pkg/util/stringorslice k8s.io/kops/pkg/validation k8s.io/kops/protokube/cmd/protokube k8s.io/kops/protokube/pkg/gossip +k8s.io/kops/protokube/pkg/gossip/aws k8s.io/kops/protokube/pkg/gossip/dns k8s.io/kops/protokube/pkg/gossip/dns/provider k8s.io/kops/protokube/pkg/gossip/gce diff --git a/pkg/model/awsmodel/api_loadbalancer.go b/pkg/model/awsmodel/api_loadbalancer.go index 40d0e0a3db..ca2a8a1932 100644 --- a/pkg/model/awsmodel/api_loadbalancer.go +++ b/pkg/model/awsmodel/api_loadbalancer.go @@ -24,8 +24,10 @@ import ( "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/dns" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" + "k8s.io/kops/upup/pkg/fi/fitasks" ) const LoadBalancerDefaultIdleTimeout = 5 * time.Minute @@ -185,6 +187,18 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(t) } + if dns.IsGossipHostname(b.Cluster.Name) { + // Ensure the ELB hostname is included in the TLS certificate, + // if we're not going to use an alias for it + // TODO: I don't love this technique for finding the task by name & modifying it + masterKeypairTask, found := c.Tasks["Keypair/master"] + if !found { + return fmt.Errorf("keypair/master task not found") + } + masterKeypair := masterKeypairTask.(*fitasks.Keypair) + masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, elb) + } + for _, ig := range b.MasterInstanceGroups() { t := &awstasks.LoadBalancerAttachment{ Name: s("api-" + ig.ObjectMeta.Name), diff --git a/pkg/model/dns.go b/pkg/model/dns.go index b86a5deadd..9525d9d541 100644 --- a/pkg/model/dns.go +++ b/pkg/model/dns.go @@ -33,6 +33,10 @@ type DNSModelBuilder struct { var _ fi.ModelBuilder = &DNSModelBuilder{} func (b *DNSModelBuilder) ensureDNSZone(c *fi.ModelBuilderContext) error { + if dns.IsGossipHostname(b.Cluster.Name) { + return nil + } + // Configuration for a DNS zone dnsZone := &awstasks.DNSZone{ Name: s(b.NameForDNSZone()), @@ -88,17 +92,19 @@ func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error { // This will point our DNS to the load balancer, and put the pieces // together for kubectl to be work - if err := b.ensureDNSZone(c); err != nil { - return err - } + if !dns.IsGossipHostname(b.Cluster.Name) { + if err := b.ensureDNSZone(c); err != nil { + return err + } - apiDnsName := &awstasks.DNSName{ - Name: s(b.Cluster.Spec.MasterPublicName), - Zone: b.LinkToDNSZone(), - ResourceType: s("A"), - TargetLoadBalancer: b.LinkToELB("api"), + apiDnsName := &awstasks.DNSName{ + Name: s(b.Cluster.Spec.MasterPublicName), + Zone: b.LinkToDNSZone(), + ResourceType: s("A"), + TargetLoadBalancer: b.LinkToELB("api"), + } + c.AddTask(apiDnsName) } - c.AddTask(apiDnsName) } if b.UsesBastionDns() { diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 5e9bca44b0..0cce87629f 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -182,6 +182,8 @@ func (b *IAMPolicyBuilder) BuildAWSIAMPolicy() (*IAMPolicy, error) { if b.HostedZoneID != "" { addRoute53Permissions(p, b.HostedZoneID) } + // dns-controller currently assumes it can list the hosted zones, even when using gossip + addRoute53ListHostedZonesPermission(p) // For S3 IAM permissions, we grant permissions to subtrees. So find the parents; // we don't need to grant mypath and mypath/child. @@ -279,7 +281,9 @@ func addRoute53Permissions(p *IAMPolicy, hostedZoneID string) { Action: stringorslice.Slice([]string{"route53:GetChange"}), Resource: stringorslice.Slice([]string{"arn:aws:route53:::change/*"}), }) +} +func addRoute53ListHostedZonesPermission(p *IAMPolicy) { wildcard := stringorslice.Slice([]string{"*"}) p.Statement = append(p.Statement, &IAMStatement{ Effect: IAMStatementEffectAllow, @@ -308,7 +312,11 @@ var _ fi.Resource = &IAMPolicyResource{} var _ fi.HasDependencies = &IAMPolicyResource{} func (b *IAMPolicyResource) GetDependencies(tasks map[string]fi.Task) []fi.Task { - return []fi.Task{b.DNSZone} + var deps []fi.Task + if b.DNSZone != nil { + deps = append(deps, b.DNSZone) + } + return deps } // Open produces the AWS IAM policy for the given role diff --git a/protokube/pkg/gossip/aws/seeds.go b/protokube/pkg/gossip/aws/seeds.go new file mode 100644 index 0000000000..dee0d73eef --- /dev/null +++ b/protokube/pkg/gossip/aws/seeds.go @@ -0,0 +1,73 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "k8s.io/kops/protokube/pkg/gossip" +) + +type SeedProvider struct { + ec2 ec2iface.EC2API + tags map[string]string +} + +var _ gossip.SeedProvider = &SeedProvider{} + +func (p *SeedProvider) GetSeeds() ([]string, error) { + request := &ec2.DescribeInstancesInput{} + for k, v := range p.tags { + filter := &ec2.Filter{ + Name: aws.String("tag:" + k), + Values: aws.StringSlice([]string{v}), + } + request.Filters = append(request.Filters, filter) + } + request.Filters = append(request.Filters, &ec2.Filter{ + Name: aws.String("instance-state-name"), + Values: aws.StringSlice([]string{"running", "pending"}), + }) + + var seeds []string + err := p.ec2.DescribeInstancesPages(request, func(p *ec2.DescribeInstancesOutput, lastPage bool) (shouldContinue bool) { + for _, r := range p.Reservations { + for _, i := range r.Instances { + ip := aws.StringValue(i.PrivateIpAddress) + if ip != "" { + seeds = append(seeds, ip) + } + } + } + return true + }) + + if err != nil { + return nil, fmt.Errorf("error querying for EC2 instances: %v", err) + } + + return seeds, nil +} + +func NewSeedProvider(ec2 ec2iface.EC2API, tags map[string]string) (*SeedProvider, error) { + return &SeedProvider{ + ec2: ec2, + tags: tags, + }, nil +} diff --git a/protokube/pkg/gossip/mesh/state.go b/protokube/pkg/gossip/mesh/state.go index a6d9765a78..48646c663a 100644 --- a/protokube/pkg/gossip/mesh/state.go +++ b/protokube/pkg/gossip/mesh/state.go @@ -90,8 +90,8 @@ func (s *state) snapshot() *gossip.GossipStateSnapshot { } s.lastSnapshot = snapshot return snapshot - } + func (s *state) put(key string, data []byte) { s.mtx.Lock() defer s.mtx.Unlock() @@ -102,6 +102,11 @@ func (s *state) put(key string, data []byte) { Data: data, Version: now, } + + if s.data.Records == nil { + s.data.Records = make(map[string]*KVStateRecord) + } + s.data.Records[key] = v s.version++ } @@ -116,6 +121,10 @@ func (s *state) updateValues(removeKeys []string, putEntries map[string]string) now := s.now() + if s.data.Records == nil { + s.data.Records = make(map[string]*KVStateRecord) + } + for _, k := range removeKeys { v := &KVStateRecord{ Tombstone: true, @@ -161,6 +170,14 @@ var _ mesh.GossipData = &KVState{} func mergeKVState(dest *KVState, src *KVState, changes *KVState) bool { changed := false + if dest.Records == nil { + dest.Records = make(map[string]*KVStateRecord) + } + + if changes != nil && changes.Records == nil { + changes.Records = make(map[string]*KVStateRecord) + } + for k, update := range src.Records { existing, found := dest.Records[k] if found && existing.Version >= update.Version { diff --git a/protokube/pkg/protokube/aws_volume.go b/protokube/pkg/protokube/aws_volume.go index 110b5aeb49..3d30a76e69 100644 --- a/protokube/pkg/protokube/aws_volume.go +++ b/protokube/pkg/protokube/aws_volume.go @@ -25,6 +25,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/golang/glog" "k8s.io/kops/protokube/pkg/gossip" + gossipaws "k8s.io/kops/protokube/pkg/gossip/aws" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "net" "strings" @@ -364,7 +365,10 @@ func (a *AWSVolumes) AttachVolume(volume *Volume) error { } func (a *AWSVolumes) GossipSeeds() (gossip.SeedProvider, error) { - return nil, fmt.Errorf("AWS seed provider not yet implemented") + tags := make(map[string]string) + tags[awsup.TagClusterName] = a.clusterTag + + return gossipaws.NewSeedProvider(a.ec2, tags) } func (a *AWSVolumes) InstanceID() string { diff --git a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template index ae265d4da7..6d7689511c 100644 --- a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template +++ b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template @@ -6,7 +6,7 @@ metadata: labels: k8s-addon: dns-controller.addons.k8s.io k8s-app: dns-controller - version: v1.6.1 + version: v1.6.2 spec: replicas: 1 selector: @@ -17,7 +17,7 @@ spec: labels: k8s-addon: dns-controller.addons.k8s.io k8s-app: dns-controller - version: v1.6.1 + version: v1.6.2 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' # For 1.6, we keep the old tolerations in case of a downgrade to 1.5 @@ -33,7 +33,7 @@ spec: serviceAccount: dns-controller containers: - name: dns-controller - image: kope/dns-controller:1.6.1 + image: kope/dns-controller:1.6.2 command: {{ range $arg := DnsControllerArgv }} - "{{ $arg }}" diff --git a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template index 50ca81d32d..43640ed109 100644 --- a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template +++ b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template @@ -6,7 +6,7 @@ metadata: labels: k8s-addon: dns-controller.addons.k8s.io k8s-app: dns-controller - version: v1.6.1 + version: v1.6.2 spec: replicas: 1 selector: @@ -17,7 +17,7 @@ spec: labels: k8s-addon: dns-controller.addons.k8s.io k8s-app: dns-controller - version: v1.6.1 + version: v1.6.2 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' scheduler.alpha.kubernetes.io/tolerations: '[{"key": "dedicated", "value": "master"}]' @@ -28,7 +28,7 @@ spec: hostNetwork: true containers: - name: dns-controller - image: {{ DnsControllerImage }}:1.6.1 + image: {{ DnsControllerImage }}:1.6.2 command: {{ range $arg := DnsControllerArgv }} - "{{ $arg }}" diff --git a/upup/pkg/fi/cloudup/awstasks/load_balancer.go b/upup/pkg/fi/cloudup/awstasks/load_balancer.go index 87c159cb29..12dd49e0d4 100644 --- a/upup/pkg/fi/cloudup/awstasks/load_balancer.go +++ b/upup/pkg/fi/cloudup/awstasks/load_balancer.go @@ -388,6 +388,26 @@ func (e *LoadBalancer) Find(c *fi.Context) (*LoadBalancer, error) { return actual, nil } +var _ fi.HasAddress = &LoadBalancer{} + +func (e *LoadBalancer) FindIPAddress(context *fi.Context) (*string, error) { + cloud := context.Cloud.(awsup.AWSCloud) + + lb, err := FindLoadBalancerByNameTag(cloud, fi.StringValue(e.Name)) + if err != nil { + return nil, err + } + if lb == nil { + return nil, nil + } + + lbDnsName := fi.StringValue(lb.DNSName) + if lbDnsName == "" { + return nil, nil + } + return &lbDnsName, nil +} + func (e *LoadBalancer) Run(c *fi.Context) error { // TODO: Make Normalize a standard method e.Normalize()