Rework protokube dns so it shares code with dns-controller

dns-controller/pkg/dns/dnscontroller.go

@@ -32,6 +32,8 @@ import (
 	"k8s.io/kubernetes/federation/pkg/dnsprovider/rrstype"
 )
 
+const DefaultTTL = time.Minute
+
 // DNSController applies the desired DNS state to the DNS backend
 type DNSController struct {
 	zoneRules *ZoneRules
@@ -252,10 +254,10 @@ func (c *DNSController) runOnce() error {
 			continue
 		}
 
-		ttl := 60
+		ttl := DefaultTTL
-		glog.Infof("Using default TTL of %d seconds", ttl)
+		glog.Infof("Using default TTL of %v", ttl)
 
-		err := op.updateRecords(k, newValues, int64(ttl))
+		err := op.updateRecords(k, newValues, int64(ttl.Seconds()))
 		if err != nil {
 			glog.Infof("error updating records for %s: %v", k, err)
 			errors = append(errors, err)
@@ -362,10 +364,12 @@ func (o *dnsOp) findZone(fqdn string) dnsprovider.Zone {
 func (o *dnsOp) deleteRecords(k recordKey) error {
 	glog.V(2).Infof("Deleting all records for %s", k)
 
-	zone := o.findZone(k.FQDN)
+	fqdn := EnsureDotSuffix(k.FQDN)
+
+	zone := o.findZone(fqdn)
 	if zone == nil {
 		// TODO: Post event into service / pod
-		return fmt.Errorf("no suitable zone found for %q", k.FQDN)
+		return fmt.Errorf("no suitable zone found for %q", fqdn)
 	}
 
 	rrsProvider, ok := zone.ResourceRecordSets()
@@ -383,8 +387,8 @@ func (o *dnsOp) deleteRecords(k recordKey) error {
 	empty := true
 	for _, rr := range rrs {
 		rrName := EnsureDotSuffix(rr.Name())
-		if rrName != k.FQDN {
+		if rrName != fqdn {
-			glog.V(8).Infof("Skipping delete of record %q (name != %s)", rrName, k.FQDN)
+			glog.V(8).Infof("Skipping delete of record %q (name != %s)", rrName, fqdn)
 			continue
 		}
 		if string(rr.Type()) != string(k.RecordType) {
@@ -411,10 +415,12 @@ func (o *dnsOp) deleteRecords(k recordKey) error {
 func (o *dnsOp) updateRecords(k recordKey, newRecords []string, ttl int64) error {
 	glog.V(2).Infof("Updating records for %s: %v", k, newRecords)
 
-	zone := o.findZone(k.FQDN)
+	fqdn := EnsureDotSuffix(k.FQDN)
+
+	zone := o.findZone(fqdn)
 	if zone == nil {
 		// TODO: Post event into service / pod
-		return fmt.Errorf("no suitable zone found for %q", k.FQDN)
+		return fmt.Errorf("no suitable zone found for %q", fqdn)
 	}
 
 	rrsProvider, ok := zone.ResourceRecordSets()
@@ -429,12 +435,13 @@ func (o *dnsOp) updateRecords(k recordKey, newRecords []string, ttl int64) error
 
 	var existing dnsprovider.ResourceRecordSet
 	for _, rr := range rrs {
-		if rr.Name() != k.FQDN {
+		rrName := EnsureDotSuffix(rr.Name())
-			glog.V(8).Infof("Skipping record %q (name != %s)", rr.Name(), k.FQDN)
+		if rrName != fqdn {
+			glog.V(8).Infof("Skipping record %q (name != %s)", rrName, fqdn)
 			continue
 		}
 		if string(rr.Type()) != string(k.RecordType) {
-			glog.V(8).Infof("Skipping record %q (type %s != %s)", rr.Name(), rr.Type(), k.RecordType)
+			glog.V(8).Infof("Skipping record %q (type %s != %s)", rrName, rr.Type(), k.RecordType)
 			continue
 		}
 
@@ -451,11 +458,11 @@ func (o *dnsOp) updateRecords(k recordKey, newRecords []string, ttl int64) error
 	}
 
 	glog.V(2).Infof("Updating resource record %s %s", k, newRecords)
-	rr := rrsProvider.New(k.FQDN, newRecords, ttl, rrstype.RrsType(k.RecordType))
+	rr := rrsProvider.New(fqdn, newRecords, ttl, rrstype.RrsType(k.RecordType))
 	cs.Add(rr)
 
 	if err := cs.Apply(); err != nil {
-		return fmt.Errorf("error updating resource record %s %s: %v", k.FQDN, rr.Type(), err)
+		return fmt.Errorf("error updating resource record %s %s: %v", fqdn, rr.Type(), err)
 	}
 
 	return nil

protokube/cmd/protokube/main.go

@@ -20,22 +20,50 @@ import (
 	"flag"
 	"fmt"
 	"github.com/golang/glog"
+	"github.com/spf13/pflag"
+	"k8s.io/kops/dns-controller/pkg/dns"
 	"k8s.io/kops/protokube/pkg/protokube"
+	"k8s.io/kubernetes/federation/pkg/dnsprovider"
 	"net"
 	"os"
 	"strings"
+
+	// Load DNS plugins
+	_ "k8s.io/kubernetes/federation/pkg/dnsprovider/providers/aws/route53"
+	_ "k8s.io/kubernetes/federation/pkg/dnsprovider/providers/google/clouddns"
+)
+
+var (
+	flags = pflag.NewFlagSet("", pflag.ExitOnError)
+
+	// value overwritten during build. This can be used to resolve issues.
+	BuildVersion = "0.1"
 )
 
 func main() {
+	fmt.Printf("protokube version %s\n", BuildVersion)
+
+	err := run()
+	if err != nil {
+		glog.Errorf("Error: %v", err)
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
+
+func run() error {
+	dnsProviderId := "aws-route53"
+	flags.StringVar(&dnsProviderId, "dns", dnsProviderId, "DNS provider we should use (aws-route53, google-clouddns)")
+
+	var zones []string
+	flags.StringSliceVarP(&zones, "zone", "z", []string{}, "Configure permitted zones and their mappings")
+
 	master := false
 	flag.BoolVar(&master, "master", master, "Act as master")
 
 	containerized := false
 	flag.BoolVar(&containerized, "containerized", containerized, "Set if we are running containerized.")
 
-	dnsZoneName := ""
-	flag.StringVar(&dnsZoneName, "dns-zone-name", dnsZoneName, "Name of zone to use for DNS")
-
 	dnsInternalSuffix := ""
 	flag.StringVar(&dnsInternalSuffix, "dns-internal-suffix", dnsInternalSuffix, "DNS suffix for internal domain names")
 
@@ -45,20 +73,24 @@ func main() {
 	flagChannels := ""
 	flag.StringVar(&flagChannels, "channels", flagChannels, "channels to install")
 
+	// Trick to avoid 'logging before flag.Parse' warning
+	flag.CommandLine.Parse([]string{})
+
 	flag.Set("logtostderr", "true")
-	flag.Parse()
+
+	flags.AddGoFlagSet(flag.CommandLine)
+
+	flags.Parse(os.Args)
 
 	volumes, err := protokube.NewAWSVolumes()
 	if err != nil {
-		glog.Errorf("Error initializing AWS: %q", err)
+		return fmt.Errorf("Error initializing AWS: %q", err)
-		os.Exit(1)
 	}
 
 	if clusterID == "" {
 		clusterID = volumes.ClusterID()
 		if clusterID == "" {
-			glog.Errorf("cluster-id is required (cannot be determined from cloud)")
+			return fmt.Errorf("cluster-id is required (cannot be determined from cloud)")
-			os.Exit(1)
 		} else {
 			glog.Infof("Setting cluster-id from cloud: %s", clusterID)
 		}
@@ -75,11 +107,6 @@ func main() {
 		dnsInternalSuffix = "." + dnsInternalSuffix
 	}
 
-	if dnsZoneName == "" {
-		tokens := strings.Split(dnsInternalSuffix, ".")
-		dnsZoneName = strings.Join(tokens[len(tokens)-2:], ".")
-	}
-
 	// Get internal IP from cloud, to avoid problems if we're in a container
 	// TODO: Just run with --net=host ??
 	//internalIP, err := findInternalIP()
@@ -89,10 +116,34 @@ func main() {
 	//}
 	internalIP := volumes.InternalIP()
 
-	dns, err := protokube.NewRoute53DNSProvider(dnsZoneName)
+	var dnsScope dns.Scope
-	if err != nil {
+	var dnsController *dns.DNSController
-		glog.Errorf("Error initializing DNS: %q", err)
+	{
-		os.Exit(1)
+		dnsProvider, err := dnsprovider.GetDnsProvider(dnsProviderId, nil)
+		if err != nil {
+			return fmt.Errorf("Error initializing DNS provider %q: %v", dnsProviderId, err)
+		}
+		if dnsProvider == nil {
+			return fmt.Errorf("DNS provider %q could not be initialized", dnsProviderId)
+		}
+
+		zoneRules, err := dns.ParseZoneRules(zones)
+		if err != nil {
+			return fmt.Errorf("unexpected zone flags: %q", err)
+		}
+
+		dnsController, err = dns.NewDNSController(dnsProvider, zoneRules)
+		if err != nil {
+			return err
+		}
+
+		dnsScope, err = dnsController.CreateScope("protokube")
+		if err != nil {
+			return err
+		}
+
+		// We don't really use readiness - our records are simple
+		dnsScope.MarkReady()
 	}
 
 	rootfs := "/"
@@ -117,7 +168,7 @@ func main() {
 		//EtcdClusters   : fromVolume
 
 		ModelDir: modelDir,
-		DNS:      dns,
+		DNSScope: dnsScope,
 
 		Channels: channels,
 
@@ -125,10 +176,11 @@ func main() {
 	}
 	k.Init(volumes)
 
+	go dnsController.Run()
+
 	k.RunSyncLoop()
 
-	glog.Infof("Unexpected exit")
+	return fmt.Errorf("Unexpected exit")
-	os.Exit(1)
 }
 
 // TODO: run with --net=host ??

protokube/pkg/protokube/aws_dns.go

@@ -1,230 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package protokube
-
-import (
-	"fmt"
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/request"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/route53"
-	"github.com/golang/glog"
-	"reflect"
-	"strings"
-	"time"
-)
-
-type Route53DNSProvider struct {
-	client *route53.Route53
-
-	zoneName string
-	zone     *route53.HostedZone
-}
-
-func NewRoute53DNSProvider(zoneName string) (*Route53DNSProvider, error) {
-	if zoneName == "" {
-		return nil, fmt.Errorf("zone name is required")
-	}
-
-	p := &Route53DNSProvider{
-		zoneName: zoneName,
-	}
-
-	s := session.New()
-	s.Handlers.Send.PushFront(func(r *request.Request) {
-		// Log requests
-		glog.V(4).Infof("AWS API Request: %s/%s", r.ClientInfo.ServiceName, r.Operation.Name)
-	})
-
-	config := aws.NewConfig()
-	config = config.WithCredentialsChainVerboseErrors(true)
-
-	p.client = route53.New(s, config)
-
-	return p, nil
-}
-
-func (p *Route53DNSProvider) getZone() (*route53.HostedZone, error) {
-	if p.zone != nil {
-		return p.zone, nil
-	}
-
-	if !strings.Contains(p.zoneName, ".") {
-		// Looks like a zone ID
-		zoneID := p.zoneName
-		glog.Infof("Querying for hosted zone by id: %q", zoneID)
-
-		request := &route53.GetHostedZoneInput{
-			Id: aws.String(zoneID),
-		}
-
-		response, err := p.client.GetHostedZone(request)
-		if err != nil {
-			if AWSErrorCode(err) == "NoSuchHostedZone" {
-				glog.Infof("Zone not found with id %q; will reattempt by name", zoneID)
-			} else {
-				return nil, fmt.Errorf("error querying for DNS HostedZones %q: %v", zoneID, err)
-			}
-		} else {
-			p.zone = response.HostedZone
-			return p.zone, nil
-		}
-	}
-
-	glog.Infof("Querying for hosted zone by name: %q", p.zoneName)
-
-	findZone := p.zoneName
-	if !strings.HasSuffix(findZone, ".") {
-		findZone += "."
-	}
-	request := &route53.ListHostedZonesByNameInput{
-		DNSName: aws.String(findZone),
-	}
-
-	response, err := p.client.ListHostedZonesByName(request)
-	if err != nil {
-		return nil, fmt.Errorf("error querying for DNS HostedZones %q: %v", findZone, err)
-	}
-
-	var zones []*route53.HostedZone
-	for _, zone := range response.HostedZones {
-		if aws.StringValue(zone.Name) == findZone {
-			zones = append(zones, zone)
-		}
-	}
-	if len(zones) == 0 {
-		return nil, nil
-	}
-	if len(zones) != 1 {
-		return nil, fmt.Errorf("found multiple hosted zones matched name %q", findZone)
-	}
-
-	p.zone = zones[0]
-
-	return p.zone, nil
-}
-
-func (p *Route53DNSProvider) findResourceRecord(hostedZoneID string, name string, resourceType string) (*route53.ResourceRecordSet, error) {
-	name = strings.TrimSuffix(name, ".")
-
-	request := &route53.ListResourceRecordSetsInput{
-		HostedZoneId: aws.String(hostedZoneID),
-		// TODO: Start at correct name?
-	}
-
-	var found *route53.ResourceRecordSet
-
-	err := p.client.ListResourceRecordSetsPages(request, func(p *route53.ListResourceRecordSetsOutput, lastPage bool) (shouldContinue bool) {
-		for _, rr := range p.ResourceRecordSets {
-			if aws.StringValue(rr.Type) != resourceType {
-				continue
-			}
-
-			rrName := aws.StringValue(rr.Name)
-			rrName = strings.TrimSuffix(rrName, ".")
-
-			if name == rrName {
-				found = rr
-				break
-			}
-		}
-
-		// TODO: Also exit if we are on the 'next' name?
-
-		return found == nil
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("error listing DNS ResourceRecords: %v", err)
-	}
-
-	if found == nil {
-		return nil, nil
-	}
-
-	return found, nil
-}
-
-func (p *Route53DNSProvider) Set(fqdn string, recordType string, value string, ttl time.Duration) error {
-	zone, err := p.getZone()
-	if err != nil {
-		return err
-	}
-
-	// More correct, and makes the simple comparisons later on work correctly
-	if !strings.HasSuffix(fqdn, ".") {
-		fqdn += "."
-	}
-
-	existing, err := p.findResourceRecord(aws.StringValue(zone.Id), fqdn, recordType)
-	if err != nil {
-		return err
-	}
-
-	rrs := &route53.ResourceRecordSet{
-		Name: aws.String(fqdn),
-		Type: aws.String(recordType),
-		TTL:  aws.Int64(int64(ttl.Seconds())),
-		ResourceRecords: []*route53.ResourceRecord{
-			{Value: aws.String(value)},
-		},
-	}
-
-	if existing != nil {
-		if reflect.DeepEqual(rrs, existing) {
-			glog.V(2).Infof("DNS %q %s record already set to %q", fqdn, recordType, value)
-			return nil
-		} else {
-			glog.Infof("ResourceRecordSet change:")
-			glog.Infof("Existing: %v", DebugString(existing))
-			glog.Infof("Desired:  %v", DebugString(rrs))
-		}
-	}
-
-	change := &route53.Change{
-		Action:            aws.String("UPSERT"),
-		ResourceRecordSet: rrs,
-	}
-
-	changeBatch := &route53.ChangeBatch{}
-	changeBatch.Changes = []*route53.Change{change}
-
-	request := &route53.ChangeResourceRecordSetsInput{}
-	request.HostedZoneId = zone.Id
-	request.ChangeBatch = changeBatch
-
-	glog.V(2).Infof("Updating DNS record %q", fqdn)
-	glog.V(4).Infof("route53 request: %s", DebugString(request))
-
-	response, err := p.client.ChangeResourceRecordSets(request)
-	if err != nil {
-		return fmt.Errorf("error creating ResourceRecordSets: %v", err)
-	}
-
-	glog.V(2).Infof("Change id is %q", aws.StringValue(response.ChangeInfo.Id))
-
-	return nil
-}
-
-// AWSErrorCode returns the aws error code, if it is an awserr.Error, otherwise ""
-func AWSErrorCode(err error) string {
-	if awsError, ok := err.(awserr.Error); ok {
-		return awsError.Code()
-	}
-	return ""
-}

protokube/pkg/protokube/kube_boot.go

@@ -18,11 +18,11 @@ package protokube
 
 import (
 	"fmt"
+	"github.com/golang/glog"
+	"k8s.io/kops/dns-controller/pkg/dns"
 	"net"
 	"os/exec"
 	"time"
-
-	"github.com/golang/glog"
 )
 
 type KubeBoot struct {
@@ -35,7 +35,7 @@ type KubeBoot struct {
 	volumeMounter   *VolumeMountController
 	etcdControllers map[string]*EtcdController
 
-	DNS DNSProvider
+	DNSScope dns.Scope
 
 	ModelDir string
 

protokube/pkg/protokube/kube_dns.go

@@ -17,22 +17,28 @@ limitations under the License.
 package protokube
 
 import (
-	"fmt"
+	"github.com/golang/glog"
+	"k8s.io/kops/dns-controller/pkg/dns"
 	"time"
 )
 
 const defaultTTL = time.Minute
 
-type DNSProvider interface {
-	Set(fqdn string, recordType string, value string, ttl time.Duration) error
-}
-
 // CreateInternalDNSNameRecord maps a FQDN to the internal IP address of the current machine
 func (k *KubeBoot) CreateInternalDNSNameRecord(fqdn string) error {
-	err := k.DNS.Set(fqdn, "A", k.InternalIP.String(), defaultTTL)
+	ttl := defaultTTL
-	if err != nil {
+	if ttl != dns.DefaultTTL {
-		return fmt.Errorf("error configuring DNS name %q: %v", fqdn, err)
+		glog.Infof("Ignoring ttl %v for %q", ttl, fqdn)
 	}
+
+	var records []dns.Record
+	records = append(records, dns.Record{
+		RecordType: dns.RecordTypeA,
+		FQDN:       fqdn,
+		Value:      k.InternalIP.String(),
+	})
+	k.DNSScope.Replace(fqdn, records)
+
 	return nil
 }
 

upup/pkg/fi/cloudup/template_functions.go

@@ -211,7 +211,9 @@ func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) {
 	}
 	// permit wildcard updates
 	argv = append(argv, "--zone=*/*")
-	argv = append(argv, "-v=8")
+
+	// Verbose, but not crazy logging
+	argv = append(argv, "-v=2")
 
 	return argv, nil
 }

upup/pkg/fi/nodeup/protokube_flags.go

@@ -17,10 +17,13 @@ limitations under the License.
 package nodeup
 
 type ProtokubeFlags struct {
-	DNSZoneName   *string `json:"dnsZoneName,omitempty" flag:"dns-zone-name"`
+	Master        *bool `json:"master,omitempty" flag:"master"`
-	Master        *bool   `json:"master,omitempty" flag:"master"`
+	Containerized *bool `json:"containerized,omitempty" flag:"containerized"`
-	Containerized *bool   `json:"containerized,omitempty" flag:"containerized"`
+	LogLevel      *int  `json:"logLevel,omitempty" flag:"v"`
-	LogLevel      *int    `json:"logLevel,omitempty" flag:"v"`
+
+	DNSProvider *string `json:"dnsProvider,omitempty" flag:"dns"`
+
+	Zone []string `json:"zone,omitempty" flag:"zone"`
 
 	Channels []string `json:"channels,omitempty" flag:"channels"`
 }

upup/pkg/fi/nodeup/template_functions.go

@@ -279,8 +279,20 @@ func (t *templateFunctions) ProtokubeFlags() *ProtokubeFlags {
 
 	f.LogLevel = fi.Int(8)
 	f.Containerized = fi.Bool(true)
-	if t.cluster.Spec.DNSZone != "" {
+
-		f.DNSZoneName = fi.String(t.cluster.Spec.DNSZone)
+	zone := t.cluster.Spec.DNSZone
+	if zone != "" {
+		if strings.Contains(zone, ".") {
+			// match by name
+			f.Zone = append(f.Zone, zone)
+		} else {
+			// match by id
+			f.Zone = append(f.Zone, "*/"+zone)
+		}
+	} else {
+		glog.Warningf("DNSZone not specified; protokube won't be able to update DNS")
+		// TODO: Should we permit wildcard updates if zone is not specified?
+		//argv = append(argv, "--zone=*/*")
 	}
 
 	return f