Rework how the expiration mailer looks for certificates

cmd/expiration-mailer/main.go

@@ -6,13 +6,18 @@
 package main
 
 import (
+	"bytes"
+	"crypto/x509"
 	"database/sql"
 	"fmt"
+	"io/ioutil"
+	"strings"
+	"text/template"
 	"time"
 
-	"github.com/cactus/go-statsd-client/statsd"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
-	"github.com/codegangsta/cli"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/codegangsta/cli"
-	"gopkg.in/gorp.v1"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1"
 
 	"github.com/letsencrypt/boulder/cmd"
 	"github.com/letsencrypt/boulder/core"
@@ -21,63 +26,114 @@ import (
 	"github.com/letsencrypt/boulder/sa"
 )
 
-type mailer struct {
+type emailContent struct {
-	stats statsd.Statter
+	ExpirationDate   time.Time
-	log   *blog.AuditLogger
+	DaysToExpiration int
-	dbMap *gorp.DbMap
+	CommonName       string
-
+	DNSNames         string
-	Mailer *mail.Mailer
-}
-
-func (m *mailer) findExpiringCertificates(warningDays []int) error {
-	var err error
-	// E.g. warningDays = [0, 1, 3, 7, 14] days from expiration
-	for _, expiresIn := range warningDays {
-		left := time.Now().Add(time.Hour * 24 * time.Duration(expiresIn))
-		right := left.Add(time.Hour * 24)
-
-		var certs []core.Certificate
-		_, err := m.dbMap.Select(
-			&certs,
-			`SELECT * FROM certificates
-       WHERE expires > :cutoff-a AND Expires < :cutoff-b AND status != "revoked"
-       ORDER BY issued ASC`,
-			map[string]interface{}{
-				"cutoff-a": left,
-				"cutoff-b": right,
-			},
-		)
-		if err == sql.ErrNoRows {
-			m.log.Info("All up to date. No expiration emails needed.")
-		} else if err != nil {
-			m.log.Err(fmt.Sprintf("Error loading certificates: %s", err))
-		} else {
-			// Do things...
-			// cert expires in expiresIn, send email to registration contacts
-			for _, cert := range certs {
-				reg, err := m.dbMap.Get(&core.Registration{}, cert.RegistrationID)
-				if err != nil {
-					return err
-				}
-
-				go m.sendWarning(cert, reg, expiresIn)
-			}
-		}
-	}
-
-	return err
 }
 
 const warningTemplate = `Hello,
 
-Your certificate for %s is going to expire in %d days (%s), make sure you run the
+Your certificate for common name {{.CommonName}} (and DNSNames {{.DNSNames}}) is
-renewer before then!
+going to expire in {{.DaysToExpiration}} days ({{.ExpirationDate}}), make sure you
+run the renewer before then!
 
 Regards,
 letsencryptbot
 `
 
-func (m *mailer) sendWarning(cert core.Certificate, reg core.Registration, expiresIn int) {
+type mailer struct {
+	stats         statsd.Statter
+	log           *blog.AuditLogger
+	dbMap         *gorp.DbMap
+	Mailer        *mail.Mailer
+	EmailTemplate *template.Template
+	WarningDays   []int
+}
+
+func (m *mailer) findExpiringCertificates() error {
+	var err error
+	now := time.Now()
+	// E.g. m.WarningDays = [1, 3, 7, 14] days from expiration
+	for i, expiresIn := range m.WarningDays {
+		left := now
+		if i > 0 {
+			left = left.Add(time.Hour * 24 * time.Duration(m.WarningDays[i-1]))
+		}
+		right := now.Add(time.Hour * 24 * time.Duration(expiresIn))
+
+		m.log.Info(fmt.Sprintf("expiration-mailer: Searching for certificates that expire between %s and %s", left, right))
+		var certs []core.Certificate
+		_, err := m.dbMap.Select(
+			&certs,
+			`SELECT cert.* FROM certificates AS cert JOIN certificateStatus AS cs on cs.serial = cert.serial
+       WHERE cert.expires > :cutoff-a AND cert.expires < :cutoff-b AND cs.expirationNagsSent < :nags AND cert.status != "revoked"
+       ORDER BY cert.expires ASC`,
+			map[string]interface{}{
+				"cutoff-a": left,
+				"cutoff-b": right,
+				"nags":     len(m.WarningDays) - i,
+			},
+		)
+		if err == sql.ErrNoRows {
+			m.log.Info("expiration-mailer: None found, No expiration emails needed.")
+			continue
+		} else if err != nil {
+			m.log.Err(fmt.Sprintf("expiration-mailer: Error loading certificates: %s", err))
+			continue
+		}
+
+		m.log.Info(fmt.Sprintf("expiration-mailer: Found %d certificates, starting sending messages", len(certs)))
+		for _, cert := range certs {
+			regObj, err := m.dbMap.Get(&core.Registration{}, cert.RegistrationID)
+			if err != nil {
+				return err
+			}
+			reg := regObj.(core.Registration)
+			parsedCert, err := x509.ParseCertificate(cert.DER)
+			if err != nil {
+				return err
+			}
+			err = m.sendWarning(parsedCert, reg)
+			if err != nil {
+				return err
+			}
+
+			// Update CertificateStatus object
+			tx, err := m.dbMap.Begin()
+			if err != nil {
+				// BAD
+				tx.Rollback()
+			}
+
+			csObj, err := tx.Get(&core.CertificateStatus{}, core.SerialToString(parsedCert.SerialNumber))
+			if err != nil {
+				// BAD
+				tx.Rollback()
+			}
+			certStatus := csObj.(core.CertificateStatus)
+			certStatus.ExpirationNagsSent = len(m.WarningDays) - i
+
+			_, err = tx.Update(certStatus)
+			if err != nil {
+				// BAD
+				tx.Rollback()
+			}
+
+			err = tx.Commit()
+			if err != nil {
+				// BAD
+				tx.Rollback()
+			}
+		}
+		m.log.Info("expiration-mailer: Finished sending messages")
+	}
+
+	return err
+}
+
+func (m *mailer) sendWarning(parsedCert *x509.Certificate, reg core.Registration) error {
 	emails := []string{}
 	for _, contact := range reg.Contact {
 		if contact.Scheme == "mailto" {
@@ -85,32 +141,44 @@ func (m *mailer) sendWarning(cert core.Certificate, reg core.Registration, expir
 		}
 	}
 	if len(emails) > 0 {
-		err = m.Mailer.SendMail(emails, fmt.Sprintf(warningTemplate, cert.CommonName, expiresIn, cert.NotAfter))
+		expiresIn := int(time.Now().Sub(parsedCert.NotAfter).Hours() / 24)
-		if err != nil {
+		email := emailContent{
-			m.log.WarningErr(err)
+			ExpirationDate:   parsedCert.NotAfter,
-			return
+			DaysToExpiration: expiresIn,
+			CommonName:       parsedCert.Subject.CommonName,
+			DNSNames:         strings.Join(parsedCert.DNSNames, ", "),
 		}
-		m.stats.Inc("Mailer.Expiration.Sent", len(emails))
+		msgBuf := new(bytes.Buffer)
+		err := m.EmailTemplate.Execute(msgBuf, email)
+		if err != nil {
+			return err
+		}
+		err = m.Mailer.SendMail(emails, msgBuf.String())
+		if err != nil {
+			return err
+		}
+		m.stats.Inc("Mailer.Expiration.Sent", int64(len(emails)), 1.0)
 	}
+	return nil
 }
 
 func main() {
 	app := cmd.NewAppShell("expiration-mailer")
 
 	app.App.Flags = append(app.App.Flags, cli.IntFlag{
-		Name:   "limit",
+		Name:   "message_limit",
-		Value:  emailLimit,
 		EnvVar: "EMAIL_LIMIT",
 		Usage:  "Maximum number of emails to send per run",
 	})
 
 	app.Config = func(c *cli.Context, config cmd.Config) cmd.Config {
-		config.Mailer.Limit = c.GlobalInt("emailLimit")
+		if c.GlobalInt("emailLimit") > 0 {
+			config.Mailer.MessageLimit = c.GlobalInt("emailLimit")
+		}
 		return config
 	}
 
 	app.Action = func(c cmd.Config) {
-		auditlogger.Info(app.VersionString())
 
 		// Set up logging
 		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
@@ -124,15 +192,35 @@ func main() {
 
 		blog.SetAuditLogger(auditlogger)
 
+		auditlogger.Info(app.VersionString())
+
 		go cmd.DebugServer(c.Mailer.DebugAddr)
 
 		// Configure DB
 		dbMap, err := sa.NewDbMap(c.Mailer.DBDriver, c.Mailer.DBConnect)
 		cmd.FailOnError(err, "Could not connect to database")
 
-		err = findExpiringCertificates()
+		// Load email template
-		if err != nil {
+		emailTmpl, err := ioutil.ReadFile(c.Mailer.EmailTemplate)
-			auditlogger.WarningErr(err)
+		cmd.FailOnError(err, fmt.Sprintf("Could not read email template file [%s]", c.Mailer.EmailTemplate))
+		tmpl, err := template.New("expiry-email").Parse(string(emailTmpl))
+		cmd.FailOnError(err, "Could not parse email template")
+
+		mailClient := mail.NewMailer(c.Mailer.Server, c.Mailer.Port, c.Mailer.Username, c.Mailer.Password)
+
+		m := mailer{
+			stats:         stats,
+			log:           auditlogger,
+			dbMap:         dbMap,
+			Mailer:        &mailClient,
+			EmailTemplate: tmpl,
+			WarningDays:   c.Mailer.ExpiryWarnings,
 		}
+
+		auditlogger.Info("expiration-mailer: Starting")
+		err = m.findExpiringCertificates()
+		cmd.FailOnError(err, "Could not connect to database")
 	}
+
+	app.Run()
 }

cmd/shell.go

@@ -127,14 +127,19 @@ type Config struct {
 		DBConnect string
 	}
 
-	Mail struct {
+	Mailer struct {
 		Server   string
 		Port     string
 		Username string
 		Password string
 
+		DBDriver  string
+		DBConnect string
+
 		MessageLimit   int
 		ExpiryWarnings []int
+		// Path to a text/template email template
+		EmailTemplate string
 
 		// DebugAddr is the address to run the /debug handlers on.
 		DebugAddr string

core/objects.go

@@ -11,12 +11,13 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	jose "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/square/go-jose"
 	"net"
 	"path/filepath"
 	"sort"
 	"strings"
 	"time"
+
+	jose "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/square/go-jose"
 )
 
 // AcmeStatus defines the state of a given authorization
@@ -555,6 +556,8 @@ type CertificateStatus struct {
 	//   code for 'unspecified').
 	RevokedReason int `db:"revokedReason"`
 
+	ExpirationNagsSent int `db:"expirationNagsSent"`
+
 	LockCol int64 `json:"-"`
 }
 

mail/mailer.go

@@ -6,6 +6,7 @@
 package mail
 
 import (
+	"net"
 	"net/smtp"
 )
 
@@ -17,7 +18,7 @@ type Mailer struct {
 	From   string
 }
 
-// NewMailer constructs a Mailer to represent an account at a particular mail
+// NewMailer constructs a Mailer to represent an account on a particular mail
 // transfer agent.
 func NewMailer(server, port, username, password string) Mailer {
 	auth := smtp.PlainAuth("", username, password, server)
@@ -32,6 +33,6 @@ func NewMailer(server, port, username, password string) Mailer {
 // SendMail sends an email to the provided list of recipients. The email body
 // is simple text.
 func (m *Mailer) SendMail(to []string, msg string) (err error) {
-	err = smtp.SendMail(m.Server+":"+m.Port, m.Auth, m.From, to, []byte(msg))
+	err = smtp.SendMail(net.JoinHostPort(m.Server, m.Port), m.Auth, m.From, to, []byte(msg))
 	return
 }

test/boulder-test-config.json

@@ -122,11 +122,17 @@
     "CreateTables": true
   },
 
-  "mail": {
+  "mailer": {
     "server": "mail.example.com",
     "port": "25",
     "username": "cert-master@example.com",
-    "password": "password"
+    "password": "password",
+    "dbDriver": "sqlite3",
+    "dbConnect": ":memory:",
+    "messageLimit": 0,
+    "expiryWarnings": [1, 3, 7, 14],
+    "EmailTemplate": "test/example-expiration-template",
+    "debugAddr": "localhost:8004"
   },
 
   "common": {