letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1480 from letsencrypt/exact-name-rl 

Exact name set rate limit
This commit is contained in:
Roland Bracewell Shoemaker 2016-02-29 13:58:08 -08:00
parent 632d2eb9c9 ee77c68549
commit f568f63f5d
18 changed files with 459 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
cmd/nameset-backfill/main.go Normal file
View File

@ -0,0 +1,180 @@
package main
import (
"crypto/sha256"
"crypto/x509"
"flag"
"fmt"
"strings"
"time"
"github.com/letsencrypt/boulder/cmd"
"github.com/letsencrypt/boulder/core"
blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/rpc"
"github.com/letsencrypt/boulder/sa"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1"
)
type resultHolder struct {
Serial string
Issued time.Time
Expires time.Time
DER []byte
}
type backfiller struct {
sa core.StorageAuthority
dbMap *gorp.DbMap
stats statsd.Statter
log *blog.AuditLogger
clk clock.Clock
}
func new(amqpConf *cmd.AMQPConfig, syslogConf cmd.SyslogConfig, statsdURI, dbURI string) (*backfiller, error) {
var stats statsd.Statter
var err error
stats, log := cmd.StatsAndLogging(cmd.StatsdConfig{Server: statsdURI, Prefix: "Boulder"}, syslogConf)
sac, err := rpc.NewStorageAuthorityClient("nameset-backfiller", amqpConf, stats)
if err != nil {
return nil, err
}
dbMap, err := sa.NewDbMap(dbURI)
if err != nil {
return nil, err
}
return &backfiller{sac, dbMap, stats, log, clock.Default()}, nil
}
func (b *backfiller) run() error {
added := 0
defer b.log.Info(fmt.Sprintf("Added %d missing certificate name sets to the fqdnSets table", added))
for {
results, err := b.findCerts()
if err != nil {
return err
}
if len(results) == 0 {
break
}
err = b.processResults(results)
if err != nil {
return err
}
added += len(results)
}
return nil
}
func (b *backfiller) findCerts() ([]resultHolder, error) {
var allResults []resultHolder
for {
var results []resultHolder
_, err := b.dbMap.Select(
&results,
`SELECT c.serial, c.issued, c.expires, c.der FROM certificates AS c
LEFT JOIN fqdnSets AS ns ON c.serial=ns.serial
WHERE ns.serial IS NULL
ORDER BY c.issued DESC
LIMIT ?
OFFSET ?`,
1000,
len(allResults),
)
if err != nil {
return nil, err
}
if len(results) == 0 {
break
}
b.stats.Inc("db-backfill.fqdnSets.missing-found", int64(len(results)), 1.0)
allResults = append(allResults, results...)
}
return allResults, nil
}
func hashNames(names []string) []byte {
names = core.UniqueLowerNames(names)
hash := sha256.Sum256([]byte(strings.Join(names, ",")))
return hash[:]
}
func (b *backfiller) processResults(results []resultHolder) error {
numResults := len(results)
added := 0
for _, r := range results {
c, err := x509.ParseCertificate(r.DER)
if err != nil {
b.log.Err(fmt.Sprintf("Failed to parse certificate [serial: %s] retrieved from database: %s", r.Serial, err))
continue
}
err = b.dbMap.Insert(&core.FQDNSet{
SetHash: hashNames(c.DNSNames),
Serial: r.Serial,
Issued: r.Issued,
Expires: r.Expires,
})
if err != nil {
b.log.Err(fmt.Sprintf("Failed to add name set for %s to database: %s", r.Serial, err))
continue
}
added++
b.stats.Inc("db-backfill.fqdnSets.added", 1, 1.0)
}
if added < numResults {
return fmt.Errorf("Didn't add all name sets, %d out of %d failed", numResults-added, numResults)
}
return nil
}
func main() {
amqpURI := flag.String("amqpURI", "", "AMQP connection URI")
amqpURIFile := flag.String("amqpURIFile", "", "File to read AMQP connection URI from")
amqpCert := flag.String("amqpCert", "", "AMQP client certificate to use")
amqpKey := flag.String("amqpKey", "", "Key for AMQP client certificate")
amqpCA := flag.String("amqpCA", "", "Root CA to trust for AMQP connections")
statsdURI := flag.String("statsdURI", "", "StatsD URI")
dbConnect := flag.String("dbConnect", "", "DB connection URI")
dbConnectFile := flag.String("dbConnectFile", "", "File to read DB connection URI from")
syslogNet := flag.String("syslogNetwork", "", "Syslog network")
syslogURI := flag.String("syslogServer", "", "Syslog URI")
syslogLevel := flag.Int("syslogLevel", 7, "Level at which to log")
flag.Parse()
dbConf := cmd.DBConfig{DBConnect: *dbConnect, DBConnectFile: *dbConnectFile}
dbURI, err := dbConf.URL()
amqpConf := &cmd.AMQPConfig{
Server: *amqpURI,
ServerURLFile: *amqpURIFile,
SA: &cmd.RPCServerConfig{
Server: "SA.server",
RPCTimeout: cmd.ConfigDuration{Duration: time.Second * 15},
},
}
if *amqpCert != "" && *amqpKey != "" && *amqpCA != "" {
amqpConf.TLS = &cmd.TLSConfig{CertFile: amqpCert, KeyFile: amqpKey, CACertFile: amqpCA}
} else {
amqpConf.Insecure = true
}
cmd.FailOnError(err, "Failed to read db URI")
b, err := new(
amqpConf,
cmd.SyslogConfig{
Network: *syslogNet,
Server: *syslogURI,
StdoutLevel: syslogLevel,
},
*statsdURI,
dbURI,
)
cmd.FailOnError(err, "Failed to create backfiller")
err = b.run()
cmd.FailOnError(err, "Failed to backfill fqdnSets table")
}

55
cmd/nameset-backfill/main_test.go Normal file
View File

@ -0,0 +1,55 @@
package main
import (
"io/ioutil"
"testing"
"time"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/core"
blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/sa"
"github.com/letsencrypt/boulder/sa/satest"
"github.com/letsencrypt/boulder/test"
"github.com/letsencrypt/boulder/test/vars"
)
func TestBackfill(t *testing.T) {
stats, _ := statsd.NewNoopClient()
// Create an SA
dbMap, err := sa.NewDbMap(vars.DBConnSA)
if err != nil {
t.Fatalf("Failed to create dbMap: %s", err)
}
fc := clock.NewFake()
fc.Add(1 * time.Hour)
sa, err := sa.NewSQLStorageAuthority(dbMap, fc)
if err != nil {
t.Fatalf("Failed to create SA: %s", err)
}
defer test.ResetSATestDatabase(t)
b := backfiller{sa, dbMap, stats, blog.GetAuditLogger(), fc}
certDER, err := ioutil.ReadFile("test-cert.der")
test.AssertNotError(t, err, "Couldn't read example cert DER")
reg := satest.CreateWorkingRegistration(t, sa)
err = dbMap.Insert(&core.Certificate{RegistrationID: reg.ID, DER: certDER, Serial: "serial"})
test.AssertNotError(t, err, "Couldn't insert stub certificate")
results, err := b.findCerts()
test.AssertNotError(t, err, "Failed to find missing name sets")
test.AssertEquals(t, len(results), 1)
test.AssertEquals(t, results[0].Serial, "serial")
err = b.processResults(results)
test.AssertNotError(t, err, "Failed to add missing name sets")
results, err = b.findCerts()
test.AssertNotError(t, err, "Failed to find missing name sets")
test.AssertEquals(t, len(results), 0)
}

BIN
cmd/nameset-backfill/test-cert.der Normal file
View File

Binary file not shown.

3
cmd/rate-limits.go
View File

@ -24,6 +24,9 @@ type RateLimitConfig struct {
// Number of pending authorizations that can exist per account. Overrides by
// key are not applied, but overrides by registration are.
PendingAuthorizationsPerAccount RateLimitPolicy `yaml:"pendingAuthorizationsPerAccount"`
// Number of certificates that can be extant containing a specific set
// of DNS names.
CertificatesPerFQDNSet RateLimitPolicy `yaml:"certificatesPerFQDNSet"`
}
// RateLimitPolicy describes a general limiting policy

1
core/interfaces.go
View File

@ -105,6 +105,7 @@ type StorageGetter interface {
CountRegistrationsByIP(net.IP, time.Time, time.Time) (int, error)
CountPendingAuthorizations(regID int64) (int, error)
GetSCTReceipt(string, string) (SignedCertificateTimestamp, error)
CountFQDNSets(time.Duration, []string) (int64, error)
}
// StorageAdder are the Boulder SA's write/update methods

10
core/objects.go
View File

@ -640,3 +640,13 @@ var RevocationReasons = map[RevocationCode]string{
9: "privilegeWithdrawn",
10: "aAcompromise",
}
// FQDNSet contains the SHA256 hash of the lowercased, comma joined dNSNames
// contained in a certificate.
type FQDNSet struct {
ID int64
SetHash []byte
Serial string
Issued time.Time
Expires time.Time
}

5
core/util.go
View File

@ -24,6 +24,7 @@ import (
"net/http"
"net/url"
"regexp"
"sort"
"strings"
"time"
@ -339,7 +340,8 @@ func GetBuildHost() (retID string) {
}
// UniqueLowerNames returns the set of all unique names in the input after all
// of them are lowercased. The returned names will be in their lowercased form.
// of them are lowercased. The returned names will be in their lowercased form
// and sorted alphabetically.
func UniqueLowerNames(names []string) (unique []string) {
nameMap := make(map[string]int, len(names))
for _, name := range names {
@ -350,6 +352,7 @@ func UniqueLowerNames(names []string) (unique []string) {
for name := range nameMap {
unique = append(unique, name)
}
sort.Strings(unique)
return
}

4
core/util_test.go
View File

@ -109,9 +109,9 @@ func TestAcmeURL(t *testing.T) {
}
func TestUniqueLowerNames(t *testing.T) {
u := UniqueLowerNames([]string{"foobar.com", "fooBAR.com", "baz.com", "foobar.com", "bar.com", "bar.com"})
u := UniqueLowerNames([]string{"foobar.com", "fooBAR.com", "baz.com", "foobar.com", "bar.com", "bar.com", "a.com"})
sort.Strings(u)
test.AssertDeepEquals(t, []string{"bar.com", "baz.com", "foobar.com"}, u)
test.AssertDeepEquals(t, []string{"a.com", "bar.com", "baz.com", "foobar.com"}, u)
}
func TestUnmarshalAcmeURL(t *testing.T) {

5
mocks/mocks.go
View File

@ -253,6 +253,11 @@ func (sa *StorageAuthority) AddSCTReceipt(sct core.SignedCertificateTimestamp) (
return
}
// CountFQDNSets is a mock
func (sa *StorageAuthority) CountFQDNSets(since time.Duration, names []string) (int64, error) {
return 0, nil
}
// GetLatestValidAuthorization is a mock
func (sa *StorageAuthority) GetLatestValidAuthorization(registrationID int64, identifier core.AcmeIdentifier) (authz core.Authorization, err error) {
if registrationID == 1 && identifier.Type == "dns" {

21
ra/registration-authority.go
View File

@ -640,6 +640,21 @@ func (ra *RegistrationAuthorityImpl) checkCertificatesPerNameLimit(names []strin
return nil
}
func (ra *RegistrationAuthorityImpl) checkCertificatesPerFQDNSetLimit(names []string, limit cmd.RateLimitPolicy, regID int64) error {
count, err := ra.SA.CountFQDNSets(limit.Window.Duration, names)
if err != nil {
return err
}
names = core.UniqueLowerNames(names)
if int(count) > limit.GetThreshold(strings.Join(names, ","), regID) {
return core.RateLimitedError(fmt.Sprintf(
"Too many certificates already issued for exact set of domains: %s",
strings.Join(names, ","),
))
}
return nil
}
func (ra *RegistrationAuthorityImpl) checkLimits(names []string, regID int64) error {
limits := ra.rlPolicies
if limits.TotalCertificates.Enabled() {
@ -661,6 +676,12 @@ func (ra *RegistrationAuthorityImpl) checkLimits(names []string, regID int64) er
return err
}
}
if limits.CertificatesPerFQDNSet.Enabled() {
err := ra.checkCertificatesPerFQDNSetLimit(names, limits.CertificatesPerFQDNSet, regID)
if err != nil {
return err
}
}
return nil
}

50
rpc/rpc-wrappers.go
View File

@ -70,6 +70,7 @@ const (
MethodAddSCTReceipt = "AddSCTReceipt" // SA
MethodSubmitToCT = "SubmitToCT" // Pub
MethodRevokeAuthorizationsByDomain = "RevokeAuthorizationsByDomain" // SA
MethodCountFQDNSets = "CountFQDNSets" // SA
)
// Request structs
@ -169,6 +170,11 @@ type revokeAuthsRequest struct {
Ident core.AcmeIdentifier
}
type countFQDNsRequest struct {
Window time.Duration
Names []string
}
// Response structs
type caaResponse struct {
Present bool
@ -181,6 +187,10 @@ type revokeAuthsResponse struct {
PendingRevoked int64
}
type countFQDNSetsResponse struct {
Count int64
}
func improperMessage(method string, err error, obj interface{}) {
log := blog.GetAuditLogger()
log.AuditErr(fmt.Errorf("Improper message. method: %s err: %s data: %+v", method, err, obj))
@ -1107,6 +1117,31 @@ func NewStorageAuthorityServer(rpc Server, impl core.StorageAuthority) error {
return nil, nil
})
rpc.Handle(MethodCountFQDNSets, func(req []byte) (response []byte, err error) {
var r countFQDNsRequest
err = json.Unmarshal(req, &r)
if err != nil {
// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
errorCondition(MethodCountFQDNSets, err, req)
return
}
count, err := impl.CountFQDNSets(r.Window, r.Names)
if err != nil {
// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
errorCondition(MethodCountFQDNSets, err, req)
return
}
response, err = json.Marshal(countFQDNSetsResponse{count})
if err != nil {
// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
errorCondition(MethodCountFQDNSets, err, req)
return
}
return
})
return nil
}
@ -1480,3 +1515,18 @@ func (cac StorageAuthorityClient) AddSCTReceipt(sct core.SignedCertificateTimest
_, err = cac.rpc.DispatchSync(MethodAddSCTReceipt, data)
return
}
// CountFQDNSets reutrns the number of currently valid sets with hash |setHash|
func (cac StorageAuthorityClient) CountFQDNSets(window time.Duration, names []string) (int64, error) {
data, err := json.Marshal(countFQDNsRequest{window, names})
if err != nil {
return 0, err
}
response, err := cac.rpc.DispatchSync(MethodCountFQDNSets, data)
if err != nil {
return 0, err
}
var count countFQDNSetsResponse
err = json.Unmarshal(response, &count)
return count.Count, err
}

20
sa/_db/migrations/20160202135920_AddFQDNSetTable.sql Normal file
View File

@ -0,0 +1,20 @@
-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied
CREATE TABLE `fqdnSets` (
`id` INT(11) NOT NULL AUTO_INCREMENT,
-- SHA256 hash of alphabetically sorted, lowercased, comma joined
-- DNS names contained in a certificate
`setHash` BINARY(32) NOT NULL,
`serial` VARCHAR(255) UNIQUE NOT NULL,
`issued` DATETIME NOT NULL,
`expires` DATETIME NOT NULL,
PRIMARY KEY (`id`),
KEY `setHash_issued_idx` (`setHash`, `issued`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back
DROP TABLE `fqdnSets`;

1
sa/database.go
View File

@ -149,4 +149,5 @@ func initTables(dbMap *gorp.DbMap) {
dbMap.AddTableWithName(core.CRL{}, "crls").SetKeys(false, "Serial")
dbMap.AddTableWithName(core.DeniedCSR{}, "deniedCSRs").SetKeys(true, "ID")
dbMap.AddTableWithName(core.SignedCertificateTimestamp{}, "sctReceipts").SetKeys(true, "ID").SetVersionCol("LockCol")
dbMap.AddTableWithName(core.FQDNSet{}, "fqdnSets").SetKeys(true, "ID")
}

42
sa/storage-authority.go
View File

@ -773,6 +773,18 @@ func (ssa *SQLStorageAuthority) AddCertificate(certDER []byte, regID int64) (dig
}
}
err = addFQDNSet(
tx,
parsedCertificate.DNSNames,
serial,
parsedCertificate.NotBefore,
parsedCertificate.NotAfter,
)
if err != nil {
tx.Rollback()
return
}
err = tx.Commit()
return
}
@ -869,3 +881,33 @@ func (ssa *SQLStorageAuthority) AddSCTReceipt(sct core.SignedCertificateTimestam
}
return err
}
func hashNames(names []string) []byte {
names = core.UniqueLowerNames(names)
hash := sha256.Sum256([]byte(strings.Join(names, ",")))
return hash[:]
}
func addFQDNSet(tx *gorp.Transaction, names []string, serial string, issued time.Time, expires time.Time) error {
return tx.Insert(&core.FQDNSet{
SetHash: hashNames(names),
Serial: serial,
Issued: issued,
Expires: expires,
})
}
// CountFQDNSets returns the number of sets with hash |setHash| within the window
// |window|
func (ssa *SQLStorageAuthority) CountFQDNSets(window time.Duration, names []string) (int64, error) {
var count int64
err := ssa.dbMap.SelectOne(
&count,
`SELECT COUNT(1) FROM fqdnSets
WHERE setHash = ?
AND issued > ?`,
hashNames(names),
ssa.clk.Now().Add(-window),
)
return count, err
}

55
sa/storage-authority_test.go
View File

@ -667,3 +667,58 @@ func TestRevokeAuthorizationsByDomain(t *testing.T) {
test.AssertEquals(t, PA.Status, core.StatusRevoked)
test.AssertEquals(t, FA.Status, core.StatusRevoked)
}
func TestFQDNSets(t *testing.T) {
sa, fc, cleanUp := initSA(t)
defer cleanUp()
tx, err := sa.dbMap.Begin()
test.AssertNotError(t, err, "Failed to open transaction")
names := []string{"a.example.com", "B.example.com"}
expires := fc.Now().Add(time.Hour * 2).UTC()
issued := fc.Now()
err = addFQDNSet(tx, names, "serial", issued, expires)
test.AssertNotError(t, err, "Failed to add name set")
test.AssertNotError(t, tx.Commit(), "Failed to commit transaction")
// only one valid
threeHours := time.Hour * 3
count, err := sa.CountFQDNSets(threeHours, names)
test.AssertNotError(t, err, "Failed to count name sets")
test.AssertEquals(t, count, int64(1))
// check hash isn't affected by changing name order/casing
count, err = sa.CountFQDNSets(threeHours, []string{"b.example.com", "A.example.COM"})
test.AssertNotError(t, err, "Failed to count name sets")
test.AssertEquals(t, count, int64(1))
// add another valid set
tx, err = sa.dbMap.Begin()
test.AssertNotError(t, err, "Failed to open transaction")
err = addFQDNSet(tx, names, "anotherSerial", issued, expires)
test.AssertNotError(t, err, "Failed to add name set")
test.AssertNotError(t, tx.Commit(), "Failed to commit transaction")
// only two valid
count, err = sa.CountFQDNSets(threeHours, names)
test.AssertNotError(t, err, "Failed to count name sets")
test.AssertEquals(t, count, int64(2))
// add an expired set
tx, err = sa.dbMap.Begin()
test.AssertNotError(t, err, "Failed to open transaction")
err = addFQDNSet(
tx,
names,
"yetAnotherSerial",
issued.Add(-threeHours),
expires.Add(-threeHours),
)
test.AssertNotError(t, err, "Failed to add name set")
test.AssertNotError(t, tx.Commit(), "Failed to commit transaction")
// only two valid
count, err = sa.CountFQDNSets(threeHours, names)
test.AssertNotError(t, err, "Failed to count name sets")
test.AssertEquals(t, count, int64(2))
}

3
test/drop_users.sql
View File

@ -19,4 +19,5 @@ GRANT USAGE ON *.* TO 'mailer'@'localhost';
DROP USER 'mailer'@'localhost';
GRANT USAGE ON *.* TO 'cert_checker'@'localhost';
DROP USER 'cert_checker'@'localhost';
GRANT USAGE ON *.* TO 'backfiller'@'localhost';
DROP USER 'backfiller'@'localhost';

3
test/rate-limit-policies.yml
View File

@ -25,3 +25,6 @@ registrationsPerIP:
pendingAuthorizationsPerAccount:
window: 168h # 1 week, should match pending authorization lifetime.
threshold: 3
certificatesPerFQDNSet:
window: 24h
threshold: 5

5
test/sa_db_users.sql
View File

@ -26,6 +26,7 @@ GRANT SELECT,INSERT ON deniedCSRs TO 'sa'@'localhost';
GRANT INSERT ON ocspResponses TO 'sa'@'localhost';
GRANT SELECT,INSERT,UPDATE ON registrations TO 'sa'@'localhost';
GRANT SELECT,INSERT,UPDATE ON challenges TO 'sa'@'localhost';
GRANT SELECT,INSERT on fqdnSets TO 'sa'@'localhost';
-- OCSP Responder
GRANT SELECT ON certificateStatus TO 'ocsp_resp'@'localhost';
@ -53,5 +54,9 @@ GRANT SELECT,UPDATE ON certificateStatus TO 'mailer'@'localhost';
-- Cert checker
GRANT SELECT ON certificates TO 'cert_checker'@'localhost';
-- Name set table backfiller
GRANT SELECT ON certificates to 'backfiller'@'localhost';
GRANT INSERT,SELECT ON fqdnSets to 'backfiller'@'localhost';
-- Test setup and teardown
GRANT ALL PRIVILEGES ON * to 'test_setup'@'localhost';