From bc28bfe90629d4cccea045cc0fe6c76e82fbfdf3 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sun, 6 Mar 2016 18:45:20 -0800 Subject: [PATCH] Implement reloadable JSON blacklist. This eliminates the need the a database to store the hostname policy, simplifying deployment. We keep the database for now, as part of our deployability guidelines: we'll deploy, then switch config to the new style. This also disables the obsolete whitelist checking code, but doesn't yet change the function signature for policy.New(), to avoid bloating the pull request. I'll fully remove the whitelist checking code in a future change when I also remove the policy database code. --- cmd/boulder-ca/main.go | 17 ++- cmd/boulder-ra/main.go | 17 ++- cmd/config.go | 8 ++ policy/policy-authority.go | 100 ++++++++++++++---- policy/policy-authority_test.go | 97 ----------------- test/boulder-config.json | 3 +- .../hostname-policy.json | 0 7 files changed, 116 insertions(+), 126 deletions(-) rename cmd/policy-loader/base-rules.json => test/hostname-policy.json (100%) diff --git a/cmd/boulder-ca/main.go b/cmd/boulder-ca/main.go index c4ff7465c..3d4a8f63c 100644 --- a/cmd/boulder-ca/main.go +++ b/cmd/boulder-ca/main.go @@ -15,6 +15,7 @@ import ( "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/helpers" "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock" "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/pkcs11key" + "github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1" "github.com/letsencrypt/boulder/ca" "github.com/letsencrypt/boulder/cmd" "github.com/letsencrypt/boulder/core" @@ -73,13 +74,21 @@ func main() { go cmd.DebugServer(c.CA.DebugAddr) - dbURL, err := c.PA.DBConfig.URL() - cmd.FailOnError(err, "Couldn't load DB URL") - paDbMap, err := sa.NewDbMap(dbURL) - cmd.FailOnError(err, "Couldn't connect to policy database") + var paDbMap *gorp.DbMap + if c.CA.HostnamePolicyFile == "" { + dbURL, err := c.PA.DBConfig.URL() + cmd.FailOnError(err, "Couldn't load DB URL") + paDbMap, err = sa.NewDbMap(dbURL) + cmd.FailOnError(err, "Couldn't connect to policy database") + } pa, err := policy.New(paDbMap, c.PA.EnforcePolicyWhitelist, c.PA.Challenges) cmd.FailOnError(err, "Couldn't create PA") + if c.CA.HostnamePolicyFile != "" { + err = pa.SetHostnamePolicyFile(c.RA.HostnamePolicyFile) + cmd.FailOnError(err, "Couldn't load hostname policy file") + } + priv, err := loadPrivateKey(c.CA.Key) cmd.FailOnError(err, "Couldn't load private key") diff --git a/cmd/boulder-ra/main.go b/cmd/boulder-ra/main.go index de1ac76ed..3834b5354 100644 --- a/cmd/boulder-ra/main.go +++ b/cmd/boulder-ra/main.go @@ -10,6 +10,7 @@ import ( "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd" "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock" + "github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1" "github.com/letsencrypt/boulder/bdns" "github.com/letsencrypt/boulder/metrics" "github.com/letsencrypt/boulder/policy" @@ -31,13 +32,21 @@ func main() { go cmd.DebugServer(c.RA.DebugAddr) - dbURL, err := c.PA.DBConfig.URL() - cmd.FailOnError(err, "Couldn't load DB URL") - paDbMap, err := sa.NewDbMap(dbURL) - cmd.FailOnError(err, "Couldn't connect to policy database") + var paDbMap *gorp.DbMap + if c.RA.HostnamePolicyFile == "" { + dbURL, err := c.PA.DBConfig.URL() + cmd.FailOnError(err, "Couldn't load DB URL") + paDbMap, err = sa.NewDbMap(dbURL) + cmd.FailOnError(err, "Couldn't connect to policy database") + } pa, err := policy.New(paDbMap, c.PA.EnforcePolicyWhitelist, c.PA.Challenges) cmd.FailOnError(err, "Couldn't create PA") + if c.RA.HostnamePolicyFile != "" { + err = pa.SetHostnamePolicyFile(c.RA.HostnamePolicyFile) + cmd.FailOnError(err, "Couldn't load hostname policy file") + } + rateLimitPolicies, err := cmd.LoadRateLimitPolicies(c.RA.RateLimitPoliciesFilename) cmd.FailOnError(err, "Couldn't load rate limit policies file") diff --git a/cmd/config.go b/cmd/config.go index df7d920ed..3ff1b575d 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -53,6 +53,7 @@ type Config struct { RA struct { ServiceConfig + HostnamePolicyConfig RateLimitPoliciesFilename string @@ -308,6 +309,7 @@ func (a *AMQPConfig) ServerURL() (string, error) { type CAConfig struct { ServiceConfig DBConfig + HostnamePolicyConfig Profile string RSAProfile string @@ -346,6 +348,12 @@ type PAConfig struct { Challenges map[string]bool } +// HostnamePolicyConfig specifies a file from which to load a policy regarding +// what hostnames to issue for. +type HostnamePolicyConfig struct { + HostnamePolicyFile string +} + // CheckChallenges checks whether the list of challenges in the PA config // actually contains valid challenge names func (pc PAConfig) CheckChallenges() error { diff --git a/policy/policy-authority.go b/policy/policy-authority.go index c967d7ca6..80dc8f976 100644 --- a/policy/policy-authority.go +++ b/policy/policy-authority.go @@ -6,16 +6,22 @@ package policy import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" "math/rand" "net" "regexp" "strings" + "sync" "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/go-jose" "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/net/publicsuffix" "github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1" "github.com/letsencrypt/boulder/core" blog "github.com/letsencrypt/boulder/log" + "github.com/letsencrypt/boulder/reloader" ) // AuthorityImpl enforces CA policy decisions. @@ -23,26 +29,31 @@ type AuthorityImpl struct { log *blog.AuditLogger DB *AuthorityDatabaseImpl - EnforceWhitelist bool + blacklist map[string]bool + blacklistMu sync.RWMutex + enabledChallenges map[string]bool pseudoRNG *rand.Rand } // New constructs a Policy Authority. -func New(dbMap *gorp.DbMap, enforceWhitelist bool, challengeTypes map[string]bool) (*AuthorityImpl, error) { +func New(dbMap *gorp.DbMap, _ bool, challengeTypes map[string]bool) (*AuthorityImpl, error) { logger := blog.GetAuditLogger() logger.Notice("Policy Authority Starting") - // Setup policy db - padb, err := NewAuthorityDatabaseImpl(dbMap) - if err != nil { - return nil, err + var padb *AuthorityDatabaseImpl + var err error + if dbMap != nil { + // Setup policy db + padb, err = NewAuthorityDatabaseImpl(dbMap) + if err != nil { + return nil, err + } } pa := AuthorityImpl{ log: logger, DB: padb, - EnforceWhitelist: enforceWhitelist, enabledChallenges: challengeTypes, // We don't need real randomness for this. pseudoRNG: rand.New(rand.NewSource(99)), @@ -51,6 +62,42 @@ func New(dbMap *gorp.DbMap, enforceWhitelist bool, challengeTypes map[string]boo return &pa, nil } +type blacklistJSON struct { + Blacklist []string +} + +// SetHostnamePolicyFile will load the given policy file, returning error if it +// fails. It will also start a reloader in case the file changes. +func (pa *AuthorityImpl) SetHostnamePolicyFile(f string) error { + _, err := reloader.New(f, pa.loadHostnamePolicy) + return err +} + +func (pa *AuthorityImpl) loadHostnamePolicy(b []byte, err error) error { + if err != nil { + pa.log.Err(fmt.Sprintf("loading hostname policy: %s", err)) + } + hash := sha256.Sum256(b) + pa.log.Info(fmt.Sprintf("loading hostname policy, sha256: %s", + hex.EncodeToString(hash[:]))) + var bl blacklistJSON + err = json.Unmarshal(b, &bl) + if err != nil { + return err + } + if len(bl.Blacklist) == 0 { + return fmt.Errorf("No entries in blacklist.") + } + nameMap := make(map[string]bool) + for _, v := range bl.Blacklist { + nameMap[v] = true + } + pa.blacklistMu.Lock() + pa.blacklist = nameMap + pa.blacklistMu.Unlock() + return nil +} + const ( maxLabels = 10 @@ -127,7 +174,7 @@ var ( // where comparison is case-independent (normalized to lower case) // // If WillingToIssue returns an error, it will be of type MalformedRequestError. -func (pa AuthorityImpl) WillingToIssue(id core.AcmeIdentifier, regID int64) error { +func (pa *AuthorityImpl) WillingToIssue(id core.AcmeIdentifier, regID int64) error { if id.Type != core.IdentifierDNS { return errInvalidIdentifier } @@ -188,28 +235,41 @@ func (pa AuthorityImpl) WillingToIssue(id core.AcmeIdentifier, regID int64) erro return errICANNTLD } - // Use the domain whitelist if the PA has been asked to. However, if the - // registration ID is from a whitelisted partner we're allowing to register - // any domain, they can get in, too. - enforceWhitelist := pa.EnforceWhitelist - if regID == whitelistedPartnerRegID { - enforceWhitelist = false - } - - // Require no match against blacklist and if enforceWhitelist is true - // require domain to match a whitelist rule. - if err := pa.DB.CheckHostLists(domain, enforceWhitelist); err != nil { + // Require no match against blacklist + if err := pa.checkHostLists(domain); err != nil { return err } return nil } +func (pa *AuthorityImpl) checkHostLists(domain string) error { + if pa.DB != nil { + return pa.DB.CheckHostLists(domain, false) + } + + pa.blacklistMu.RLock() + defer pa.blacklistMu.RUnlock() + + if pa.blacklist == nil { + return fmt.Errorf("Hostname policy not yet loaded.") + } + + labels := strings.Split(domain, ".") + for i := range labels { + joined := strings.Join(labels[i:], ".") + if pa.blacklist[joined] { + return errBlacklisted + } + } + return nil +} + // ChallengesFor makes a decision of what challenges, and combinations, are // acceptable for the given identifier. // // Note: Current implementation is static, but future versions may not be. -func (pa AuthorityImpl) ChallengesFor(identifier core.AcmeIdentifier, accountKey *jose.JsonWebKey) ([]core.Challenge, [][]int) { +func (pa *AuthorityImpl) ChallengesFor(identifier core.AcmeIdentifier, accountKey *jose.JsonWebKey) ([]core.Challenge, [][]int) { challenges := []core.Challenge{} if pa.enabledChallenges[core.ChallengeTypeHTTP01] { diff --git a/policy/policy-authority_test.go b/policy/policy-authority_test.go index ecf4e5795..55fdb261d 100644 --- a/policy/policy-authority_test.go +++ b/policy/policy-authority_test.go @@ -213,100 +213,3 @@ func TestChallengesFor(t *testing.T) { test.AssertEquals(t, len(seenChalls), len(enabledChallenges)) test.AssertDeepEquals(t, expectedCombos, combinations) } - -func TestWillingToIssueWithWhitelist(t *testing.T) { - dbMap, cleanUp := paDBMap(t) - defer cleanUp() - pa, err := New(dbMap, true, nil) - test.AssertNotError(t, err, "Couldn't create policy implementation") - googID := core.AcmeIdentifier{ - Type: core.IdentifierDNS, - Value: "www.google.com", - } - zomboID := core.AcmeIdentifier{ - Type: core.IdentifierDNS, - Value: "www.zombo.com", - } - - type listTestCase struct { - regID int64 - id core.AcmeIdentifier - err error - } - pa.DB.LoadRules(RuleSet{ - Whitelist: []WhitelistRule{ - {Host: "www.zombo.com"}, - }, - }) - - // Note that www.google.com is not in the blacklist for this test. We no - // longer have a hardcoded blacklist. - testCases := []listTestCase{ - {100, googID, errNotWhitelisted}, - {whitelistedPartnerRegID, googID, nil}, - {100, zomboID, nil}, - {whitelistedPartnerRegID, zomboID, nil}, - } - for _, tc := range testCases { - err := pa.WillingToIssue(tc.id, tc.regID) - if err != tc.err { - t.Errorf("%#v, %d: want %#v, got %#v", tc.id.Value, tc.regID, tc.err, err) - } - } - - pa.DB.LoadRules(RuleSet{ - Blacklist: []BlacklistRule{ - {Host: "www.google.com"}, - }, - Whitelist: []WhitelistRule{ - {Host: "www.zombo.com"}, - }, - }) - - exampleID := core.AcmeIdentifier{ - Type: core.IdentifierDNS, - Value: "www.example.com", - } - - testCases = []listTestCase{ - // This errNotWhitelisted is surprising and accidental from the ordering - // of the whitelist and blacklist check. The whitelist will be gone soon - // enough. - {100, googID, errNotWhitelisted}, - {whitelistedPartnerRegID, googID, errBlacklisted}, - {100, zomboID, nil}, - {whitelistedPartnerRegID, zomboID, nil}, - {100, exampleID, errNotWhitelisted}, - {whitelistedPartnerRegID, exampleID, nil}, - } - for _, tc := range testCases { - err := pa.WillingToIssue(tc.id, tc.regID) - if err != tc.err { - t.Errorf("%#v, %d: want %#v, got %#v", tc.id.Value, tc.regID, tc.err, err) - } - } - - pa.DB.LoadRules(RuleSet{ - Blacklist: []BlacklistRule{ - {Host: "www.google.com"}, - }, - Whitelist: []WhitelistRule{ - {Host: "www.zombo.com"}, - {Host: "www.google.com"}, - }, - }) - testCases = []listTestCase{ - {100, googID, errBlacklisted}, - {whitelistedPartnerRegID, googID, errBlacklisted}, - {100, zomboID, nil}, - {whitelistedPartnerRegID, zomboID, nil}, - {100, exampleID, errNotWhitelisted}, - {whitelistedPartnerRegID, exampleID, nil}, - } - for _, tc := range testCases { - err := pa.WillingToIssue(tc.id, tc.regID) - if err != tc.err { - t.Errorf("%#v, %d: want %#v, got %#v", tc.id.Value, tc.regID, tc.err, err) - } - } -} diff --git a/test/boulder-config.json b/test/boulder-config.json index 77a533ac8..d86f01746 100644 --- a/test/boulder-config.json +++ b/test/boulder-config.json @@ -47,6 +47,7 @@ "maxNames": 1000, "doNotForceCN": true, "enableMustStaple": true, + "hostnamePolicyFile": "test/hostname-policy.json", "cfssl": { "signing": { "profiles": { @@ -150,7 +151,6 @@ }, "pa": { - "dbConnect": "mysql+tcp://policy@localhost:3306/boulder_policy_integration?readTimeout=800ms&writeTimeout=800ms", "challenges": { "http-01": true, "tls-sni-01": true, @@ -164,6 +164,7 @@ "maxContactsPerRegistration": 100, "dnsTries": 3, "debugAddr": "localhost:8002", + "hostnamePolicyFile": "test/hostname-policy.json", "amqp": { "serverURLFile": "test/secrets/amqp_url", "insecure": true, diff --git a/cmd/policy-loader/base-rules.json b/test/hostname-policy.json similarity index 100% rename from cmd/policy-loader/base-rules.json rename to test/hostname-policy.json