From e5e4fb744a15226927bb5239f8ce8aefe883f39d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 29 Apr 2016 12:12:24 -0700 Subject: [PATCH] * Delete Policy DB.This is no longer needed now that we have a JSON policy file.* Fix tests.* Revert Dockerfile.* Fix create_db* Simplify user addition.* Fix tests.* Fix tests* Review fixes.https://github.com/letsencrypt/boulder/pull/1773 * Delete Policy DB. This is no longer needed now that we have a JSON policy file. * Fix tests. * Revert Dockerfile. * Fix create_db * Simplify user addition. * Fix tests. * Fix tests * Review fixes. https://github.com/letsencrypt/boulder/pull/1773 --- ca/certificate-authority_test.go | 33 +--- cmd/boulder-ca/main.go | 18 +- cmd/boulder-ra/main.go | 18 +- cmd/cert-checker/main.go | 16 +- cmd/cert-checker/main_test.go | 42 ++-- cmd/config.go | 1 + cmd/policy-loader/format.md | 14 -- cmd/policy-loader/main.go | 129 ------------ policy/_db/dbconf.yml | 6 - .../20150825180747_InitialSchema.sql | 19 -- policy/policy-authority-data.go | 187 ------------------ policy/policy-authority-data_test.go | 149 -------------- policy/policy-authority.go | 22 +-- policy/policy-authority_test.go | 40 ++-- ra/registration-authority_test.go | 10 +- test/create_db.sh | 48 +++-- test/db-common.sh | 2 - test/db.go | 6 - test/hostname-policy.json | 1 - test/policy_db_users.sql | 26 --- test/secrets/pa_dburl | 1 - 21 files changed, 86 insertions(+), 702 deletions(-) delete mode 100644 cmd/policy-loader/format.md delete mode 100644 cmd/policy-loader/main.go delete mode 100644 policy/_db/dbconf.yml delete mode 100644 policy/_db/migrations/20150825180747_InitialSchema.sql delete mode 100644 policy/policy-authority-data.go delete mode 100644 policy/policy-authority-data_test.go delete mode 100644 test/policy_db_users.sql delete mode 100644 test/secrets/pa_dburl diff --git a/ca/certificate-authority_test.go b/ca/certificate-authority_test.go index 3ddb711c2..1438e9390 100644 --- a/ca/certificate-authority_test.go +++ b/ca/certificate-authority_test.go @@ -27,9 +27,7 @@ import ( blog "github.com/letsencrypt/boulder/log" "github.com/letsencrypt/boulder/mocks" "github.com/letsencrypt/boulder/policy" - "github.com/letsencrypt/boulder/sa" "github.com/letsencrypt/boulder/test" - "github.com/letsencrypt/boulder/test/vars" ) var ( @@ -159,7 +157,6 @@ type testCtx struct { keyPolicy core.KeyPolicy fc clock.FakeClock stats *mocks.Statter - cleanUp func() } type mockSA struct { @@ -191,15 +188,10 @@ func setup(t *testing.T) *testCtx { fc := clock.NewFake() fc.Add(1 * time.Hour) - paDbMap, err := sa.NewDbMap(vars.DBConnPolicy) - test.AssertNotError(t, err, "Could not construct dbMap") - pa, err := policy.New(paDbMap, false, nil) - test.AssertNotError(t, err, "Couldn't create PADB") - paDBCleanUp := test.ResetPolicyTestDatabase(t) - - cleanUp := func() { - paDBCleanUp() - } + pa, err := policy.New(nil) + test.AssertNotError(t, err, "Couldn't create PA") + err = pa.SetHostnamePolicyFile("../test/hostname-policy.json") + test.AssertNotError(t, err, "Couldn't set hostname policy") // Create a CA caConfig := cmd.CAConfig{ @@ -283,13 +275,11 @@ func setup(t *testing.T) *testCtx { keyPolicy, fc, stats, - cleanUp, } } func TestFailNoSerial(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() testCtx.caConfig.SerialPrefix = 0 _, err := NewCertificateAuthorityImpl( @@ -303,7 +293,6 @@ func TestFailNoSerial(t *testing.T) { func TestIssueCertificate(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, testCtx.fc, @@ -368,7 +357,6 @@ func TestIssueCertificate(t *testing.T) { // Test issuing when multiple issuers are present. func TestIssueCertificateMultipleIssuers(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() // Load multiple issuers, and ensure the first one in the list is used. newIssuerCert, err := core.LoadCert("../test/test-ca2.pem") test.AssertNotError(t, err, "Failed to load new cert") @@ -406,7 +394,6 @@ func TestIssueCertificateMultipleIssuers(t *testing.T) { func TestOCSP(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, testCtx.fc, @@ -501,7 +488,6 @@ func TestOCSP(t *testing.T) { func TestNoHostnames(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, testCtx.fc, @@ -522,7 +508,6 @@ func TestNoHostnames(t *testing.T) { func TestRejectTooManyNames(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, testCtx.fc, @@ -544,7 +529,6 @@ func TestRejectTooManyNames(t *testing.T) { func TestDeduplication(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, testCtx.fc, @@ -572,7 +556,6 @@ func TestDeduplication(t *testing.T) { func TestRejectValidityTooLong(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, testCtx.fc, @@ -599,7 +582,6 @@ func TestRejectValidityTooLong(t *testing.T) { func TestShortKey(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, testCtx.fc, @@ -620,7 +602,6 @@ func TestShortKey(t *testing.T) { func TestAllowNoCN(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, testCtx.fc, @@ -661,7 +642,6 @@ func TestAllowNoCN(t *testing.T) { func TestLongCommonName(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, testCtx.fc, @@ -681,7 +661,6 @@ func TestLongCommonName(t *testing.T) { func TestRejectBadAlgorithm(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, testCtx.fc, @@ -702,7 +681,6 @@ func TestRejectBadAlgorithm(t *testing.T) { func TestCapitalizedLetters(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() testCtx.caConfig.MaxNames = 3 ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, @@ -729,7 +707,6 @@ func TestCapitalizedLetters(t *testing.T) { func TestWrongSignature(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() testCtx.caConfig.MaxNames = 3 ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, @@ -752,7 +729,6 @@ func TestWrongSignature(t *testing.T) { func TestProfileSelection(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() testCtx.caConfig.MaxNames = 3 ca, _ := NewCertificateAuthorityImpl( testCtx.caConfig, @@ -802,7 +778,6 @@ func countMustStaple(t *testing.T, cert *x509.Certificate) (count int) { func TestExtensions(t *testing.T) { testCtx := setup(t) - defer testCtx.cleanUp() testCtx.caConfig.MaxNames = 3 ca, err := NewCertificateAuthorityImpl( testCtx.caConfig, diff --git a/cmd/boulder-ca/main.go b/cmd/boulder-ca/main.go index 02ca4ce30..9389c3002 100644 --- a/cmd/boulder-ca/main.go +++ b/cmd/boulder-ca/main.go @@ -15,7 +15,6 @@ import ( "github.com/cloudflare/cfssl/helpers" "github.com/jmhodges/clock" "github.com/letsencrypt/pkcs11key" - "gopkg.in/gorp.v1" "github.com/letsencrypt/boulder/ca" "github.com/letsencrypt/boulder/cmd" @@ -24,7 +23,6 @@ import ( "github.com/letsencrypt/boulder/metrics" "github.com/letsencrypt/boulder/policy" "github.com/letsencrypt/boulder/rpc" - "github.com/letsencrypt/boulder/sa" ) const clientName = "CA" @@ -114,20 +112,14 @@ func main() { go cmd.DebugServer(c.CA.DebugAddr) - var paDbMap *gorp.DbMap - if c.CA.HostnamePolicyFile == "" { - dbURL, err := c.PA.DBConfig.URL() - cmd.FailOnError(err, "Couldn't load DB URL") - paDbMap, err = sa.NewDbMap(dbURL) - cmd.FailOnError(err, "Couldn't connect to policy database") - } - pa, err := policy.New(paDbMap, c.PA.EnforcePolicyWhitelist, c.PA.Challenges) + pa, err := policy.New(c.PA.Challenges) cmd.FailOnError(err, "Couldn't create PA") - if c.CA.HostnamePolicyFile != "" { - err = pa.SetHostnamePolicyFile(c.CA.HostnamePolicyFile) - cmd.FailOnError(err, "Couldn't load hostname policy file") + if c.RA.HostnamePolicyFile == "" { + cmd.FailOnError(nil, "HostnamePolicyFile was empty.") } + err = pa.SetHostnamePolicyFile(c.RA.HostnamePolicyFile) + cmd.FailOnError(err, "Couldn't load hostname policy file") issuers, err := loadIssuers(c) cmd.FailOnError(err, "Couldn't load issuers") diff --git a/cmd/boulder-ra/main.go b/cmd/boulder-ra/main.go index baaa0ea78..f1de8ca72 100644 --- a/cmd/boulder-ra/main.go +++ b/cmd/boulder-ra/main.go @@ -12,8 +12,6 @@ import ( "github.com/letsencrypt/boulder/bdns" "github.com/letsencrypt/boulder/metrics" "github.com/letsencrypt/boulder/policy" - "github.com/letsencrypt/boulder/sa" - "gopkg.in/gorp.v1" "github.com/letsencrypt/boulder/cmd" blog "github.com/letsencrypt/boulder/log" @@ -31,20 +29,14 @@ func main() { go cmd.DebugServer(c.RA.DebugAddr) - var paDbMap *gorp.DbMap - if c.RA.HostnamePolicyFile == "" { - dbURL, err := c.PA.DBConfig.URL() - cmd.FailOnError(err, "Couldn't load DB URL") - paDbMap, err = sa.NewDbMap(dbURL) - cmd.FailOnError(err, "Couldn't connect to policy database") - } - pa, err := policy.New(paDbMap, c.PA.EnforcePolicyWhitelist, c.PA.Challenges) + pa, err := policy.New(c.PA.Challenges) cmd.FailOnError(err, "Couldn't create PA") - if c.RA.HostnamePolicyFile != "" { - err = pa.SetHostnamePolicyFile(c.RA.HostnamePolicyFile) - cmd.FailOnError(err, "Couldn't load hostname policy file") + if c.RA.HostnamePolicyFile == "" { + cmd.FailOnError(nil, "HostnamePolicyFile must be provided.") } + err = pa.SetHostnamePolicyFile(c.RA.HostnamePolicyFile) + cmd.FailOnError(err, "Couldn't load hostname policy file") rateLimitPolicies, err := cmd.LoadRateLimitPolicies(c.RA.RateLimitPoliciesFilename) cmd.FailOnError(err, "Couldn't load rate limit policies file") diff --git a/cmd/cert-checker/main.go b/cmd/cert-checker/main.go index 82f4fa3e2..02abf10de 100644 --- a/cmd/cert-checker/main.go +++ b/cmd/cert-checker/main.go @@ -74,9 +74,7 @@ type certChecker struct { stats metrics.Statter } -func newChecker(saDbMap *gorp.DbMap, paDbMap *gorp.DbMap, clk clock.Clock, enforceWhitelist bool, challengeTypes map[string]bool, period time.Duration) certChecker { - pa, err := policy.New(paDbMap, enforceWhitelist, challengeTypes) - cmd.FailOnError(err, "Failed to create PA") +func newChecker(saDbMap *gorp.DbMap, clk clock.Clock, pa core.PolicyAuthority, period time.Duration) certChecker { c := certChecker{ pa: pa, dbMap: saDbMap, @@ -305,17 +303,15 @@ func main() { saDbMap, err := sa.NewDbMap(saDbURL) cmd.FailOnError(err, "Could not connect to database") - paDbURL, err := config.PA.DBConfig.URL() - cmd.FailOnError(err, "Couldn't load DB URL") - paDbMap, err := sa.NewDbMap(paDbURL) - cmd.FailOnError(err, "Could not connect to policy database") + pa, err := policy.New(config.PA.Challenges) + cmd.FailOnError(err, "Failed to create PA") + err = pa.SetHostnamePolicyFile(config.CertChecker.HostnamePolicyFile) + cmd.FailOnError(err, "Failed to load HostnamePolicyFile") checker := newChecker( saDbMap, - paDbMap, clock.Default(), - config.PA.EnforcePolicyWhitelist, - config.PA.Challenges, + pa, config.CertChecker.CheckPeriod.Duration, ) fmt.Fprintf(os.Stderr, "# Getting certificates issued in the last %s\n", config.CertChecker.CheckPeriod) diff --git a/cmd/cert-checker/main_test.go b/cmd/cert-checker/main_test.go index 3e52ad337..4ee51bf48 100644 --- a/cmd/cert-checker/main_test.go +++ b/cmd/cert-checker/main_test.go @@ -11,6 +11,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "fmt" + "log" "math/big" mrand "math/rand" "sync" @@ -22,29 +23,38 @@ import ( "github.com/letsencrypt/boulder/core" blog "github.com/letsencrypt/boulder/log" + "github.com/letsencrypt/boulder/policy" "github.com/letsencrypt/boulder/sa" "github.com/letsencrypt/boulder/sa/satest" "github.com/letsencrypt/boulder/test" "github.com/letsencrypt/boulder/test/vars" ) +var pa *policy.AuthorityImpl + +func init() { + var err error + pa, err = policy.New(map[string]bool{}) + if err != nil { + log.Fatal(err) + } + err = pa.SetHostnamePolicyFile("../../test/hostname-policy.json") + if err != nil { + log.Fatal(err) + } +} + func BenchmarkCheckCert(b *testing.B) { saDbMap, err := sa.NewDbMap(vars.DBConnSA) if err != nil { fmt.Println("Couldn't connect to database") return } - paDbMap, err := sa.NewDbMap(vars.DBConnPolicy) - if err != nil { - fmt.Println("Couldn't connect to database") - return - } defer func() { test.ResetSATestDatabase(b)() - test.ResetPolicyTestDatabase(b)() }() - checker := newChecker(saDbMap, paDbMap, clock.Default(), false, nil, expectedValidityPeriod) + checker := newChecker(saDbMap, clock.Default(), pa, expectedValidityPeriod) testKey, _ := rsa.GenerateKey(rand.Reader, 1024) expiry := time.Now().AddDate(0, 0, 1) serial := big.NewInt(1337) @@ -74,19 +84,15 @@ func TestCheckCert(t *testing.T) { saDbMap, err := sa.NewDbMap(vars.DBConnSA) test.AssertNotError(t, err, "Couldn't connect to database") saCleanup := test.ResetSATestDatabase(t) - paDbMap, err := sa.NewDbMap(vars.DBConnPolicy) - test.AssertNotError(t, err, "Couldn't connect to policy database") - paCleanup := test.ResetPolicyTestDatabase(t) defer func() { saCleanup() - paCleanup() }() testKey, _ := rsa.GenerateKey(rand.Reader, 1024) fc := clock.NewFake() fc.Add(time.Hour * 24 * 90) - checker := newChecker(saDbMap, paDbMap, fc, false, nil, expectedValidityPeriod) + checker := newChecker(saDbMap, fc, pa, expectedValidityPeriod) issued := checker.clock.Now().Add(-time.Hour * 24 * 45) goodExpiry := issued.Add(expectedValidityPeriod) @@ -131,17 +137,17 @@ func TestCheckCert(t *testing.T) { "Certificate has incorrect key usage extensions": 1, "Certificate has common name >64 characters long (65)": 1, } - test.AssertEquals(t, len(problems), 8) for _, p := range problems { _, ok := problemsMap[p] if !ok { - t.Errorf("Expected problem '%s' but didn't find it.", p) + t.Errorf("Found unexpected problem '%s'.", p) } delete(problemsMap, p) } for k := range problemsMap { - t.Errorf("Found unexpected problem '%s'.", k) + t.Errorf("Expected problem but didn't find it: '%s'.", k) } + test.AssertEquals(t, len(problems), 8) // Same settings as above, but the stored serial number in the DB is invalid. cert.Serial = "not valid" @@ -175,18 +181,14 @@ func TestCheckCert(t *testing.T) { func TestGetAndProcessCerts(t *testing.T) { saDbMap, err := sa.NewDbMap(vars.DBConnSA) test.AssertNotError(t, err, "Couldn't connect to database") - paDbMap, err := sa.NewDbMap(vars.DBConnPolicy) - test.AssertNotError(t, err, "Couldn't connect to policy database") fc := clock.NewFake() - checker := newChecker(saDbMap, paDbMap, fc, false, nil, expectedValidityPeriod) + checker := newChecker(saDbMap, fc, pa, expectedValidityPeriod) sa, err := sa.NewSQLStorageAuthority(saDbMap, fc, blog.NewMock()) test.AssertNotError(t, err, "Couldn't create SA to insert certificates") saCleanUp := test.ResetSATestDatabase(t) - paCleanUp := test.ResetPolicyTestDatabase(t) defer func() { saCleanUp() - paCleanUp() }() testKey, _ := rsa.GenerateKey(rand.Reader, 1024) diff --git a/cmd/config.go b/cmd/config.go index d67d59550..e99b6fbc4 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -182,6 +182,7 @@ type Config struct { CertChecker struct { DBConfig + HostnamePolicyConfig Workers int ReportDirectoryPath string diff --git a/cmd/policy-loader/format.md b/cmd/policy-loader/format.md deleted file mode 100644 index 50adb75e7..000000000 --- a/cmd/policy-loader/format.md +++ /dev/null @@ -1,14 +0,0 @@ -# `policy-loader` rule file format - -Both `blacklist` and `whitelist` rules are loaded into the policy database in the -same JSON file. This rule file has the following structure, currently the only allowed -types are `whitelist` and `blacklist`. `base-rules.json` in this directory contains -a number of blacklist rules for special-use domains but this should be built upon -further with high-value domains. - -``` -{ - "Blacklist": ["example.com", ...], - "Whitelist:" ["another-example.com", ...] -} -``` diff --git a/cmd/policy-loader/main.go b/cmd/policy-loader/main.go deleted file mode 100644 index f49e53787..000000000 --- a/cmd/policy-loader/main.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2015 ISRG. All rights reserved -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "strings" - - "github.com/codegangsta/cli" - _ "github.com/go-sql-driver/mysql" - "github.com/letsencrypt/boulder/sa" - - "github.com/letsencrypt/boulder/cmd" - "github.com/letsencrypt/boulder/policy" -) - -func main() { - app := cli.NewApp() - app.Name = "policy-loader" - app.Usage = "Loads/dumps rules into/from the policy database" - app.Version = cmd.Version() - app.Author = "Boulder contributors" - app.Email = "ca-dev@letsencrypt.org" - - app.Flags = []cli.Flag{ - cli.StringFlag{ - Name: "config", - Value: "config.json", - EnvVar: "BOULDER_CONFIG", - Usage: "Path to Boulder JSON configuration file", - }, - cli.StringFlag{ - Name: "rule-file", - Value: "rules.json", - EnvVar: "BOULDER_POLICY_RULES", - Usage: "Path to Boulder policy whitelist and blacklist rule file", - }, - } - - app.Commands = append(app.Commands, []cli.Command{ - { - Name: "dump-rules", - Usage: "Write out whitelist and blacklist from database to a rule file", - Action: func(c *cli.Context) { - padb, ruleFile := setupFromContext(c) - ruleSet, err := padb.DumpRules() - cmd.FailOnError(err, "Couldn't retrieve whitelist rules") - var rules struct { - Blacklist []string - Whitelist []string - } - for _, r := range ruleSet.Blacklist { - rules.Blacklist = append(rules.Blacklist, r.Host) - } - for _, r := range ruleSet.Whitelist { - rules.Whitelist = append(rules.Whitelist, r.Host) - } - rulesJSON, err := json.Marshal(rules) - cmd.FailOnError(err, "Couldn't marshal rule list") - err = ioutil.WriteFile(ruleFile, rulesJSON, os.ModePerm) - cmd.FailOnError(err, "Failed to write the rule file") - fmt.Printf("# Saved rule list to %s\n", ruleFile) - }, - }, - { - Name: "load-rules", - Usage: "Load whitelist and blacklist into database from a rule file", - Action: func(c *cli.Context) { - padb, ruleFile := setupFromContext(c) - - rulesJSON, err := ioutil.ReadFile(ruleFile) - cmd.FailOnError(err, "Couldn't read configuration file") - rules := policy.RawRuleSet{} - err = json.Unmarshal(rulesJSON, &rules) - cmd.FailOnError(err, "Couldn't unmarshal rules list") - rs := policy.RuleSet{} - for _, r := range rules.Blacklist { - rs.Blacklist = append(rs.Blacklist, policy.BlacklistRule{ - Host: strings.ToLower(r), - }) - } - for _, r := range rules.Whitelist { - rs.Whitelist = append(rs.Whitelist, policy.WhitelistRule{ - Host: strings.ToLower(r), - }) - } - - err = padb.LoadRules(rs) - cmd.FailOnError(err, "Couldn't load rules") - - fmt.Println("# Loaded whitelist and blacklist into database") - }, - }, - }...) - - err := app.Run(os.Args) - cmd.FailOnError(err, "Couldn't run application") -} - -func setupFromContext(context *cli.Context) (*policy.AuthorityDatabaseImpl, string) { - configFileName := context.GlobalString("config") - configJSON, err := ioutil.ReadFile(configFileName) - cmd.FailOnError(err, "Couldn't read configuration file") - var c cmd.Config - err = json.Unmarshal(configJSON, &c) - cmd.FailOnError(err, "Couldn't unmarshal configuration object") - - dbURL, err := c.PA.DBConfig.URL() - cmd.FailOnError(err, "Couldn't load DB URL") - dbMap, err := sa.NewDbMap(dbURL) - cmd.FailOnError(err, "Failed to create DB map") - - padb, err := policy.NewAuthorityDatabaseImpl(dbMap) - cmd.FailOnError(err, "Could not connect to PADB") - - ruleFile := context.GlobalString("rule-file") - if ruleFile == "" { - fmt.Println("rule-file argument is required") - os.Exit(1) - } - - return padb, ruleFile -} diff --git a/policy/_db/dbconf.yml b/policy/_db/dbconf.yml deleted file mode 100644 index e6f3b185d..000000000 --- a/policy/_db/dbconf.yml +++ /dev/null @@ -1,6 +0,0 @@ -test: - driver: mysql - open: root@tcp(boulder-mysql:3306)/boulder_policy_test -integration: - driver: mysql - open: root@tcp(boulder-mysql:3306)/boulder_policy_integration diff --git a/policy/_db/migrations/20150825180747_InitialSchema.sql b/policy/_db/migrations/20150825180747_InitialSchema.sql deleted file mode 100644 index 3000c61d9..000000000 --- a/policy/_db/migrations/20150825180747_InitialSchema.sql +++ /dev/null @@ -1,19 +0,0 @@ - --- +goose Up --- SQL in section 'Up' is executed when this migration is applied - -CREATE TABLE `blacklist` ( - `host` varchar(255) NOT NULL, - PRIMARY KEY (`host`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `whitelist` ( - `host` varchar(255) NOT NULL, - PRIMARY KEY (`host`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- +goose Down --- SQL section 'Down' is executed when this migration is rolled back - -DROP TABLE 'blacklist'; -DROP TABLE 'whitelist'; diff --git a/policy/policy-authority-data.go b/policy/policy-authority-data.go deleted file mode 100644 index d426b9ae1..000000000 --- a/policy/policy-authority-data.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2015 ISRG. All rights reserved -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package policy - -import ( - "database/sql" - "fmt" - "strings" - - "github.com/letsencrypt/boulder/sa" - gorp "gopkg.in/gorp.v1" - - "github.com/letsencrypt/boulder/core" - blog "github.com/letsencrypt/boulder/log" -) - -var errDBFailure = core.InternalServerError("Error checking policy DB.") - -const whitelisted = "whitelist" -const blacklisted = "blacklist" - -type domainRule struct { - Host string `db:"host"` -} - -// BlacklistRule is used to hold rules blacklisting a DNS name -type BlacklistRule domainRule - -// WhitelistRule is used to hold rules whitelisting a DNS name -type WhitelistRule domainRule - -// RawRuleSet describes the rule set file format -type RawRuleSet struct { - Blacklist []string - Whitelist []string -} - -// RuleSet describes the rules to load into the policy database -type RuleSet struct { - Blacklist []BlacklistRule - Whitelist []WhitelistRule -} - -type gorpDbMap interface { - AddTableWithName(interface{}, string) *gorp.TableMap - Begin() (*gorp.Transaction, error) - SelectOne(interface{}, string, ...interface{}) error - Select(interface{}, string, ...interface{}) ([]interface{}, error) -} - -// AuthorityDatabaseImpl enforces policy decisions based on various rule -// lists -type AuthorityDatabaseImpl struct { - log blog.Logger - dbMap gorpDbMap -} - -// NewAuthorityDatabaseImpl constructs a Policy Authority Database (and -// creates tables if they are non-existent) -func NewAuthorityDatabaseImpl(dbMap gorpDbMap) (padb *AuthorityDatabaseImpl, err error) { - logger := blog.Get() - - dbMap.AddTableWithName(BlacklistRule{}, "blacklist") - dbMap.AddTableWithName(WhitelistRule{}, "whitelist") - - padb = &AuthorityDatabaseImpl{ - dbMap: dbMap, - log: logger, - } - - return padb, nil -} - -// LoadRules loads the whitelist and blacklist into the database in a transaction -// deleting any previous content -func (padb *AuthorityDatabaseImpl) LoadRules(rs RuleSet) error { - tx, err := padb.dbMap.Begin() - if err != nil { - return sa.Rollback(tx, err) - } - _, err = tx.Exec("DELETE FROM blacklist") - if err != nil { - return sa.Rollback(tx, err) - } - for _, r := range rs.Blacklist { - r.Host = core.ReverseName(r.Host) - err = tx.Insert(&r) - if err != nil { - return sa.Rollback(tx, err) - } - } - _, err = tx.Exec("DELETE FROM whitelist") - if err != nil { - return sa.Rollback(tx, err) - } - for _, r := range rs.Whitelist { - err = tx.Insert(&r) - if err != nil { - return sa.Rollback(tx, err) - } - } - - err = tx.Commit() - return err -} - -// DumpRules retrieves all domainRules in the database so they can be written to -// disk -func (padb *AuthorityDatabaseImpl) DumpRules() (rs RuleSet, err error) { - var bList []BlacklistRule - _, err = padb.dbMap.Select(&bList, "SELECT * FROM blacklist") - if err != nil { - return - } - for _, r := range bList { - r.Host = core.ReverseName(r.Host) - rs.Blacklist = append(rs.Blacklist, r) - } - var wList []WhitelistRule - _, err = padb.dbMap.Select(&wList, "SELECT * FROM whitelist") - if err != nil { - return - } - rs.Whitelist = wList - return rs, err -} - -// allowedByBlacklist returns nil if the host is allowed, errBlacklisted if the -// host is disallowed, or an InternalServerError if there was another problem -// checking the database. -func (padb *AuthorityDatabaseImpl) allowedByBlacklist(host string) error { - var rule BlacklistRule - // Use lexical ordering to quickly find blacklisted root domains - err := padb.dbMap.SelectOne( - &rule, - `SELECT * FROM blacklist WHERE :host >= host ORDER BY host DESC LIMIT 1`, - map[string]interface{}{"host": host}, - ) - if err != nil { - // No rows means not blacklisted, so no error. - if err == sql.ErrNoRows { - return nil - } - padb.log.Err(fmt.Sprintf("Error checking policy DB: %s", err)) - return errDBFailure - } - if host == rule.Host || strings.HasPrefix(host, rule.Host+".") { - return errBlacklisted - } - // If we got a result but it's not a match, that means the host is not - // blacklisted. - return nil -} - -func (padb *AuthorityDatabaseImpl) allowedByWhitelist(host string) bool { - var rule WhitelistRule - err := padb.dbMap.SelectOne( - &rule, - `SELECT * FROM whitelist WHERE :host = host LIMIT 1`, - map[string]interface{}{"host": host}, - ) - if err != nil { - if err == sql.ErrNoRows { - return false - } - return false - } - return true -} - -// CheckHostLists will query the database for white/blacklist rules that match host, -// if both whitelist and blacklist rules are found the blacklist will always win -// Returns errNotWhitelisted, errBlacklisted, or errDBFailure for the -// appropriate problems, or nil if the host is allowable. -func (padb *AuthorityDatabaseImpl) CheckHostLists(host string, requireWhitelisted bool) error { - if requireWhitelisted { - if !padb.allowedByWhitelist(host) { - return errNotWhitelisted - } - } - // Overrides the whitelist if a blacklist rule is found - host = core.ReverseName(host) - return padb.allowedByBlacklist(host) -} diff --git a/policy/policy-authority-data_test.go b/policy/policy-authority-data_test.go deleted file mode 100644 index aa6675864..000000000 --- a/policy/policy-authority-data_test.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2015 ISRG. All rights reserved -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package policy - -import ( - "fmt" - "testing" - - "github.com/letsencrypt/boulder/sa" - "github.com/letsencrypt/boulder/test" - "github.com/letsencrypt/boulder/test/vars" - - gorp "gopkg.in/gorp.v1" -) - -func padbImpl(t *testing.T) (*AuthorityDatabaseImpl, func()) { - dbMap, err := sa.NewDbMap(vars.DBConnPolicy) - test.AssertNotError(t, err, "Could not construct dbMap") - - padb, err := NewAuthorityDatabaseImpl(dbMap) - test.AssertNotError(t, err, "Couldn't create PADB") - - cleanUp := test.ResetPolicyTestDatabase(t) - - return padb, cleanUp -} - -func TestLoadAndDumpRules(t *testing.T) { - p, cleanup := padbImpl(t) - defer cleanup() - - load := RuleSet{ - Blacklist: []BlacklistRule{ - { - Host: "bad.com", - }, - }, - Whitelist: []WhitelistRule{ - { - Host: "good.bad.com", - }, - }, - } - err := p.LoadRules(load) - test.AssertNotError(t, err, "Couldn't load rules") - - dumped, err := p.DumpRules() - test.AssertNotError(t, err, "Couldn't dump rules") - test.AssertEquals(t, len(dumped.Blacklist), 1) - test.AssertEquals(t, len(dumped.Whitelist), 1) - - test.AssertEquals(t, dumped.Whitelist[0], load.Whitelist[0]) - test.AssertEquals(t, dumped.Blacklist[0], load.Blacklist[0]) -} - -// An implementation of the gorpDbMap interface that always returns an error -// from SelectOne. -type failureDB struct{} - -func (f *failureDB) AddTableWithName(interface{}, string) *gorp.TableMap { - return nil // not implemented -} - -func (f *failureDB) Begin() (*gorp.Transaction, error) { - return nil, nil // not implemented -} -func (f *failureDB) SelectOne(interface{}, string, ...interface{}) error { - return fmt.Errorf("DB failure") -} - -func (f *failureDB) Select(interface{}, string, ...interface{}) ([]interface{}, error) { - return nil, nil // not implemented -} - -func TestBlacklistError(t *testing.T) { - p, err := NewAuthorityDatabaseImpl(&failureDB{}) - test.AssertNotError(t, err, "Couldn't make PA") - err = p.CheckHostLists("bad.com", false) - test.AssertEquals(t, err, errDBFailure) -} - -func TestBlacklist(t *testing.T) { - p, cleanup := padbImpl(t) - defer cleanup() - - err := p.LoadRules(RuleSet{ - Blacklist: []BlacklistRule{ - { - Host: "bad.com", - }, - }, - Whitelist: []WhitelistRule{ - { - Host: "good.bad.com", - }, - }, - }) - test.AssertNotError(t, err, "Couldn't load rules") - - err = p.CheckHostLists("bad.com", false) - test.AssertError(t, err, "Hostname should be blacklisted") - err = p.CheckHostLists("still.bad.com", false) - test.AssertError(t, err, "Hostname should be blacklisted") - err = p.CheckHostLists("badminton.com", false) - test.AssertNotError(t, err, "Hostname shouldn't be blacklisted") - // Whitelisted subdomain of blacklisted root should still be blacklsited - err = p.CheckHostLists("good.bad.com", true) - test.AssertError(t, err, "Blacklist should beat whitelist") - // Not blacklisted - err = p.CheckHostLists("good.com", false) - test.AssertNotError(t, err, "Hostname shouldn't be blacklisted") -} - -func TestWhitelist(t *testing.T) { - p, cleanup := padbImpl(t) - defer cleanup() - - err := p.LoadRules(RuleSet{ - Blacklist: []BlacklistRule{ - { - Host: "bad.com", - }, - }, - Whitelist: []WhitelistRule{ - { - Host: "good.bad.com", - }, - { - Host: "good.com", - }, - }, - }) - test.AssertNotError(t, err, "Couldn't load rules") - - err = p.CheckHostLists("bad.com", true) - test.AssertError(t, err, "Hostname should be blacklisted") - // Whitelisted subdomain of blacklisted root should still be blacklsited - err = p.CheckHostLists("good.bad.com", true) - test.AssertError(t, err, "Blacklist should beat whitelist") - // Non-existent domain should fail - err = p.CheckHostLists("not-good.com", true) - test.AssertError(t, err, "Hostname isn't on whitelist") - // Whitelisted - err = p.CheckHostLists("good.com", true) - test.AssertNotError(t, err, "Hostname is on whitelist") -} diff --git a/policy/policy-authority.go b/policy/policy-authority.go index 5d885fa00..0f048a290 100644 --- a/policy/policy-authority.go +++ b/policy/policy-authority.go @@ -21,13 +21,11 @@ import ( "github.com/letsencrypt/boulder/reloader" "github.com/letsencrypt/net/publicsuffix" "github.com/square/go-jose" - "gopkg.in/gorp.v1" ) // AuthorityImpl enforces CA policy decisions. type AuthorityImpl struct { log blog.Logger - DB *AuthorityDatabaseImpl blacklist map[string]bool blacklistMu sync.RWMutex @@ -39,22 +37,10 @@ type AuthorityImpl struct { // New constructs a Policy Authority. // TODO(https://github.com/letsencrypt/boulder/issues/1616): Remove the _ bool // argument (used to be enforceWhitelist). Update all callers. -func New(dbMap *gorp.DbMap, _ bool, challengeTypes map[string]bool) (*AuthorityImpl, error) { - logger := blog.Get() - - var padb *AuthorityDatabaseImpl - if dbMap != nil { - // Setup policy db - var err error - padb, err = NewAuthorityDatabaseImpl(dbMap) - if err != nil { - return nil, err - } - } +func New(challengeTypes map[string]bool) (*AuthorityImpl, error) { pa := AuthorityImpl{ - log: logger, - DB: padb, + log: blog.Get(), enabledChallenges: challengeTypes, // We don't need real randomness for this. pseudoRNG: rand.New(rand.NewSource(99)), @@ -246,10 +232,6 @@ func (pa *AuthorityImpl) WillingToIssue(id core.AcmeIdentifier, regID int64) err } func (pa *AuthorityImpl) checkHostLists(domain string) error { - if pa.DB != nil { - return pa.DB.CheckHostLists(domain, false) - } - pa.blacklistMu.RLock() defer pa.blacklistMu.RUnlock() diff --git a/policy/policy-authority_test.go b/policy/policy-authority_test.go index e9f3eb185..262c2bb0b 100644 --- a/policy/policy-authority_test.go +++ b/policy/policy-authority_test.go @@ -7,16 +7,15 @@ package policy import ( "encoding/json" + "io/ioutil" + "os" "testing" "github.com/square/go-jose" "github.com/letsencrypt/boulder/core" blog "github.com/letsencrypt/boulder/log" - "github.com/letsencrypt/boulder/sa" "github.com/letsencrypt/boulder/test" - "github.com/letsencrypt/boulder/test/vars" - "gopkg.in/gorp.v1" ) var log = blog.UseMock() @@ -27,21 +26,12 @@ var enabledChallenges = map[string]bool{ core.ChallengeTypeDNS01: true, } -func paImpl(t *testing.T) (*AuthorityImpl, func()) { - dbMap, cleanUp := paDBMap(t) - pa, err := New(dbMap, false, enabledChallenges) +func paImpl(t *testing.T) *AuthorityImpl { + pa, err := New(enabledChallenges) if err != nil { - cleanUp() t.Fatalf("Couldn't create policy implementation: %s", err) } - return pa, cleanUp -} - -func paDBMap(t *testing.T) (*gorp.DbMap, func()) { - dbMap, err := sa.NewDbMap(vars.DBConnPolicy) - test.AssertNotError(t, err, "Could not construct dbMap") - cleanUp := test.ResetPolicyTestDatabase(t) - return dbMap, cleanUp + return pa } func TestWillingToIssue(t *testing.T) { @@ -127,14 +117,17 @@ func TestWillingToIssue(t *testing.T) { "www.zom-bo.com", } - pa, cleanup := paImpl(t) - defer cleanup() + pa := paImpl(t) - rules := RuleSet{} - for _, b := range shouldBeBlacklisted { - rules.Blacklist = append(rules.Blacklist, BlacklistRule{Host: b}) - } - err := pa.DB.LoadRules(rules) + blacklistBytes, err := json.Marshal(blacklistJSON{ + Blacklist: shouldBeBlacklisted, + }) + test.AssertNotError(t, err, "Couldn't serialize blacklist") + f, _ := ioutil.TempFile("", "test-blacklist.txt") + defer os.Remove(f.Name()) + err = ioutil.WriteFile(f.Name(), blacklistBytes, 0640) + test.AssertNotError(t, err, "Couldn't write blacklist") + err = pa.SetHostnamePolicyFile(f.Name()) test.AssertNotError(t, err, "Couldn't load rules") // Test for invalid identifier type @@ -187,8 +180,7 @@ var accountKeyJSON = `{ }` func TestChallengesFor(t *testing.T) { - pa, cleanup := paImpl(t) - defer cleanup() + pa := paImpl(t) var accountKey *jose.JsonWebKey err := json.Unmarshal([]byte(accountKeyJSON), &accountKey) diff --git a/ra/registration-authority_test.go b/ra/registration-authority_test.go index 2c6f35647..4cea67b4b 100644 --- a/ra/registration-authority_test.go +++ b/ra/registration-authority_test.go @@ -184,13 +184,10 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, *sa.SQLStorageAut va := &DummyValidationAuthority{} - paDbMap, err := sa.NewDbMap(vars.DBConnPolicy) - if err != nil { - t.Fatalf("Failed to create dbMap: %s", err) - } - policyDBCleanUp := test.ResetPolicyTestDatabase(t) - pa, err := policy.New(paDbMap, false, SupportedChallenges) + pa, err := policy.New(SupportedChallenges) test.AssertNotError(t, err, "Couldn't create PA") + err = pa.SetHostnamePolicyFile("../test/hostname-policy.json") + test.AssertNotError(t, err, "Couldn't set hostname policy") stats, _ := statsd.NewNoopClient() @@ -199,7 +196,6 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, *sa.SQLStorageAut } cleanUp := func() { saDBCleanUp() - policyDBCleanUp() } block, _ := pem.Decode(CSRPEM) diff --git a/test/create_db.sh b/test/create_db.sh index c05bf740d..030304976 100755 --- a/test/create_db.sh +++ b/test/create_db.sh @@ -22,36 +22,32 @@ mysql $dbconn -e "SET GLOBAL binlog_format = 'MIXED';" # Drop all users to get a fresh start mysql $dbconn < test/drop_users.sql -for svc in $SERVICES; do - for dbenv in $DBENVS; do - ( - db="boulder_${svc}_${dbenv}" - create_script="drop database if exists \`${db}\`; create database if not exists \`${db}\`;" +for dbenv in $DBENVS; do + ( + db="boulder_sa_${dbenv}" + create_script="drop database if exists \`${db}\`; create database if not exists \`${db}\`;" - mysql $dbconn -e "$create_script" || die "unable to create ${db}" + mysql $dbconn -e "$create_script" || die "unable to create ${db}" - echo "created empty ${db} database" + echo "created empty ${db} database" - goose -path=./$svc/_db/ -env=$dbenv up || die "unable to migrate ${db}" - echo "migrated ${db} database" + goose -path=./sa/_db/ -env=$dbenv up || die "unable to migrate ${db}" + echo "migrated ${db} database" - # With MYSQL_CONTAINER, patch the GRANT statements to - # use 127.0.0.1, not localhost, as MySQL may interpret - # 'username'@'localhost' to mean only users for UNIX - # socket connections. - USERS_SQL=test/${svc}_db_users.sql - if [[ -f $USERS_SQL ]]; then - if [[ $MYSQL_CONTAINER ]]; then - sed -e "s/'localhost'/'%'/g" < $USERS_SQL | \ - mysql $dbconn -D $db || die "unable to add users to ${db}" - else - sed -e "s/'localhost'/'127.%'/g" < $USERS_SQL | \ - mysql $dbconn -D $db < $USERS_SQL || die "unable to add users to ${db}" - fi - echo "added users to ${db}" - fi - ) & - done + # With MYSQL_CONTAINER, patch the GRANT statements to + # use 127.0.0.1, not localhost, as MySQL may interpret + # 'username'@'localhost' to mean only users for UNIX + # socket connections. + USERS_SQL=test/sa_db_users.sql + if [[ ${MYSQL_CONTAINER} ]]; then + sed -e "s/'localhost'/'%'/g" < ${USERS_SQL} | \ + mysql $dbconn -D $db || die "unable to add users to ${db}" + else + sed -e "s/'localhost'/'127.%'/g" < $USERS_SQL | \ + mysql $dbconn -D $db < $USERS_SQL || die "unable to add users to ${db}" + fi + echo "added users to ${db}" + ) & done wait diff --git a/test/db-common.sh b/test/db-common.sh index 622da86a9..594c6472e 100644 --- a/test/db-common.sh +++ b/test/db-common.sh @@ -6,7 +6,5 @@ function die() { exit 1 } -SERVICES="sa -policy" DBENVS="test integration" diff --git a/test/db.go b/test/db.go index b2dd6a67c..e6b42ce13 100644 --- a/test/db.go +++ b/test/db.go @@ -34,12 +34,6 @@ func ResetSATestDatabase(t testing.TB) func() { return resetTestDatabase(t, "sa") } -// ResetPolicyTestDatabase deletes all rows in all tables in the Policy DB. It -// acts the same as ResetSATestDatabase. -func ResetPolicyTestDatabase(t testing.TB) func() { - return resetTestDatabase(t, "policy") -} - func resetTestDatabase(t testing.TB, dbType string) func() { db, err := sql.Open("mysql", fmt.Sprintf("test_setup@tcp(boulder-mysql:3306)/boulder_%s_test", dbType)) if err != nil { diff --git a/test/hostname-policy.json b/test/hostname-policy.json index a5782e2b8..6397ee9a3 100644 --- a/test/hostname-policy.json +++ b/test/hostname-policy.json @@ -2,7 +2,6 @@ "Blacklist": [ "in-addr.arpa", "example", - "example.com", "example.net", "example.org", "invalid", diff --git a/test/policy_db_users.sql b/test/policy_db_users.sql deleted file mode 100644 index e1a4f4c96..000000000 --- a/test/policy_db_users.sql +++ /dev/null @@ -1,26 +0,0 @@ --- --- Copyright 2015 ISRG. All rights reserved --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. --- --- This file defines the default users for the primary database, used by --- all the parts of Boulder except the Certificate Authority module, which --- utilizes its own database. --- - --- Create users for each component with the appropriate permissions. We want to --- drop each user and recreate them, but if the user doesn't already exist, the --- drop command will fail. So we grant the dummy `USAGE` privilege to make sure --- the user exists and then drop the user. - --- Policy loader, CA, RA --- Note: The same config section, "pa" is used by the policy loader (for writes) --- and the CA and RA (for reads). So right now we have the one user that has --- both read and write permission, even though it would be better to give only --- read permission to CA and RA. -GRANT SELECT,INSERT,DELETE ON blacklist TO 'policy'@'localhost'; -GRANT SELECT,INSERT,DELETE ON whitelist TO 'policy'@'localhost'; - --- Test setup and teardown -GRANT ALL PRIVILEGES ON * to 'test_setup'@'localhost'; diff --git a/test/secrets/pa_dburl b/test/secrets/pa_dburl deleted file mode 100644 index 5f8d042b8..000000000 --- a/test/secrets/pa_dburl +++ /dev/null @@ -1 +0,0 @@ -mysql+tcp://policy@boulder-mysql:3306/boulder_policy_integration?readTimeout=800ms&writeTimeout=800ms