From 5383ab4aa92d252befb4e2df20ff6ccf53ed36f6 Mon Sep 17 00:00:00 2001 From: Jeff Hodges Date: Fri, 25 Sep 2015 12:52:43 -0700 Subject: [PATCH 1/7] remove the ca database The last step of removing the ca incremented serial number generation. Fixes #813. --- ca/_db/dbconf.yml | 9 --- .../20150818205601_InitialSchema.sql | 23 ------- .../20150821232907_FutzWithSerialNumber.sql | 30 --------- ca/certificate-authority-data.go | 60 ------------------ ca/certificate-authority-data_test.go | 61 ------------------- ca/certificate-authority.go | 26 +------- ca/certificate-authority_test.go | 24 +++----- cmd/boulder-ca/main.go | 8 +-- ra/registration-authority_test.go | 25 -------- test/boulder-config.json | 1 - test/db-common.sh | 3 +- 11 files changed, 13 insertions(+), 257 deletions(-) delete mode 100644 ca/_db/dbconf.yml delete mode 100644 ca/_db/migrations/20150818205601_InitialSchema.sql delete mode 100644 ca/_db/migrations/20150821232907_FutzWithSerialNumber.sql delete mode 100644 ca/certificate-authority-data.go delete mode 100644 ca/certificate-authority-data_test.go diff --git a/ca/_db/dbconf.yml b/ca/_db/dbconf.yml deleted file mode 100644 index d1e3814fa..000000000 --- a/ca/_db/dbconf.yml +++ /dev/null @@ -1,9 +0,0 @@ -development: - driver: mysql - open: boulder@tcp(localhost:3306)/boulder_ca_development -test: - driver: mysql - open: boulder@tcp(localhost:3306)/boulder_ca_test -integration: - driver: mysql - open: boulder@tcp(localhost:3306)/boulder_ca_integration \ No newline at end of file diff --git a/ca/_db/migrations/20150818205601_InitialSchema.sql b/ca/_db/migrations/20150818205601_InitialSchema.sql deleted file mode 100644 index eebcd3149..000000000 --- a/ca/_db/migrations/20150818205601_InitialSchema.sql +++ /dev/null @@ -1,23 +0,0 @@ - --- +goose Up --- SQL in section 'Up' is executed when this migration is applied - -CREATE TABLE `serialNumber` ( - `id` int(11) DEFAULT NULL, - `number` int(11) DEFAULT NULL, - `lastUpdated` datetime DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -INSERT INTO `serialNumber` - (`id`, - `number`, - `lastUpdated`) -VALUES (1, - 1, - now() -); - --- +goose Down --- SQL section 'Down' is executed when this migration is rolled back - -DROP TABLE `serialNumber` diff --git a/ca/_db/migrations/20150821232907_FutzWithSerialNumber.sql b/ca/_db/migrations/20150821232907_FutzWithSerialNumber.sql deleted file mode 100644 index 3f906b2ab..000000000 --- a/ca/_db/migrations/20150821232907_FutzWithSerialNumber.sql +++ /dev/null @@ -1,30 +0,0 @@ - --- +goose Up --- SQL in section 'Up' is executed when this migration is applied - -DROP TABLE `serialNumber`; -CREATE TABLE `serialNumber` ( - `id` bigint(20) unsigned NOT NULL auto_increment, - `stub` char(1) NOT NULL default '', - PRIMARY KEY (`id`), - UNIQUE KEY `stub` (`stub`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- +goose Down --- SQL section 'Down' is executed when this migration is rolled back - -DROP TABLE `serialNumber`; -CREATE TABLE `serialNumber` ( - `id` int(11) DEFAULT NULL, - `number` int(11) DEFAULT NULL, - `lastUpdated` datetime DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -INSERT INTO `serialNumber` - (`id`, - `number`, - `lastUpdated`) -VALUES (1, - 1, - now() -); diff --git a/ca/certificate-authority-data.go b/ca/certificate-authority-data.go deleted file mode 100644 index 3b5a827e8..000000000 --- a/ca/certificate-authority-data.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2015 ISRG. All rights reserved -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package ca - -import ( - "time" - - blog "github.com/letsencrypt/boulder/log" - - gorp "github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1" -) - -// CertificateAuthorityDatabaseImpl represents a database used by the CA; it -// enforces transaction semantics, and is effectively single-threaded. -type CertificateAuthorityDatabaseImpl struct { - log *blog.AuditLogger - dbMap *gorp.DbMap -} - -// SerialNumber defines the database table used to hold the serial number. -type SerialNumber struct { - ID int `db:"id"` - Number int64 `db:"number"` - LastUpdated time.Time `db:"lastUpdated"` -} - -// NewCertificateAuthorityDatabaseImpl constructs a Database for the -// Certificate Authority. -func NewCertificateAuthorityDatabaseImpl(dbMap *gorp.DbMap) (cadb *CertificateAuthorityDatabaseImpl, err error) { - logger := blog.GetAuditLogger() - - dbMap.AddTableWithName(SerialNumber{}, "serialNumber").SetKeys(true, "ID") - - cadb = &CertificateAuthorityDatabaseImpl{ - dbMap: dbMap, - log: logger, - } - return cadb, nil -} - -// Begin starts a transaction at the GORP wrapper. -func (cadb *CertificateAuthorityDatabaseImpl) Begin() (*gorp.Transaction, error) { - return cadb.dbMap.Begin() -} - -// IncrementAndGetSerial returns the next-available serial number, incrementing -// it in the database before returning. There must be an active transaction to -// call this method. Callers should Begin the transaction, call this method, -// perform any other work, and Commit at the end once the certificate is issued. -func (cadb *CertificateAuthorityDatabaseImpl) IncrementAndGetSerial(tx *gorp.Transaction) (int64, error) { - r, err := tx.Exec("REPLACE INTO serialNumber (stub) VALUES ('a');") - if err != nil { - return -1, err - } - - return r.LastInsertId() -} diff --git a/ca/certificate-authority-data_test.go b/ca/certificate-authority-data_test.go deleted file mode 100644 index 7b1936eee..000000000 --- a/ca/certificate-authority-data_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2015 ISRG. All rights reserved -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package ca - -import ( - "testing" - - "github.com/letsencrypt/boulder/sa" - "github.com/letsencrypt/boulder/test" -) - -func TestGetSetSequenceOutsideTx(t *testing.T) { - cadb, cleanUp := caDBImpl(t) - defer cleanUp() - tx, err := cadb.Begin() - test.AssertNotError(t, err, "Could not begin") - tx.Commit() - _, err = cadb.IncrementAndGetSerial(tx) - test.AssertError(t, err, "Not permitted") - - tx2, err := cadb.Begin() - test.AssertNotError(t, err, "Could not begin") - tx2.Rollback() - _, err = cadb.IncrementAndGetSerial(tx2) - test.AssertError(t, err, "Not permitted") -} - -func TestGetSetSequenceNumber(t *testing.T) { - cadb, cleanUp := caDBImpl(t) - defer cleanUp() - tx, err := cadb.Begin() - test.AssertNotError(t, err, "Could not begin") - - num, err := cadb.IncrementAndGetSerial(tx) - test.AssertNotError(t, err, "Could not get number") - - num2, err := cadb.IncrementAndGetSerial(tx) - test.AssertNotError(t, err, "Could not get number") - test.Assert(t, num+1 == num2, "Numbers should be incrementing") - - err = tx.Commit() - test.AssertNotError(t, err, "Could not commit") -} - -func caDBImpl(t *testing.T) (*CertificateAuthorityDatabaseImpl, func()) { - dbMap, err := sa.NewDbMap(caDBConnStr) - if err != nil { - t.Fatalf("Could not construct dbMap: %s", err) - } - - cadb, err := NewCertificateAuthorityDatabaseImpl(dbMap) - if err != nil { - t.Fatalf("Could not construct CA DB: %s", err) - } - - cleanUp := test.ResetTestDatabase(t, dbMap.Db) - return cadb, cleanUp -} diff --git a/ca/certificate-authority.go b/ca/certificate-authority.go index 6c6168ae1..cd637158b 100644 --- a/ca/certificate-authority.go +++ b/ca/certificate-authority.go @@ -54,7 +54,6 @@ type CertificateAuthorityImpl struct { OCSPSigner ocsp.Signer SA core.StorageAuthority PA core.PolicyAuthority - DB core.CertificateAuthorityDatabase Publisher core.Publisher Clk clock.Clock // TODO(jmhodges): should be private, like log log *blog.AuditLogger @@ -70,7 +69,7 @@ type CertificateAuthorityImpl struct { // using CFSSL's authenticated signature scheme. A CA created in this way // issues for a single profile on the remote signer, which is indicated // by name in this constructor. -func NewCertificateAuthorityImpl(cadb core.CertificateAuthorityDatabase, config cmd.CAConfig, clk clock.Clock, issuerCert string) (*CertificateAuthorityImpl, error) { +func NewCertificateAuthorityImpl(config cmd.CAConfig, clk clock.Clock, issuerCert string) (*CertificateAuthorityImpl, error) { var ca *CertificateAuthorityImpl var err error logger := blog.GetAuditLogger() @@ -127,7 +126,6 @@ func NewCertificateAuthorityImpl(cadb core.CertificateAuthorityDatabase, config Signer: signer, OCSPSigner: ocspSigner, profile: config.Profile, - DB: cadb, Prefix: config.SerialPrefix, Clk: clk, log: logger, @@ -306,15 +304,6 @@ func (ca *CertificateAuthorityImpl) IssueCertificate(csr x509.CertificateRequest Bytes: csr.Raw, })) - // Get the next serial number - tx, err := ca.DB.Begin() - if err != nil { - err = core.InternalServerError(err.Error()) - // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 - ca.log.AuditErr(err) - return emptyCert, err - } - // Hack: CFSSL always sticks a 64-bit random number at the end of the // serialSeq we provide, but we want 136 bits of random number, plus an 8-bit // instance id prefix. For now, we generate the extra 72 bits of randomness @@ -328,7 +317,6 @@ func (ca *CertificateAuthorityImpl) IssueCertificate(csr x509.CertificateRequest err = core.InternalServerError(err.Error()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 ca.log.Audit(fmt.Sprintf("Serial randomness failed, err=[%v]", err)) - tx.Rollback() return emptyCert, err } serialHex := hex.EncodeToString([]byte{byte(ca.Prefix)}) + hex.EncodeToString(randSlice) @@ -349,7 +337,6 @@ func (ca *CertificateAuthorityImpl) IssueCertificate(csr x509.CertificateRequest err = core.InternalServerError(err.Error()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 ca.log.Audit(fmt.Sprintf("Signer failed, rolling back: serial=[%s] err=[%v]", serialHex, err)) - tx.Rollback() return emptyCert, err } @@ -357,7 +344,6 @@ func (ca *CertificateAuthorityImpl) IssueCertificate(csr x509.CertificateRequest err = core.InternalServerError("No certificate returned by server") // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 ca.log.Audit(fmt.Sprintf("PEM empty from Signer, rolling back: serial=[%s] err=[%v]", serialHex, err)) - tx.Rollback() return emptyCert, err } @@ -366,7 +352,6 @@ func (ca *CertificateAuthorityImpl) IssueCertificate(csr x509.CertificateRequest err = core.InternalServerError("Invalid certificate value returned") // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 ca.log.Audit(fmt.Sprintf("PEM decode error, aborting and rolling back issuance: pem=[%s] err=[%v]", certPEM, err)) - tx.Rollback() return emptyCert, err } certDER := block.Bytes @@ -380,7 +365,6 @@ func (ca *CertificateAuthorityImpl) IssueCertificate(csr x509.CertificateRequest err = core.InternalServerError(err.Error()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 ca.log.Audit(fmt.Sprintf("Uncaught error, aborting and rolling back issuance: pem=[%s] err=[%v]", certPEM, err)) - tx.Rollback() return emptyCert, err } @@ -390,14 +374,6 @@ func (ca *CertificateAuthorityImpl) IssueCertificate(csr x509.CertificateRequest err = core.InternalServerError(err.Error()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 ca.log.Audit(fmt.Sprintf("Failed RPC to store at SA, orphaning certificate: pem=[%s] err=[%v]", certPEM, err)) - tx.Rollback() - return emptyCert, err - } - - if err = tx.Commit(); err != nil { - err = core.InternalServerError(err.Error()) - // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 - ca.log.Audit(fmt.Sprintf("Failed to commit, orphaning certificate: pem=[%s] err=[%v]", certPEM, err)) return emptyCert, err } diff --git a/ca/certificate-authority_test.go b/ca/certificate-authority_test.go index dd0fc7fe2..5cc5bde54 100644 --- a/ca/certificate-authority_test.go +++ b/ca/certificate-authority_test.go @@ -91,7 +91,6 @@ const caCertFile = "../test/test-ca.pem" const ( paDBConnStr = "mysql+tcp://boulder@localhost:3306/boulder_policy_test" - caDBConnStr = "mysql+tcp://boulder@localhost:3306/boulder_ca_test" saDBConnStr = "mysql+tcp://boulder@localhost:3306/boulder_sa_test" ) @@ -104,7 +103,6 @@ func mustRead(path string) []byte { } type testCtx struct { - caDB core.CertificateAuthorityDatabase sa core.StorageAuthority caConfig cmd.CAConfig reg core.Registration @@ -126,7 +124,6 @@ func setup(t *testing.T) *testCtx { t.Fatalf("Failed to create SA: %s", err) } saDBCleanUp := test.ResetTestDatabase(t, dbMap.Db) - cadb, caDBCleanUp := caDBImpl(t) paDbMap, err := sa.NewDbMap(paDBConnStr) test.AssertNotError(t, err, "Could not construct dbMap") @@ -136,7 +133,6 @@ func setup(t *testing.T) *testCtx { cleanUp := func() { saDBCleanUp() - caDBCleanUp() paDBCleanUp() } @@ -188,7 +184,7 @@ func setup(t *testing.T) *testCtx { }, }, } - return &testCtx{cadb, ssa, caConfig, reg, pa, fc, cleanUp} + return &testCtx{ssa, caConfig, reg, pa, fc, cleanUp} } func TestFailNoSerial(t *testing.T) { @@ -196,14 +192,14 @@ func TestFailNoSerial(t *testing.T) { defer ctx.cleanUp() ctx.caConfig.SerialPrefix = 0 - _, err := NewCertificateAuthorityImpl(ctx.caDB, ctx.caConfig, ctx.fc, caCertFile) + _, err := NewCertificateAuthorityImpl(ctx.caConfig, ctx.fc, caCertFile) test.AssertError(t, err, "CA should have failed with no SerialPrefix") } func TestRevoke(t *testing.T) { ctx := setup(t) defer ctx.cleanUp() - ca, err := NewCertificateAuthorityImpl(ctx.caDB, ctx.caConfig, ctx.fc, caCertFile) + ca, err := NewCertificateAuthorityImpl(ctx.caConfig, ctx.fc, caCertFile) test.AssertNotError(t, err, "Failed to create CA") ca.PA = ctx.pa ca.SA = ctx.sa @@ -244,7 +240,7 @@ func TestRevoke(t *testing.T) { func TestIssueCertificate(t *testing.T) { ctx := setup(t) defer ctx.cleanUp() - ca, err := NewCertificateAuthorityImpl(ctx.caDB, ctx.caConfig, ctx.fc, caCertFile) + ca, err := NewCertificateAuthorityImpl(ctx.caConfig, ctx.fc, caCertFile) test.AssertNotError(t, err, "Failed to create CA") ca.Publisher = &mocks.MockPublisher{} ca.PA = ctx.pa @@ -321,7 +317,7 @@ func TestIssueCertificate(t *testing.T) { func TestRejectNoName(t *testing.T) { ctx := setup(t) defer ctx.cleanUp() - ca, err := NewCertificateAuthorityImpl(ctx.caDB, ctx.caConfig, ctx.fc, caCertFile) + ca, err := NewCertificateAuthorityImpl(ctx.caConfig, ctx.fc, caCertFile) test.AssertNotError(t, err, "Failed to create CA") ca.Publisher = &mocks.MockPublisher{} ca.PA = ctx.pa @@ -338,7 +334,7 @@ func TestRejectNoName(t *testing.T) { func TestRejectTooManyNames(t *testing.T) { ctx := setup(t) defer ctx.cleanUp() - ca, err := NewCertificateAuthorityImpl(ctx.caDB, ctx.caConfig, ctx.fc, caCertFile) + ca, err := NewCertificateAuthorityImpl(ctx.caConfig, ctx.fc, caCertFile) test.AssertNotError(t, err, "Failed to create CA") ca.Publisher = &mocks.MockPublisher{} ca.PA = ctx.pa @@ -355,7 +351,7 @@ func TestRejectTooManyNames(t *testing.T) { func TestDeduplication(t *testing.T) { ctx := setup(t) defer ctx.cleanUp() - ca, err := NewCertificateAuthorityImpl(ctx.caDB, ctx.caConfig, ctx.fc, caCertFile) + ca, err := NewCertificateAuthorityImpl(ctx.caConfig, ctx.fc, caCertFile) test.AssertNotError(t, err, "Failed to create CA") ca.Publisher = &mocks.MockPublisher{} ca.PA = ctx.pa @@ -379,7 +375,7 @@ func TestDeduplication(t *testing.T) { func TestRejectValidityTooLong(t *testing.T) { ctx := setup(t) defer ctx.cleanUp() - ca, err := NewCertificateAuthorityImpl(ctx.caDB, ctx.caConfig, ctx.fc, caCertFile) + ca, err := NewCertificateAuthorityImpl(ctx.caConfig, ctx.fc, caCertFile) test.AssertNotError(t, err, "Failed to create CA") ca.Publisher = &mocks.MockPublisher{} ca.PA = ctx.pa @@ -397,7 +393,7 @@ func TestRejectValidityTooLong(t *testing.T) { func TestShortKey(t *testing.T) { ctx := setup(t) defer ctx.cleanUp() - ca, err := NewCertificateAuthorityImpl(ctx.caDB, ctx.caConfig, ctx.fc, caCertFile) + ca, err := NewCertificateAuthorityImpl(ctx.caConfig, ctx.fc, caCertFile) ca.Publisher = &mocks.MockPublisher{} ca.PA = ctx.pa ca.SA = ctx.sa @@ -413,7 +409,7 @@ func TestShortKey(t *testing.T) { func TestRejectBadAlgorithm(t *testing.T) { ctx := setup(t) defer ctx.cleanUp() - ca, err := NewCertificateAuthorityImpl(ctx.caDB, ctx.caConfig, ctx.fc, caCertFile) + ca, err := NewCertificateAuthorityImpl(ctx.caConfig, ctx.fc, caCertFile) ca.Publisher = &mocks.MockPublisher{} ca.PA = ctx.pa ca.SA = ctx.sa diff --git a/cmd/boulder-ca/main.go b/cmd/boulder-ca/main.go index 1c5600e8d..30970a8b5 100644 --- a/cmd/boulder-ca/main.go +++ b/cmd/boulder-ca/main.go @@ -33,18 +33,12 @@ func main() { go cmd.DebugServer(c.CA.DebugAddr) - dbMap, err := sa.NewDbMap(c.CA.DBConnect) - cmd.FailOnError(err, "Couldn't connect to CA database") - - cadb, err := ca.NewCertificateAuthorityDatabaseImpl(dbMap) - cmd.FailOnError(err, "Failed to create CA database") - paDbMap, err := sa.NewDbMap(c.PA.DBConnect) cmd.FailOnError(err, "Couldn't connect to policy database") pa, err := policy.NewPolicyAuthorityImpl(paDbMap, c.PA.EnforcePolicyWhitelist) cmd.FailOnError(err, "Couldn't create PA") - cai, err := ca.NewCertificateAuthorityImpl(cadb, c.CA, clock.Default(), c.Common.IssuerCert) + cai, err := ca.NewCertificateAuthorityImpl(c.CA, clock.Default(), c.Common.IssuerCert) cmd.FailOnError(err, "Failed to create CA impl") cai.PA = pa diff --git a/ra/registration-authority_test.go b/ra/registration-authority_test.go index a4f5c9883..34b369206 100644 --- a/ra/registration-authority_test.go +++ b/ra/registration-authority_test.go @@ -128,7 +128,6 @@ var ( const ( paDBConnStr = "mysql+tcp://boulder@localhost:3306/boulder_policy_test" - caDBConnStr = "mysql+tcp://boulder@localhost:3306/boulder_ca_test" saDBConnStr = "mysql+tcp://boulder@localhost:3306/boulder_sa_test" ) @@ -187,13 +186,11 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, *sa.SQLStorageAut policyDBCleanUp := test.ResetTestDatabase(t, paDbMap.Db) pa, err := policy.NewPolicyAuthorityImpl(paDbMap, false) test.AssertNotError(t, err, "Couldn't create PA") - cadb, caDBCleanUp := caDBImpl(t) ca := ca.CertificateAuthorityImpl{ Signer: signer, OCSPSigner: ocspSigner, SA: ssa, PA: pa, - DB: cadb, Publisher: &mocks.MockPublisher{}, ValidityPeriod: time.Hour * 2190, NotAfter: time.Now().Add(time.Hour * 8761), @@ -201,7 +198,6 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, *sa.SQLStorageAut } cleanUp := func() { saDBCleanUp() - caDBCleanUp() policyDBCleanUp() } @@ -229,27 +225,6 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, *sa.SQLStorageAut return va, ssa, &ra, fc, cleanUp } -// This is an unfortunate bit of tech debt that is being taken on in -// order to get the more important change of using MySQL/MariaDB in -// all of our tests working without SQLite. We already had issues with -// the RA here getting a real CertificateAuthority instead of a -// CertificateAuthorityClient, so this is only marginally worse. -// TODO(Issue #628): use a CAClient fake instead of a CAImpl instance -func caDBImpl(t *testing.T) (core.CertificateAuthorityDatabase, func()) { - dbMap, err := sa.NewDbMap(caDBConnStr) - if err != nil { - t.Fatalf("Could not construct dbMap: %s", err) - } - - cadb, err := ca.NewCertificateAuthorityDatabaseImpl(dbMap) - if err != nil { - t.Fatalf("Could not construct CA DB: %s", err) - } - - cleanUp := test.ResetTestDatabase(t, dbMap.Db) - return cadb, cleanUp -} - func assertAuthzEqual(t *testing.T, a1, a2 core.Authorization) { test.Assert(t, a1.ID == a2.ID, "ret != DB: ID") test.Assert(t, a1.Identifier == a2.Identifier, "ret != DB: Identifier") diff --git a/test/boulder-config.json b/test/boulder-config.json index 10692373f..52812313e 100644 --- a/test/boulder-config.json +++ b/test/boulder-config.json @@ -54,7 +54,6 @@ "ca": { "serialPrefix": 255, "profile": "ee", - "dbConnect": "mysql+tcp://boulder@localhost:3306/boulder_ca_integration", "debugAddr": "localhost:8001", "Key": { "File": "test/test-ca.key" diff --git a/test/db-common.sh b/test/db-common.sh index 5bfb68256..a91f5e930 100644 --- a/test/db-common.sh +++ b/test/db-common.sh @@ -6,8 +6,7 @@ function die() { exit 1 } -SERVICES="ca -sa +SERVICES="sa policy" DBENVS="development test From ecd08c0798c653f11638ddda3d6910b36b4948cd Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sun, 27 Sep 2015 12:36:18 -0700 Subject: [PATCH 2/7] Allow HEAD for all GET resources. --- wfe/web-front-end.go | 7 +++++-- wfe/web-front-end_test.go | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/wfe/web-front-end.go b/wfe/web-front-end.go index 322ee66a0..874fbc72e 100644 --- a/wfe/web-front-end.go +++ b/wfe/web-front-end.go @@ -154,8 +154,9 @@ func (mrw BodylessResponseWriter) Write(buf []byte) (int, error) { // * Respond http.StatusMethodNotAllowed for HTTP methods other than // those listed. // -// * Never send a body in response to a HEAD request. (Anything -// written by the handler will be discarded if the method is HEAD.) +// * Never send a body in response to a HEAD request. Anything +// written by the handler will be discarded if the method is HEAD. Also, all +// handlers that accept GET automatically accept HEAD. func (wfe *WebFrontEndImpl) HandleFunc(mux *http.ServeMux, pattern string, h func(http.ResponseWriter, *http.Request), methods ...string) { methodsOK := make(map[string]bool) for _, m := range methods { @@ -176,6 +177,8 @@ func (wfe *WebFrontEndImpl) HandleFunc(mux *http.ServeMux, pattern string, h fun // should still comply with HTTP spec by not // sending a body. response = BodylessResponseWriter{response} + h(response, request) + return case "OPTIONS": // TODO, #469 } diff --git a/wfe/web-front-end_test.go b/wfe/web-front-end_test.go index c4bfc12bf..94a64ced6 100644 --- a/wfe/web-front-end_test.go +++ b/wfe/web-front-end_test.go @@ -308,9 +308,8 @@ func TestHandleFunc(t *testing.T) { // Disallowed method special case: response to HEAD has got no body runWrappedHandler(&http.Request{Method: "HEAD"}, "GET", "POST") - test.AssertEquals(t, stubCalled, false) + test.AssertEquals(t, stubCalled, true) test.AssertEquals(t, rw.Body.String(), "") - test.AssertEquals(t, sortHeader(rw.Header().Get("Allow")), "GET, POST") } func TestStandardHeaders(t *testing.T) { From 1fbf2d49900d1d852e12bafb009d85406e28533a Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sun, 27 Sep 2015 23:49:59 -0700 Subject: [PATCH 3/7] Fix HEAD to work only for GET resources. --- wfe/web-front-end.go | 6 ++---- wfe/web-front-end_test.go | 7 +++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/wfe/web-front-end.go b/wfe/web-front-end.go index 874fbc72e..e09984437 100644 --- a/wfe/web-front-end.go +++ b/wfe/web-front-end.go @@ -171,16 +171,14 @@ func (wfe *WebFrontEndImpl) HandleFunc(mux *http.ServeMux, pattern string, h fun } response.Header().Set("Access-Control-Allow-Origin", "*") - switch request.Method { - case "HEAD": + // Return a bodyless response to HEAD for any resource that allows GET. + if _, ok := methodsOK["GET"]; ok && request.Method == "HEAD" { // We'll be sending an error anyway, but we // should still comply with HTTP spec by not // sending a body. response = BodylessResponseWriter{response} h(response, request) return - case "OPTIONS": - // TODO, #469 } if _, ok := methodsOK[request.Method]; !ok { diff --git a/wfe/web-front-end_test.go b/wfe/web-front-end_test.go index 94a64ced6..a41435011 100644 --- a/wfe/web-front-end_test.go +++ b/wfe/web-front-end_test.go @@ -310,6 +310,13 @@ func TestHandleFunc(t *testing.T) { runWrappedHandler(&http.Request{Method: "HEAD"}, "GET", "POST") test.AssertEquals(t, stubCalled, true) test.AssertEquals(t, rw.Body.String(), "") + + // HEAD doesn't work with POST-only endpoints + runWrappedHandler(&http.Request{Method: "HEAD"}, "POST") + test.AssertEquals(t, stubCalled, false) + test.AssertEquals(t, rw.Header().Get("Content-Type"), "application/problem+json") + test.AssertEquals(t, rw.Body.String(), `{"type":"urn:acme:error:malformed","detail":"Method not allowed"}`) + test.AssertEquals(t, rw.Header().Get("Allow"), "POST") } func TestStandardHeaders(t *testing.T) { From 9287d7ecd809239ccd624d89bce45621ed134ce2 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Mon, 28 Sep 2015 16:17:41 -0700 Subject: [PATCH 4/7] Check a CT submission happens during integration tests --- test/amqp-integration-test.py | 11 +++++- test/ct-test-srv/main.go | 73 +++++++++++++++++++++++------------ 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/test/amqp-integration-test.py b/test/amqp-integration-test.py index eac41c2d3..60eca0e4f 100644 --- a/test/amqp-integration-test.py +++ b/test/amqp-integration-test.py @@ -7,12 +7,13 @@ import socket import subprocess import sys import tempfile +import urllib2 import startservers class ExitStatus: - OK, PythonFailure, NodeFailure, Error, OCSPFailure = range(5) + OK, PythonFailure, NodeFailure, Error, OCSPFailure, CTFailure = range(6) class ProcInfo: @@ -62,6 +63,13 @@ def verify_ocsp_revoked(certFile, url): die(ExitStatus.OCSPFailure) pass +def verify_ct_submission(expectedSubmissions, url): + resp = urllib2.urlopen(url) + submissionStr = resp.read() + if int(submissionStr) != expectedSubmissions: + print "Expected %d submissions, found %d" % (expectedSubmissions, int(submissionStr)) + die(ExitStatus.CTFailure) + def run_node_test(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: @@ -91,6 +99,7 @@ def run_node_test(): # Also verify that the static OCSP responder, which answers with a # pre-signed, long-lived response for the CA cert, also works. verify_ocsp_good("../test-ca.der", issuer_ocsp_url) + verify_ct_submission(1, "http://localhost:4500/submissions") if subprocess.Popen(''' node revoke.js %s %s http://localhost:4000/acme/revoke-cert diff --git a/test/ct-test-srv/main.go b/test/ct-test-srv/main.go index 7b55d836e..8e4b8cef8 100644 --- a/test/ct-test-srv/main.go +++ b/test/ct-test-srv/main.go @@ -10,47 +10,70 @@ package main import ( "encoding/json" + "fmt" "io/ioutil" "log" "net/http" + "sync/atomic" ) type ctSubmissionRequest struct { Chain []string `json:"chain"` } -func handler(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" || r.URL.Path != "/ct/v1/add-chain" { +type integrationSrv struct { + submissions int64 +} + +func (is *integrationSrv) handler(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/ct/v1/add-chain": + if r.Method != "POST" { + http.NotFound(w, r) + return + } + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + + var addChainReq ctSubmissionRequest + err = json.Unmarshal(bodyBytes, &addChainReq) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + + w.WriteHeader(http.StatusOK) + // id is a sha256 of a random EC key. Generate your own with: + // openssl ecparam -name prime256v1 -genkey -outform der | openssl sha256 -binary | base64 + w.Write([]byte(`{ + "sct_version": 0, + "id": "8fjM8cvLPOhzCFwI62IYJhjkOcvWFLx1dMJbs0uhxJU=", + "timestamp": 1442400000, + "extensions": "", + "signature": "BAMARzBFAiBB5wKED8KqKhADT37n0y28fZIPiGbCfZRVKq0wNo0hrwIhAOIa2tPBF/rB1y30Y/ROh4LBmJ0mItAbTWy8XZKh7Wcp" + }`)) + atomic.AddInt64(&is.submissions, 1) + case "/submissions": + if r.Method != "GET" { + http.NotFound(w, r) + return + } + + submissions := atomic.LoadInt64(&is.submissions) + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf("%d", submissions))) + default: http.NotFound(w, r) return } - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - } - - var addChainReq ctSubmissionRequest - err = json.Unmarshal(bodyBytes, &addChainReq) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - } - - w.WriteHeader(http.StatusOK) - // id is a sha256 of a random EC key. Generate your own with: - // openssl ecparam -name prime256v1 -genkey -outform der | openssl sha256 -binary | base64 - w.Write([]byte(`{ - "sct_version": 0, - "id": "8fjM8cvLPOhzCFwI62IYJhjkOcvWFLx1dMJbs0uhxJU=", - "timestamp": 1442400000, - "extensions": "", - "signature": "BAMARzBFAiBB5wKED8KqKhADT37n0y28fZIPiGbCfZRVKq0wNo0hrwIhAOIa2tPBF/rB1y30Y/ROh4LBmJ0mItAbTWy8XZKh7Wcp" - }`)) } func main() { + is := integrationSrv{} s := &http.Server{ - Addr: ":4500", - Handler: http.HandlerFunc(handler), + Addr: "localhost:4500", + Handler: http.HandlerFunc(is.handler), } log.Fatal(s.ListenAndServe()) } From f9dff78ca2d9590a6c4553de7a94bcf2be25c1a2 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 28 Sep 2015 19:13:03 -0700 Subject: [PATCH 5/7] Enable SQL debug in SA test. --- sa/database.go | 2 +- sa/storage-authority_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sa/database.go b/sa/database.go index dc2ad3031..609f23665 100644 --- a/sa/database.go +++ b/sa/database.go @@ -119,7 +119,7 @@ func SetSQLDebug(dbMap *gorp.DbMap, state bool) { // SQLLogger adapts the AuditLogger to a format GORP can use. type SQLLogger struct { - log *blog.AuditLogger + log blog.SyslogWriter } // Printf adapts the AuditLogger to GORP's interface diff --git a/sa/storage-authority_test.go b/sa/storage-authority_test.go index e781b9447..817e263b4 100644 --- a/sa/storage-authority_test.go +++ b/sa/storage-authority_test.go @@ -38,6 +38,7 @@ func initSA(t *testing.T) (*SQLStorageAuthority, clock.FakeClock, func()) { if err != nil { t.Fatalf("Failed to create dbMap: %s", err) } + dbMap.TraceOn("SQL: ", &SQLLogger{log}) fc := clock.NewFake() fc.Add(1 * time.Hour) From e97880aaa790a5308bd0c992af47e5139551184b Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 29 Sep 2015 17:16:03 -0700 Subject: [PATCH 6/7] Audit log version info as early as possible. This means after parsing the config file, setting up stats, and dialing the syslogger. But it is still before trying to initialize the given server. This means that we are more likely to get version numbers logged for some common runtime failures. --- cmd/activity-monitor/main.go | 4 +--- cmd/boulder-ca/main.go | 3 +-- cmd/boulder-publisher/main.go | 3 +-- cmd/boulder-ra/main.go | 3 +-- cmd/boulder-sa/main.go | 3 +-- cmd/boulder-va/main.go | 3 +-- cmd/boulder-wfe/main.go | 3 +-- cmd/cert-checker/main.go | 2 +- cmd/expiration-mailer/main.go | 3 +-- cmd/ocsp-responder/main.go | 3 +-- cmd/ocsp-updater/main.go | 3 +-- 11 files changed, 11 insertions(+), 22 deletions(-) diff --git a/cmd/activity-monitor/main.go b/cmd/activity-monitor/main.go index cf6db100d..763d3bba4 100644 --- a/cmd/activity-monitor/main.go +++ b/cmd/activity-monitor/main.go @@ -118,8 +118,8 @@ func main() { cmd.FailOnError(err, "Could not connect to statsd") auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) - cmd.FailOnError(err, "Could not connect to Syslog") + auditlogger.Info(app.VersionString()) blog.SetAuditLogger(auditlogger) @@ -131,8 +131,6 @@ func main() { go cmd.ProfileCmd("AM", stats) - auditlogger.Info(app.VersionString()) - startMonitor(ch, auditlogger, stats) } diff --git a/cmd/boulder-ca/main.go b/cmd/boulder-ca/main.go index 30970a8b5..beffffed3 100644 --- a/cmd/boulder-ca/main.go +++ b/cmd/boulder-ca/main.go @@ -25,6 +25,7 @@ func main() { // Set up logging auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) cmd.FailOnError(err, "Could not connect to Syslog") + auditlogger.Info(app.VersionString()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 defer auditlogger.AuditPanic() @@ -65,8 +66,6 @@ func main() { cmd.FailOnError(err, "Unable to create CA RPC server") rpc.NewCertificateAuthorityServer(cas, cai) - auditlogger.Info(app.VersionString()) - err = cas.Start(c) cmd.FailOnError(err, "Unable to run CA RPC server") } diff --git a/cmd/boulder-publisher/main.go b/cmd/boulder-publisher/main.go index 448ca88df..9eda05194 100644 --- a/cmd/boulder-publisher/main.go +++ b/cmd/boulder-publisher/main.go @@ -23,6 +23,7 @@ func main() { // Set up logging auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) cmd.FailOnError(err, "Could not connect to syslog") + auditlogger.Info(app.VersionString()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 defer auditlogger.AuditPanic() @@ -49,8 +50,6 @@ func main() { cmd.FailOnError(err, "Unable to create Publisher RPC server") rpc.NewPublisherServer(pubs, &pubi) - auditlogger.Info(app.VersionString()) - err = pubs.Start(c) cmd.FailOnError(err, "Unable to run Publisher RPC server") } diff --git a/cmd/boulder-ra/main.go b/cmd/boulder-ra/main.go index 820907b58..a4d7886c6 100644 --- a/cmd/boulder-ra/main.go +++ b/cmd/boulder-ra/main.go @@ -29,6 +29,7 @@ func main() { // Set up logging auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) cmd.FailOnError(err, "Could not connect to Syslog") + auditlogger.Info(app.VersionString()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 defer auditlogger.AuditPanic() @@ -82,8 +83,6 @@ func main() { cmd.FailOnError(err, "Unable to create RA RPC server") rpc.NewRegistrationAuthorityServer(ras, &rai) - auditlogger.Info(app.VersionString()) - err = ras.Start(c) cmd.FailOnError(err, "Unable to run RA RPC server") } diff --git a/cmd/boulder-sa/main.go b/cmd/boulder-sa/main.go index f1a255eb2..44fd1dbf7 100644 --- a/cmd/boulder-sa/main.go +++ b/cmd/boulder-sa/main.go @@ -23,6 +23,7 @@ func main() { // Set up logging auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) cmd.FailOnError(err, "Could not connect to Syslog") + auditlogger.Info(app.VersionString()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 defer auditlogger.AuditPanic() @@ -46,8 +47,6 @@ func main() { cmd.FailOnError(err, "Unable to create SA RPC server") rpc.NewStorageAuthorityServer(sas, sai) - auditlogger.Info(app.VersionString()) - err = sas.Start(c) cmd.FailOnError(err, "Unable to run SA RPC server") } diff --git a/cmd/boulder-va/main.go b/cmd/boulder-va/main.go index b854cb0ca..d4adbcc73 100644 --- a/cmd/boulder-va/main.go +++ b/cmd/boulder-va/main.go @@ -27,6 +27,7 @@ func main() { // Set up logging auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) cmd.FailOnError(err, "Could not connect to Syslog") + auditlogger.Info(app.VersionString()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 defer auditlogger.AuditPanic() @@ -75,8 +76,6 @@ func main() { cmd.FailOnError(err, "Unable to create VA RPC server") rpc.NewValidationAuthorityServer(vas, vai) - auditlogger.Info(app.VersionString()) - err = vas.Start(c) cmd.FailOnError(err, "Unable to run VA RPC server") } diff --git a/cmd/boulder-wfe/main.go b/cmd/boulder-wfe/main.go index e100ff9c5..431ba2a56 100644 --- a/cmd/boulder-wfe/main.go +++ b/cmd/boulder-wfe/main.go @@ -66,6 +66,7 @@ func main() { auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) cmd.FailOnError(err, "Could not connect to Syslog") + auditlogger.Info(app.VersionString()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 defer auditlogger.AuditPanic() @@ -120,8 +121,6 @@ func main() { h, err := wfe.Handler() cmd.FailOnError(err, "Problem setting up HTTP handlers") - auditlogger.Info(app.VersionString()) - httpMonitor := metrics.NewHTTPMonitor(stats, h, "WFE") auditlogger.Info(fmt.Sprintf("Server running, listening on %s...\n", c.WFE.ListenAddress)) diff --git a/cmd/cert-checker/main.go b/cmd/cert-checker/main.go index 91dca3b35..2f01215cb 100644 --- a/cmd/cert-checker/main.go +++ b/cmd/cert-checker/main.go @@ -239,9 +239,9 @@ func main() { auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) cmd.FailOnError(err, "Could not connect to Syslog") + auditlogger.Info(app.VersionString()) blog.SetAuditLogger(auditlogger) - auditlogger.Info(app.VersionString()) saDbMap, err := sa.NewDbMap(c.CertChecker.DBConnect) cmd.FailOnError(err, "Could not connect to database") diff --git a/cmd/expiration-mailer/main.go b/cmd/expiration-mailer/main.go index 83337de62..c95c9c52a 100644 --- a/cmd/expiration-mailer/main.go +++ b/cmd/expiration-mailer/main.go @@ -230,14 +230,13 @@ func main() { auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) cmd.FailOnError(err, "Could not connect to Syslog") + auditlogger.Info(app.VersionString()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 defer auditlogger.AuditPanic() blog.SetAuditLogger(auditlogger) - auditlogger.Info(app.VersionString()) - go cmd.DebugServer(c.Mailer.DebugAddr) // Configure DB diff --git a/cmd/ocsp-responder/main.go b/cmd/ocsp-responder/main.go index 02e1c6724..f807528fd 100644 --- a/cmd/ocsp-responder/main.go +++ b/cmd/ocsp-responder/main.go @@ -131,6 +131,7 @@ func main() { auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) cmd.FailOnError(err, "Could not connect to Syslog") + auditlogger.Info(app.VersionString()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 defer auditlogger.AuditPanic() @@ -141,8 +142,6 @@ func main() { go cmd.ProfileCmd("OCSP", stats) - auditlogger.Info(app.VersionString()) - config := c.OCSPResponder var source cfocsp.Source url, err := url.Parse(config.Source) diff --git a/cmd/ocsp-updater/main.go b/cmd/ocsp-updater/main.go index 5feb76085..af86fcdcb 100644 --- a/cmd/ocsp-updater/main.go +++ b/cmd/ocsp-updater/main.go @@ -205,6 +205,7 @@ func main() { auditlogger, err := blog.Dial(c.Syslog.Network, c.Syslog.Server, c.Syslog.Tag, stats) cmd.FailOnError(err, "Could not connect to Syslog") + auditlogger.Info(app.VersionString()) // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 defer auditlogger.AuditPanic() @@ -229,8 +230,6 @@ func main() { } }() - auditlogger.Info(app.VersionString()) - updater := &OCSPUpdater{ cac: cac, dbMap: dbMap, From 4696b706470085f98712015b76b3a71775770a40 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Wed, 30 Sep 2015 16:40:16 -0700 Subject: [PATCH 7/7] Add missing ct publisher errors --- publisher/publisher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publisher/publisher.go b/publisher/publisher.go index 8e5584dd8..8de4070a9 100644 --- a/publisher/publisher.go +++ b/publisher/publisher.go @@ -44,11 +44,11 @@ func (logDesc *LogDescription) UnmarshalJSON(data []byte) error { // Load Key pkBytes, err := base64.StdEncoding.DecodeString(rawLogDesc.PublicKey) if err != nil { - return fmt.Errorf("") + return fmt.Errorf("Failed to decode base64 log public key") } pk, err := x509.ParsePKIXPublicKey(pkBytes) if err != nil { - return fmt.Errorf("") + return fmt.Errorf("Failed to parse log public key") } ecdsaKey, ok := pk.(*ecdsa.PublicKey) if !ok {