letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into delete_old_challenges

This commit is contained in:
Jeff Hodges 2015-12-10 23:04:00 -08:00
parent c9010744b4 4f975b2c9d
commit 8300b06ad6
14 changed files with 331 additions and 425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Godeps/Godeps.json generated
View File

@ -109,8 +109,8 @@
},
{
"ImportPath": "github.com/letsencrypt/go-safe-browsing-api",
"Comment": "2.0.0-2-g814cea4",
"Rev": "814cea4d6d3063540dc15c3d93754eff4eaa756b"
"Comment": "2.0.0-9-gf1b4fa4",
"Rev": "f1b4fa409d9efcee2ceee9047549413861603033"
},
{
"ImportPath": "github.com/letsencrypt/net/publicsuffix",

18
Godeps/_workspace/src/github.com/letsencrypt/go-safe-browsing-api/safebrowsinglist.go generated vendored
View File

@ -50,7 +50,12 @@ type SafeBrowsingList struct {
FullHashRequested *HatTrie
FullHashes *HatTrie
Logger logger
Logger logger
// fsLock is wrapped around the filesystem modifications and a call to
// updateLock to prevent more than one set of fs modifications happening at
// once.
fsLock *sync.Mutex
// updateLock prevents more than one pointer swap
updateLock *sync.RWMutex
}
@ -64,6 +69,7 @@ func newSafeBrowsingList(name string, filename string) (sbl *SafeBrowsingList) {
FullHashes: NewTrie(),
DeleteChunks: make(map[ChunkData_ChunkType]map[ChunkNum]bool),
Logger: &DefaultLogger{},
fsLock: new(sync.Mutex),
updateLock: new(sync.RWMutex),
}
sbl.DeleteChunks[CHUNK_TYPE_ADD] = make(map[ChunkNum]bool)
@ -113,8 +119,8 @@ func (sbl *SafeBrowsingList) load(newChunks []*ChunkData) (err error) {
// defer debug.FreeOSMemory()
sbl.Logger.Info("Reloading %s", sbl.Name)
sbl.updateLock.Lock()
defer sbl.updateLock.Unlock()
sbl.fsLock.Lock()
defer sbl.fsLock.Unlock()
// get the input stream
f, err := os.Open(sbl.FileName)
@ -266,7 +272,6 @@ func (sbl *SafeBrowsingList) load(newChunks []*ChunkData) (err error) {
}
func (sbl *SafeBrowsingList) updateLookupMap(chunk *ChunkData) {
hashlen := 0
hasheslen := len(chunk.Hashes)
@ -279,7 +284,8 @@ func (sbl *SafeBrowsingList) updateLookupMap(chunk *ChunkData) {
for i := 0; (i + hashlen) <= hasheslen; i += hashlen {
hash := chunk.Hashes[i:(i + hashlen)]
// We may have to make this more fine grained
sbl.updateLock.Lock()
switch hashlen {
case PREFIX_4B_SZ:
// we are a hash-prefix
@ -311,5 +317,7 @@ func (sbl *SafeBrowsingList) updateLookupMap(chunk *ChunkData) {
sbl.FullHashes.Delete(lookupHash)
}
}
sbl.updateLock.Unlock()
}
}

7
cmd/activity-monitor/main.go
View File

@ -10,6 +10,8 @@ package main
// broker to look for anomalies.
import (
"expvar"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/streadway/amqp"
@ -26,11 +28,14 @@ func main() {
go cmd.DebugServer(c.ActivityMonitor.DebugAddr)
amqpConf := c.ActivityMonitor.AMQP
server, err := rpc.NewAmqpRPCServer(amqpConf, 0, stats)
server, err := rpc.NewMonitorServer(amqpConf, 0, stats)
cmd.FailOnError(err, "Could not connect to AMQP")
ae := analysisengine.NewLoggingAnalysisEngine()
messages := expvar.NewInt("messages")
server.HandleDeliveries(rpc.DeliveryHandler(func(d amqp.Delivery) {
messages.Add(1)
ae.ProcessMessage(d)
}))

25
cmd/boulder-publisher/main.go
View File

@ -6,9 +6,13 @@
package main
import (
"os"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
ct "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/google/certificate-transparency/go"
"github.com/letsencrypt/boulder/cmd"
"github.com/letsencrypt/boulder/core"
blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/publisher"
"github.com/letsencrypt/boulder/rpc"
@ -19,8 +23,25 @@ const clientName = "Publisher"
func main() {
app := cmd.NewAppShell("boulder-publisher", "Submits issued certificates to CT logs")
app.Action = func(c cmd.Config, stats statsd.Statter, auditlogger *blog.AuditLogger) {
pubi, err := publisher.NewPublisherImpl(c.Common.CT)
cmd.FailOnError(err, "Could not setup Publisher")
logs := make([]*publisher.Log, len(c.Common.CT.Logs))
var err error
for i, ld := range c.Common.CT.Logs {
logs[i], err = publisher.NewLog(ld.URI, ld.Key)
cmd.FailOnError(err, "Unable to parse CT log description")
}
if c.Common.CT.IntermediateBundleFilename == "" {
auditlogger.Err("No CT submission bundle provided")
os.Exit(1)
}
pemBundle, err := core.LoadCertBundle(c.Common.CT.IntermediateBundleFilename)
cmd.FailOnError(err, "Failed to load CT submission bundle")
bundle := []ct.ASN1Cert{}
for _, cert := range pemBundle {
bundle = append(bundle, ct.ASN1Cert(cert.Raw))
}
pubi := publisher.NewPublisherImpl(bundle, logs)
go cmd.DebugServer(c.Publisher.DebugAddr)
go cmd.ProfileCmd("Publisher", stats)

13
cmd/config.go
View File

@ -16,7 +16,6 @@ import (
cfsslConfig "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/config"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/crypto/pkcs11key"
"github.com/letsencrypt/boulder/core"
"github.com/letsencrypt/boulder/publisher"
"github.com/letsencrypt/boulder/va"
)
@ -163,7 +162,10 @@ type Config struct {
DNSTimeout string
DNSAllowLoopbackAddresses bool
CT publisher.CTConfig
CT struct {
Logs []LogDescription
IntermediateBundleFilename string
}
}
CertChecker struct {
@ -409,3 +411,10 @@ func (d *ConfigDuration) UnmarshalYAML(unmarshal func(interface{}) error) error
d.Duration = dur
return nil
}
// LogDescription contains the information needed to submit certificates
// to a CT log and verify returned receipts
type LogDescription struct {
URI string
Key string
}

84
core/objects.go
View File

@ -8,12 +8,9 @@ package core
import (
"crypto/subtle"
"crypto/x509"
"encoding/asn1"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"math/big"
"net"
"strings"
"time"
@ -592,8 +589,8 @@ type OCSPSigningRequest struct {
RevokedAt time.Time
}
// SignedCertificateTimestamp represents objects used by Certificate Transparency
// to demonstrate that a certificate was submitted to a CT log. See RFC 6962.
// SignedCertificateTimestamp is the internal representation of ct.SignedCertificateTimestamp
// that is used to maintain backwards compatibility with our old CT implementation.
type SignedCertificateTimestamp struct {
ID int `db:"id"`
// The version of the protocol to which the SCT conforms
@ -614,83 +611,6 @@ type SignedCertificateTimestamp struct {
LockCol int64
}
// RPCSignedCertificateTimestamp is a wrapper around SignedCertificateTimestamp
// so that it can be passed through the RPC layer properly. Without this wrapper
// the UnmarshalJSON method below will be used when marshaling/unmarshaling the
// object, which is not what we want as it is not symmetrical (as it is intended
// to unmarshal a rawSignedCertificateTimestamp into a SignedCertificateTimestamp)
type RPCSignedCertificateTimestamp SignedCertificateTimestamp
type rawSignedCertificateTimestamp struct {
Version uint8 `json:"sct_version"`
LogID string `json:"id"`
Timestamp uint64 `json:"timestamp"`
Signature string `json:"signature"`
Extensions string `json:"extensions"`
}
// UnmarshalJSON parses the add-chain response from a CT log. It fills all of
// the fields in the SignedCertificateTimestamp struct except for ID and
// CertificateSerial, which are used for local recordkeeping in the Boulder DB.
func (sct *SignedCertificateTimestamp) UnmarshalJSON(data []byte) error {
var err error
var rawSCT rawSignedCertificateTimestamp
if err = json.Unmarshal(data, &rawSCT); err != nil {
return fmt.Errorf("Failed to unmarshal SCT receipt, %s", err)
}
sct.LogID = rawSCT.LogID
if err != nil {
return fmt.Errorf("Failed to decode log ID, %s", err)
}
sct.Signature, err = base64.StdEncoding.DecodeString(rawSCT.Signature)
if err != nil {
return fmt.Errorf("Failed to decode SCT signature, %s", err)
}
sct.Extensions, err = base64.StdEncoding.DecodeString(rawSCT.Extensions)
if err != nil {
return fmt.Errorf("Failed to decode SCT extensions, %s", err)
}
sct.SCTVersion = rawSCT.Version
sct.Timestamp = rawSCT.Timestamp
return nil
}
const (
sctHashSHA256 = 4
sctSigECDSA = 3
)
// CheckSignature validates that the returned SCT signature is a valid SHA256 +
// ECDSA signature but does not verify that a specific public key signed it.
func (sct *SignedCertificateTimestamp) CheckSignature() error {
if len(sct.Signature) < 4 {
return errors.New("SCT signature is truncated")
}
// Since all of the known logs currently only use SHA256 hashes and ECDSA
// keys, only allow those
if sct.Signature[0] != sctHashSHA256 {
return fmt.Errorf("Unsupported SCT hash function [%d]", sct.Signature[0])
}
if sct.Signature[1] != sctSigECDSA {
return fmt.Errorf("Unsupported SCT signature algorithm [%d]", sct.Signature[1])
}
var ecdsaSig struct {
R, S *big.Int
}
// Ignore the two length bytes and attempt to unmarshal the signature directly
signatureBytes := sct.Signature[4:]
signatureBytes, err := asn1.Unmarshal(signatureBytes, &ecdsaSig)
if err != nil {
return fmt.Errorf("Failed to parse SCT signature, %s", err)
}
if len(signatureBytes) > 0 {
return fmt.Errorf("Trailing garbage after signature")
}
return nil
}
// RevocationCode is used to specify a certificate revocation reason
type RevocationCode int

279
publisher/publisher.go
View File

@ -6,242 +6,120 @@
package publisher
import (
"bytes"
"crypto/ecdsa"
"crypto/sha256"
"crypto/x509"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
ct "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/google/certificate-transparency/go"
ctClient "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/google/certificate-transparency/go/client"
"github.com/letsencrypt/boulder/core"
blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/sa"
)
// LogDescription tells you how to connect to a log and verify its statements.
type LogDescription struct {
ID string
URI string
PublicKey *ecdsa.PublicKey
// Log contains the CT client and signature verifier for a particular CT log
type Log struct {
client *ctClient.LogClient
verifier *ct.SignatureVerifier
}
type rawLogDescription struct {
URI string `json:"uri"`
PublicKey string `json:"key"`
}
// UnmarshalJSON parses a simple JSON format for log descriptions. Both the
// URI and the public key are expected to be strings. The public key is a
// base64-encoded PKIX public key structure.
func (logDesc *LogDescription) UnmarshalJSON(data []byte) error {
var rawLogDesc rawLogDescription
if err := json.Unmarshal(data, &rawLogDesc); err != nil {
return fmt.Errorf("Failed to unmarshal log description, %s", err)
// NewLog returns a initialized Log struct
func NewLog(uri, b64PK string) (*Log, error) {
if strings.HasSuffix(uri, "/") {
uri = uri[0 : len(uri)-2]
}
logDesc.URI = rawLogDesc.URI
// Load Key
pkBytes, err := base64.StdEncoding.DecodeString(rawLogDesc.PublicKey)
client := ctClient.New(uri)
pkBytes, err := base64.StdEncoding.DecodeString(b64PK)
if err != nil {
return fmt.Errorf("Failed to decode base64 log public key")
return nil, fmt.Errorf("Failed to decode base64 log public key")
}
pk, err := x509.ParsePKIXPublicKey(pkBytes)
if err != nil {
return fmt.Errorf("Failed to parse log public key")
}
ecdsaKey, ok := pk.(*ecdsa.PublicKey)
if !ok {
return fmt.Errorf("Failed to unmarshal log description for %s, unsupported public key type", logDesc.URI)
}
logDesc.PublicKey = ecdsaKey
// Generate key hash for log ID
pkHash := sha256.Sum256(pkBytes)
logDesc.ID = base64.StdEncoding.EncodeToString(pkHash[:])
if len(logDesc.ID) != 44 {
return fmt.Errorf("Invalid log ID length [%d]", len(logDesc.ID))
return nil, fmt.Errorf("Failed to parse log public key")
}
return nil
}
verifier, err := ct.NewSignatureVerifier(pk)
if err != nil {
return nil, err
}
// CTConfig defines the JSON configuration file schema
type CTConfig struct {
Logs []LogDescription `json:"logs"`
SubmissionRetries int `json:"submissionRetries"`
SubmissionBackoffString string `json:"submissionBackoff"`
IntermediateBundleFilename string `json:"intermediateBundleFilename"`
return &Log{client, verifier}, nil
}
type ctSubmissionRequest struct {
Chain []string `json:"chain"`
}
const (
sctVersion = 0
sctSigType = 0
sctX509EntryType = 0
)
// PublisherImpl defines a Publisher
type PublisherImpl struct {
log *blog.AuditLogger
client *http.Client
submissionBackoff time.Duration
submissionRetries int
issuerBundle []string
ctLogs []LogDescription
log *blog.AuditLogger
client *http.Client
issuerBundle []ct.ASN1Cert
ctLogs []*Log
SA core.StorageAuthority
}
// NewPublisherImpl creates a Publisher that will submit certificates
// to any CT logs configured in CTConfig
func NewPublisherImpl(ctConfig CTConfig) (pub PublisherImpl, err error) {
func NewPublisherImpl(bundle []ct.ASN1Cert, logs []*Log) (pub PublisherImpl) {
logger := blog.GetAuditLogger()
logger.Notice("Publisher Authority Starting")
if ctConfig.IntermediateBundleFilename == "" {
err = fmt.Errorf("No CT submission bundle provided")
return
}
bundle, err := core.LoadCertBundle(ctConfig.IntermediateBundleFilename)
if err != nil {
return
}
for _, cert := range bundle {
pub.issuerBundle = append(pub.issuerBundle, base64.StdEncoding.EncodeToString(cert.Raw))
}
ctBackoff, err := time.ParseDuration(ctConfig.SubmissionBackoffString)
if err != nil {
return
}
for _, log := range ctConfig.Logs {
if !strings.HasPrefix(log.URI, "https://") && !strings.HasPrefix(log.URI, "http://") {
err = fmt.Errorf("Log URI [%s] is not absolute", log.URI)
return
}
}
pub.issuerBundle = bundle
pub.log = logger
pub.client = &http.Client{}
pub.submissionBackoff = ctBackoff
pub.submissionRetries = ctConfig.SubmissionRetries
pub.ctLogs = ctConfig.Logs
pub.ctLogs = logs
return
}
func (pub *PublisherImpl) submitToCTLog(serial string, jsonSubmission []byte, log LogDescription) error {
done := false
var sct core.SignedCertificateTimestamp
backoff := pub.submissionBackoff
var retries int
for retries = 0; retries <= pub.submissionRetries; retries++ {
if retries > 0 {
time.Sleep(backoff)
}
resp, err := postJSON(pub.client, fmt.Sprintf("%s%s", log.URI, "/ct/v1/add-chain"), jsonSubmission, &sct)
if err != nil {
// Retry the request, log the error
pub.log.Warning(fmt.Sprintf("Error POSTing JSON to CT log submission endpoint [%s]: %s", log.URI, err))
backoff = pub.submissionBackoff
continue
} else if resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode == http.StatusServiceUnavailable {
// Retry the request after either the configured backoff period or the period
// specified by the Retry-After header
backoff = pub.submissionBackoff
if seconds, err := strconv.Atoi(resp.Header.Get("Retry-After")); err == nil {
backoff = time.Second * time.Duration(seconds)
}
continue
} else if resp.StatusCode != http.StatusOK {
// If we hit an otherwise unexpected status code break the loop and return
// an error immediately
return fmt.Errorf("Unexpected status code returned from CT log: %d", resp.StatusCode)
}
done = true
break
}
if !done {
err := fmt.Errorf(
"Unable to submit certificate to CT log [Serial: %s, Log URI: %s, Retries: %d]",
serial,
log.URI,
retries,
)
return err
}
if err := sct.CheckSignature(); err != nil {
return err
}
pub.log.Debug(fmt.Sprintf(
"Submitted certificate to CT log [Serial: %s, Log URI: %s, Retries: %d, Signature: %x]",
serial,
log.URI,
retries, sct.Signature,
))
// Set certificate serial and add SCT to DB
sct.CertificateSerial = serial
err := pub.SA.AddSCTReceipt(sct)
if err != nil {
if _, ok := err.(sa.ErrDuplicateReceipt); ok {
pub.log.Warning(fmt.Sprintf(
"SCT receipt for [Serial: %s, Log URI: %s] already exists in database",
serial,
log.URI,
))
return nil
} else if err != nil {
err = fmt.Errorf(
"Error adding SCT receipt for [%s to %s]: %s",
sct.CertificateSerial,
log.URI,
err,
)
return err
}
}
pub.log.Info(fmt.Sprintf(
"Stored SCT receipt from CT log submission [Serial: %s, Log URI: %s]",
serial,
log.URI,
))
return nil
}
// SubmitToCT will submit the certificate represented by certDER to any CT
// logs configured in pub.CT.Logs
func (pub *PublisherImpl) SubmitToCT(der []byte) error {
cert, err := x509.ParseCertificate(der)
if err != nil {
pub.log.Err(fmt.Sprintf("Unable to parse certificate, %s", err))
return err
}
submission := ctSubmissionRequest{Chain: []string{base64.StdEncoding.EncodeToString(cert.Raw)}}
// Add all intermediate certificates needed for submission
submission.Chain = append(submission.Chain, pub.issuerBundle...)
jsonSubmission, err := json.Marshal(submission)
if err != nil {
pub.log.Err(fmt.Sprintf("Unable to marshal CT submission, %s", err))
pub.log.Audit(fmt.Sprintf("Failed to parse certificate: %s", err))
return err
}
chain := append([]ct.ASN1Cert{der}, pub.issuerBundle...)
for _, ctLog := range pub.ctLogs {
err = pub.submitToCTLog(core.SerialToString(cert.SerialNumber), jsonSubmission, ctLog)
sct, err := ctLog.client.AddChain(chain)
if err != nil {
// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
pub.log.AuditErr(err)
pub.log.Audit(fmt.Sprintf("Failed to submit certificate to CT log: %s", err))
continue
}
err = ctLog.verifier.VerifySCTSignature(*sct, ct.LogEntry{
Leaf: ct.MerkleTreeLeaf{
LeafType: ct.TimestampedEntryLeafType,
TimestampedEntry: ct.TimestampedEntry{
X509Entry: ct.ASN1Cert(der),
EntryType: ct.X509LogEntryType,
},
},
})
if err != nil {
// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
pub.log.Audit(fmt.Sprintf("Failed to verify SCT receipt: %s", err))
continue
}
internalSCT, err := sctToInternal(sct, core.SerialToString(cert.SerialNumber))
if err != nil {
// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
pub.log.Audit(fmt.Sprintf("Failed to convert SCT receipt: %s", err))
continue
}
err = pub.SA.AddSCTReceipt(internalSCT)
if err != nil {
// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
pub.log.Audit(fmt.Sprintf("Failed to store SCT receipt in database: %s", err))
continue
}
}
@ -249,30 +127,17 @@ func (pub *PublisherImpl) SubmitToCT(der []byte) error {
return nil
}
func postJSON(client *http.Client, uri string, data []byte, respObj interface{}) (*http.Response, error) {
req, err := http.NewRequest("POST", uri, bytes.NewBuffer(data))
func sctToInternal(sct *ct.SignedCertificateTimestamp, serial string) (core.SignedCertificateTimestamp, error) {
sig, err := ct.MarshalDigitallySigned(sct.Signature)
if err != nil {
return nil, fmt.Errorf("Creating request failed, %s", err)
return core.SignedCertificateTimestamp{}, err
}
req.Header.Set("Keep-Alive", "timeout=15, max=100")
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("Request failed, %s", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("Failed to read response body, %s", err)
}
err = json.Unmarshal(body, respObj)
if err != nil {
return nil, fmt.Errorf("Failed to unmarshal SCT receipt, %s", err)
}
}
return resp, nil
return core.SignedCertificateTimestamp{
CertificateSerial: serial,
SCTVersion: uint8(sct.SCTVersion),
LogID: sct.LogID.Base64String(),
Timestamp: sct.Timestamp,
Extensions: sct.Extensions,
Signature: sig,
}, nil
}

219
publisher/publisher_test.go
View File

@ -6,11 +6,17 @@
package publisher
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/x509"
"encoding/asn1"
"encoding/base64"
"encoding/json"
"encoding/pem"
"fmt"
"math/big"
"net"
"net/http"
"net/http/httptest"
@ -19,9 +25,10 @@ import (
"testing"
"time"
ct "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/google/certificate-transparency/go"
ctClient "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/google/certificate-transparency/go/client"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/core"
"github.com/letsencrypt/boulder/mocks"
"github.com/letsencrypt/boulder/test"
)
@ -127,7 +134,54 @@ func getPort(hs *httptest.Server) (int, error) {
return int(port), nil
}
func logSrv() *httptest.Server {
func createSignedSCT(leaf []byte, k *ecdsa.PrivateKey) string {
rawKey, _ := x509.MarshalPKIXPublicKey(&k.PublicKey)
pkHash := sha256.Sum256(rawKey)
sct := ct.SignedCertificateTimestamp{
SCTVersion: ct.V1,
LogID: pkHash,
Timestamp: 1337,
}
serialized, _ := ct.SerializeSCTSignatureInput(sct, ct.LogEntry{
Leaf: ct.MerkleTreeLeaf{
LeafType: ct.TimestampedEntryLeafType,
TimestampedEntry: ct.TimestampedEntry{
X509Entry: ct.ASN1Cert(leaf),
EntryType: ct.X509LogEntryType,
},
},
})
hashed := sha256.Sum256(serialized)
var ecdsaSig struct {
R, S *big.Int
}
ecdsaSig.R, ecdsaSig.S, _ = ecdsa.Sign(rand.Reader, k, hashed[:])
sig, _ := asn1.Marshal(ecdsaSig)
ds := ct.DigitallySigned{
HashAlgorithm: ct.SHA256,
SignatureAlgorithm: ct.ECDSA,
Signature: sig,
}
var jsonSCTObj struct {
SCTVersion ct.Version `json:"sct_version"`
ID string `json:"id"`
Timestamp uint64 `json:"timestamp"`
Extensions string `json:"extensions"`
Signature string `json:"signature"`
}
jsonSCTObj.SCTVersion = ct.V1
jsonSCTObj.ID = base64.StdEncoding.EncodeToString(pkHash[:])
jsonSCTObj.Timestamp = 1337
jsonSCTObj.Signature, _ = ds.Base64String()
jsonSCT, _ := json.Marshal(jsonSCTObj)
return string(jsonSCT)
}
func logSrv(leaf []byte, k *ecdsa.PrivateKey) *httptest.Server {
sct := createSignedSCT(leaf, k)
m := http.NewServeMux()
m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
decoder := json.NewDecoder(r.Body)
@ -138,7 +192,7 @@ func logSrv() *httptest.Server {
}
// Submissions should always contain at least one cert
if len(jsonReq.Chain) >= 1 {
fmt.Fprint(w, `{"signature":"BAMASDBGAiEAknaySJVdB3FqG9bUKHgyu7V9AdEabpTc71BELUp6/iECIQDObrkwlQq6Azfj5XOA5E12G/qy/WuRn97z7qMSXXc82Q=="}`)
fmt.Fprint(w, sct)
}
})
@ -147,12 +201,24 @@ func logSrv() *httptest.Server {
return server
}
func retryableLogSrv(retries int, after *int) *httptest.Server {
func errorLogSrv() *httptest.Server {
m := http.NewServeMux()
m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
})
server := httptest.NewUnstartedServer(m)
server.Start()
return server
}
func retryableLogSrv(leaf []byte, k *ecdsa.PrivateKey, retries int, after *int) *httptest.Server {
hits := 0
sct := createSignedSCT(leaf, k)
m := http.NewServeMux()
m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if hits >= retries {
fmt.Fprint(w, `{"signature":"BAMASDBGAiEAknaySJVdB3FqG9bUKHgyu7V9AdEabpTc71BELUp6/iECIQDObrkwlQq6Azfj5XOA5E12G/qy/WuRn97z7qMSXXc82Q=="}`)
fmt.Fprint(w, sct)
} else {
hits++
if after != nil {
@ -167,7 +233,7 @@ func retryableLogSrv(retries int, after *int) *httptest.Server {
return server
}
func emptyLogSrv() *httptest.Server {
func badLogSrv() *httptest.Server {
m := http.NewServeMux()
m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
decoder := json.NewDecoder(r.Body)
@ -178,7 +244,7 @@ func emptyLogSrv() *httptest.Server {
}
// Submissions should always contain at least one cert
if len(jsonReq.Chain) >= 1 {
fmt.Fprint(w, `{"signature":""}`)
fmt.Fprint(w, `{"signature":"BAMASDBGAiEAknaySJVdB3FqG9bUKHgyu7V9AdEabpTc71BELUp6/iECIQDObrkwlQq6Azfj5XOA5E12G/qy/WuRn97z7qMSXXc82Q=="}`)
}
})
@ -187,155 +253,134 @@ func emptyLogSrv() *httptest.Server {
return server
}
func setup(t *testing.T, port, retries int) (PublisherImpl, *x509.Certificate) {
func setup(t *testing.T) (*PublisherImpl, *x509.Certificate, *ecdsa.PrivateKey) {
intermediatePEM, _ := pem.Decode([]byte(testIntermediate))
pub, err := NewPublisherImpl(CTConfig{
Logs: []LogDescription{LogDescription{URI: fmt.Sprintf("http://localhost:%d", port)}},
SubmissionBackoffString: "0s",
IntermediateBundleFilename: issuerPath,
SubmissionRetries: retries,
})
test.AssertNotError(t, err, "Couldn't create new Publisher")
pub.issuerBundle = append(pub.issuerBundle, base64.StdEncoding.EncodeToString(intermediatePEM.Bytes))
pub := NewPublisherImpl(nil, nil)
pub.issuerBundle = append(pub.issuerBundle, ct.ASN1Cert(intermediatePEM.Bytes))
pub.SA = mocks.NewStorageAuthority(clock.NewFake())
leafPEM, _ := pem.Decode([]byte(testLeaf))
leaf, err := x509.ParseCertificate(leafPEM.Bytes)
test.AssertNotError(t, err, "Couldn't parse leafPEM.Bytes")
return pub, leaf
k, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
test.AssertNotError(t, err, "Couldn't generate test key")
return &pub, leaf, k
}
func TestNewPublisherImpl(t *testing.T) {
// Allowed
ctConf := CTConfig{SubmissionBackoffString: "0s", IntermediateBundleFilename: issuerPath}
_, err := NewPublisherImpl(ctConf)
test.AssertNotError(t, err, "Couldn't create new Publisher")
func addLog(t *testing.T, pub *PublisherImpl, port int, pubKey *ecdsa.PublicKey) {
verifier, err := ct.NewSignatureVerifier(pubKey)
test.AssertNotError(t, err, "Couldn't create signature verifier")
ctConf = CTConfig{Logs: []LogDescription{LogDescription{URI: "http://localhost"}}, SubmissionBackoffString: "0s", IntermediateBundleFilename: issuerPath}
_, err = NewPublisherImpl(ctConf)
test.AssertNotError(t, err, "Couldn't create new Publisher")
pub.ctLogs = append(pub.ctLogs, &Log{
client: ctClient.New(fmt.Sprintf("http://localhost:%d", port)),
verifier: verifier,
})
}
func TestCheckSignature(t *testing.T) {
// Based on a submission to the aviator log
goodSigBytes, err := base64.StdEncoding.DecodeString("BAMASDBGAiEAknaySJVdB3FqG9bUKHgyu7V9AdEabpTc71BELUp6/iECIQDObrkwlQq6Azfj5XOA5E12G/qy/WuRn97z7qMSXXc82Q==")
test.AssertNotError(t, err, "Couldn't decode signature")
func TestBasicSuccessful(t *testing.T) {
pub, leaf, k := setup(t)
testReceipt := core.SignedCertificateTimestamp{
Signature: goodSigBytes,
}
// Good signature
err = testReceipt.CheckSignature()
test.AssertNotError(t, err, "Valid signature check failed")
// Invalid signature (too short, trailing garbage)
testReceipt.Signature = goodSigBytes[1:]
err = testReceipt.CheckSignature()
test.AssertError(t, err, "Invalid signature check failed")
testReceipt.Signature = append(goodSigBytes, []byte{0, 0, 1}...)
err = testReceipt.CheckSignature()
test.AssertError(t, err, "Invalid signature check failed")
}
func TestSubmitToCT(t *testing.T) {
server := logSrv()
server := logSrv(leaf.Raw, k)
defer server.Close()
port, err := getPort(server)
test.AssertNotError(t, err, "Failed to get test server port")
pub, leaf := setup(t, port, 0)
addLog(t, pub, port, &k.PublicKey)
log.Clear()
err = pub.SubmitToCT(leaf.Raw)
test.AssertNotError(t, err, "Certificate submission failed")
test.AssertEquals(t, len(log.GetAllMatching("Failed to.*")), 0)
// No Intermediate
pub.issuerBundle = []string{}
pub.issuerBundle = []ct.ASN1Cert{}
log.Clear()
err = pub.SubmitToCT(leaf.Raw)
test.AssertNotError(t, err, "Certificate submission failed")
test.AssertEquals(t, len(log.GetAllMatching("Failed to.*")), 0)
}
func TestGoodRetry(t *testing.T) {
server := retryableLogSrv(1, nil)
pub, leaf, k := setup(t)
server := retryableLogSrv(leaf.Raw, k, 1, nil)
defer server.Close()
port, err := getPort(server)
test.AssertNotError(t, err, "Failed to get test server port")
addLog(t, pub, port, &k.PublicKey)
pub, leaf := setup(t, port, 1)
log.Clear()
err = pub.SubmitToCT(leaf.Raw)
test.AssertNotError(t, err, "Certificate submission failed")
test.AssertEquals(t, len(log.GetAllMatching("Failed to.*")), 0)
}
func TestUnexpectedError(t *testing.T) {
pub, leaf, k := setup(t)
srv := errorLogSrv()
defer srv.Close()
port, err := getPort(srv)
test.AssertNotError(t, err, "Failed to get test server port")
addLog(t, pub, port, &k.PublicKey)
log.Clear()
err = pub.SubmitToCT(leaf.Raw)
test.AssertNotError(t, err, "Certificate submission failed")
}
func TestFatalRetry(t *testing.T) {
server := retryableLogSrv(1, nil)
defer server.Close()
port, err := getPort(server)
test.AssertNotError(t, err, "Failed to get test server port")
pub, leaf := setup(t, port, 0)
log.Clear()
err = pub.SubmitToCT(leaf.Raw)
test.AssertEquals(t, len(log.GetAllMatching("Unable to submit certificate to CT log.*")), 1)
}
func TestUnexpectedError(t *testing.T) {
pub, leaf := setup(t, 0, 0)
log.Clear()
_ = pub.SubmitToCT(leaf.Raw)
test.AssertEquals(t, len(log.GetAllMatching("Unable to submit certificate to CT log.*")), 1)
}
func TestRetryAfter(t *testing.T) {
pub, leaf, k := setup(t)
retryAfter := 2
server := retryableLogSrv(2, &retryAfter)
server := retryableLogSrv(leaf.Raw, k, 2, &retryAfter)
defer server.Close()
port, err := getPort(server)
test.AssertNotError(t, err, "Failed to get test server port")
pub, leaf := setup(t, port, 2)
addLog(t, pub, port, &k.PublicKey)
log.Clear()
startedWaiting := time.Now()
err = pub.SubmitToCT(leaf.Raw)
test.AssertNotError(t, err, "Certificate submission failed")
test.Assert(t, time.Since(startedWaiting) >= time.Duration(retryAfter*2)*time.Second, fmt.Sprintf("Submitter retried submission too fast: %s", time.Since(startedWaiting)))
test.AssertEquals(t, len(log.GetAllMatching("Failed to.*")), 0)
test.Assert(t, time.Since(startedWaiting) < time.Duration(retryAfter*2)*time.Second, fmt.Sprintf("Submitter retried submission too fast: %s", time.Since(startedWaiting)))
}
func TestMultiLog(t *testing.T) {
srvA := logSrv()
pub, leaf, k := setup(t)
srvA := logSrv(leaf.Raw, k)
defer srvA.Close()
srvB := logSrv()
srvB := logSrv(leaf.Raw, k)
defer srvB.Close()
portA, err := getPort(srvA)
test.AssertNotError(t, err, "Failed to get test server port")
portB, err := getPort(srvB)
test.AssertNotError(t, err, "Failed to get test server port")
pub, leaf := setup(t, portA, 0)
pub.ctLogs = append(pub.ctLogs, LogDescription{URI: fmt.Sprintf("http://localhost:%d", portB)})
addLog(t, pub, portA, &k.PublicKey)
addLog(t, pub, portB, &k.PublicKey)
log.Clear()
err = pub.SubmitToCT(leaf.Raw)
test.AssertNotError(t, err, "Certificate submission failed")
test.AssertEquals(t, len(log.GetAllMatching("Failed to.*")), 0)
}
func TestBadServer(t *testing.T) {
srv := emptyLogSrv()
pub, leaf, k := setup(t)
srv := badLogSrv()
defer srv.Close()
port, err := getPort(srv)
test.AssertNotError(t, err, "Failed to get test server port")
pub, leaf := setup(t, port, 0)
addLog(t, pub, port, &k.PublicKey)
log.Clear()
err = pub.SubmitToCT(leaf.Raw)
test.AssertEquals(t, len(log.GetAllMatching("SCT signature is truncated")), 1)
test.AssertNotError(t, err, "Certificate submission failed")
test.AssertEquals(t, len(log.GetAllMatching("Failed to verify SCT receipt")), 1)
}

18
rpc/amqp-rpc.go
View File

@ -59,7 +59,7 @@ const (
)
// A simplified way to declare and subscribe to an AMQP queue
func amqpSubscribe(ch amqpChannel, name string) (<-chan amqp.Delivery, error) {
func amqpSubscribe(ch amqpChannel, name, routingKey string) (<-chan amqp.Delivery, error) {
var err error
_, err = ch.QueueDeclare(
@ -73,8 +73,6 @@ func amqpSubscribe(ch amqpChannel, name string) (<-chan amqp.Delivery, error) {
return nil, fmt.Errorf("could not declare queue: %s", err)
}
routingKey := name
err = ch.QueueBind(
name,
routingKey,
@ -131,6 +129,20 @@ type AmqpRPCServer struct {
clk clock.Clock
}
const wildcardRoutingKey = "#"
// NewMonitorServer creates an AmqpRPCServer that binds its queue to the
// wildcard routing key instead of the default of binding to the queue name.
// This allows Activity Monitor to observe all messages sent to the exchange.
func NewMonitorServer(amqpConf *cmd.AMQPConfig, maxConcurrentRPCServerRequests int64, stats statsd.Statter) (*AmqpRPCServer, error) {
server, err := NewAmqpRPCServer(amqpConf, maxConcurrentRPCServerRequests, stats)
if err != nil {
return nil, err
}
server.connection.routingKey = wildcardRoutingKey
return server, nil
}
// NewAmqpRPCServer creates a new RPC server for the given queue and will begin
// consuming requests from the queue. To start the server you must call Start().
func NewAmqpRPCServer(amqpConf *cmd.AMQPConfig, maxConcurrentRPCServerRequests int64, stats statsd.Statter) (*AmqpRPCServer, error) {

14
rpc/connection.go
View File

@ -20,6 +20,7 @@ func newAMQPConnector(
) *amqpConnector {
return &amqpConnector{
queueName: queueName,
routingKey: queueName,
chMaker: defaultChannelMaker{},
clk: clock.Default(),
retryTimeoutBase: retryTimeoutBase,
@ -42,10 +43,13 @@ func (d defaultChannelMaker) makeChannel(conf *cmd.AMQPConfig) (amqpChannel, err
// queue, plus appropriate locking for its members. It provides reconnect logic,
// and allows publishing via the channel onto an arbitrary queue.
type amqpConnector struct {
mu sync.RWMutex
chMaker channelMaker
channel amqpChannel
queueName string
mu sync.RWMutex
chMaker channelMaker
channel amqpChannel
queueName string
// Usually this is the same as queueName, except for Activity Monitor, which
// sets it to "#".
routingKey string
closeChan chan *amqp.Error
msgs <-chan amqp.Delivery
retryTimeoutBase time.Duration
@ -73,7 +77,7 @@ func (ac *amqpConnector) connect(config *cmd.AMQPConfig) error {
if err != nil {
return fmt.Errorf("channel connect failed for %s: %s", ac.queueName, err)
}
msgs, err := amqpSubscribe(channel, ac.queueName)
msgs, err := amqpSubscribe(channel, ac.queueName, ac.routingKey)
if err != nil {
return fmt.Errorf("queue subscribe failed for %s: %s", ac.queueName, err)
}

1
rpc/connection_test.go
View File

@ -30,6 +30,7 @@ func setup(t *testing.T) (*amqpConnector, *MockamqpChannel, func()) {
channel: mockChannel,
},
queueName: "fooqueue",
routingKey: "fooqueue",
retryTimeoutBase: time.Second,
clk: clock.NewFake(),
}

4
rpc/rpc-wrappers.go
View File

@ -1136,7 +1136,7 @@ func NewStorageAuthorityServer(rpc Server, impl core.StorageAuthority) error {
}
sct, err := impl.GetSCTReceipt(gsctReq.Serial, gsctReq.LogID)
jsonResponse, err := json.Marshal(core.RPCSignedCertificateTimestamp(sct))
jsonResponse, err := json.Marshal(sct)
if err != nil {
// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
errorCondition(MethodGetSCTReceipt, err, req)
@ -1147,7 +1147,7 @@ func NewStorageAuthorityServer(rpc Server, impl core.StorageAuthority) error {
})
rpc.Handle(MethodAddSCTReceipt, func(req []byte) (response []byte, err error) {
var sct core.RPCSignedCertificateTimestamp
var sct core.SignedCertificateTimestamp
err = json.Unmarshal(req, &sct)
if err != nil {
// AUDIT[ Improper Messages ] 0786b6f2-91ca-4f48-9883-842a19084c64

2
test/boulder-config.json
View File

@ -295,8 +295,6 @@
"key": "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA=="
}
],
"submissionRetries": 1,
"submissionBackoff": "1s",
"intermediateBundleFilename": "test/test-ca.pem"
}
},

68
test/integration-test.py
View File

@ -2,6 +2,7 @@
import argparse
import atexit
import base64
import json
import os
import re
import shutil
@ -9,8 +10,8 @@ import socket
import subprocess
import sys
import tempfile
import urllib
import time
import urllib
import urllib2
import startservers
@ -212,6 +213,11 @@ def run_client_tests():
if subprocess.Popen(cmd, shell=True, cwd=root, executable='/bin/bash').wait() != 0:
die(ExitStatus.PythonFailure)
def check_activity_monitor():
"""Ensure that the activity monitor is running and received some messages."""
resp = urllib2.urlopen("http://localhost:8007/debug/vars")
debug_vars = json.loads(resp.read())
assert debug_vars['messages'] > 0, "Activity Monitor received zero messages."
@atexit.register
def cleanup():
@ -220,36 +226,48 @@ def cleanup():
if exit_status == ExitStatus.OK:
print("\n\nSUCCESS")
else:
print("\n\nFAILURE %d" % exit_status)
if exit_status:
print("\n\nFAILURE %d" % exit_status)
exit_status = ExitStatus.OK
exit_status = None
tempdir = tempfile.mkdtemp()
parser = argparse.ArgumentParser(description='Run integration tests')
parser.add_argument('--all', dest="run_all", action="store_true",
help="run all of the clients' integration tests")
parser.add_argument('--letsencrypt', dest='run_letsencrypt', action='store_true',
help="run the letsencrypt's (the python client's) integration tests")
parser.add_argument('--node', dest="run_node", action="store_true",
help="run the node client's integration tests")
parser.set_defaults(run_all=False, run_letsencrypt=False, run_node=False)
args = parser.parse_args()
def main():
parser = argparse.ArgumentParser(description='Run integration tests')
parser.add_argument('--all', dest="run_all", action="store_true",
help="run all of the clients' integration tests")
parser.add_argument('--letsencrypt', dest='run_letsencrypt', action='store_true',
help="run the letsencrypt's (the python client's) integration tests")
parser.add_argument('--node', dest="run_node", action="store_true",
help="run the node client's integration tests")
parser.set_defaults(run_all=False, run_letsencrypt=False, run_node=False)
args = parser.parse_args()
if not (args.run_all or args.run_letsencrypt or args.run_node):
print >> sys.stderr, "must run at least one of the letsencrypt or node tests with --all, --letsencrypt, or --node"
die(ExitStatus.IncorrectCommandLineArgs)
if not (args.run_all or args.run_letsencrypt or args.run_node):
print >> sys.stderr, "must run at least one of the letsencrypt or node tests with --all, --letsencrypt, or --node"
die(ExitStatus.IncorrectCommandLineArgs)
if not startservers.start(race_detection=True):
die(ExitStatus.Error)
if not startservers.start(race_detection=True):
die(ExitStatus.Error)
if args.run_all or args.run_node:
run_node_test()
if args.run_all or args.run_node:
run_node_test()
# Simulate a disconnection from RabbitMQ to make sure reconnects work.
startservers.bounce_forward()
# Simulate a disconnection from RabbitMQ to make sure reconnects work.
startservers.bounce_forward()
if args.run_all or args.run_letsencrypt:
run_client_tests()
if args.run_all or args.run_letsencrypt:
run_client_tests()
if not startservers.check():
die(ExitStatus.Error)
check_activity_monitor()
if not startservers.check():
die(ExitStatus.Error)
exit_status = ExitStatus.OK
if __name__ == "__main__":
try:
main()
except Exception:
exit_status = ExitStatus.Error
raise