letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add a Akamai CCU client and use it to purge OCSP responses on revocation and update 

Adds a (currently gated) Akamai CCU API client used to purge GET OCSP responses
from the CDN. It also contains a small tool (cmd/akamai-purger) that can be used
to purge ARLs from the command line.
This commit is contained in:
Roland Shoemaker 2015-10-27 21:24:34 -07:00
parent 8a1e97a4ae
commit 7675f33317
6 changed files with 467 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

245
akamai/cache-client.go Normal file
View File

@ -0,0 +1,245 @@
package akamai
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/core"
blog "github.com/letsencrypt/boulder/log"
)
const (
purgePath = "/ccu/v2/queues/default"
timestampFormat = "20060102T15:04:05-0700"
)
type purgeRequest struct {
Objects []string `json:"objects"`
Type string `json:"type"`
Action string `json:"action"`
}
type purgeResponse struct {
HTTPStatus int `json:"httpStatus"`
Detail string `json:"detail"`
EstimatedSeconds int `json:"estimatedSeconds"`
PurgeID string `json:"purgeId"`
}
// CachePurgeClient talks to the Akamai CCU REST API. It is safe to make concurrent
// purge requests.
type CachePurgeClient struct {
client *http.Client
apiEndpoint string
apiHost string
apiScheme string
clientToken string
clientSecret string
accessToken string
retries int
retryBackoff time.Duration
log *blog.AuditLogger
stats statsd.Statter
clk clock.Clock
}
// errFatal is used by CachePurgeClient.purge to indicate that it failed for a
// reason that cannot be remediated by retying a purge request
type errFatal string
func (e errFatal) Error() string { return string(e) }
var (
// ErrAllRetriesFailed lets the caller of Purge to know if all the purge submission
// attempts failed
ErrAllRetriesFailed = errors.New("All attempts to submit purge request failed")
)
// NewCachePurgeClient constructs a new CachePurgeClient
func NewCachePurgeClient(
endpoint,
clientToken,
clientSecret,
accessToken string,
retries int,
retryBackoff time.Duration,
log *blog.AuditLogger,
stats statsd.Statter,
) (*CachePurgeClient, error) {
if strings.HasSuffix(endpoint, "/") {
endpoint = endpoint[:len(endpoint)-1]
}
apiURL, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
return &CachePurgeClient{
client: new(http.Client),
apiEndpoint: endpoint,
apiHost: apiURL.Host,
apiScheme: strings.ToLower(apiURL.Scheme),
clientToken: clientToken,
clientSecret: clientSecret,
accessToken: accessToken,
retries: retries,
retryBackoff: retryBackoff,
log: log,
stats: stats,
clk: clock.Default(),
}, nil
}
// Akamai uses a special authorization header to identify clients to their EdgeGrid
// APIs, their docs (https://developer.akamai.com/introduction/Client_Auth.html)
// provide a description of the required generation process.
func (cpc *CachePurgeClient) constructAuthHeader(request *http.Request, body []byte, apiPath string, nonce string) (string, error) {
// The akamai API is very time sensitive (recommending reliance on a stratum 2
// or better time source) and, although it doesn't say it anywhere, really wants
// the timestamp to be in the UTC timezone for some reason.
timestamp := cpc.clk.Now().UTC().Format(timestampFormat)
header := fmt.Sprintf(
"EG1-HMAC-SHA256 client_token=%s;access_token=%s;timestamp=%s;nonce=%s;",
cpc.clientToken,
cpc.accessToken,
timestamp,
nonce,
)
bodyHash := sha256.Sum256(body)
tbs := fmt.Sprintf(
"%s\t%s\t%s\t%s\t%s\t%s\t%s",
"POST",
cpc.apiScheme,
cpc.apiHost,
apiPath,
"", // We don't need to send any signed headers for a purge so this can be blank
base64.StdEncoding.EncodeToString(bodyHash[:]),
header,
)
// Create signing key using a HMAC of the client secret over the timestamp
h := hmac.New(sha256.New, []byte(cpc.clientSecret))
h.Write([]byte(timestamp))
key := make([]byte, base64.StdEncoding.EncodedLen(32))
base64.StdEncoding.Encode(key, h.Sum(nil))
h = hmac.New(sha256.New, key)
h.Write([]byte(tbs))
return fmt.Sprintf(
"%ssignature=%s",
header,
base64.StdEncoding.EncodeToString(h.Sum(nil)),
), nil
}
// purge actually sends the individual requests to the Akamai endpoint and checks
// if they are successful
func (cpc *CachePurgeClient) purge(urls []string) error {
purgeReq := purgeRequest{
Objects: urls,
Action: "remove",
Type: "arl",
}
reqJSON, err := json.Marshal(purgeReq)
if err != nil {
return errFatal(err.Error())
}
req, err := http.NewRequest(
"POST",
fmt.Sprintf("%s%s", cpc.apiEndpoint, purgePath),
bytes.NewBuffer(reqJSON),
)
if err != nil {
return errFatal(err.Error())
}
// Create authorization header for request
authHeader, err := cpc.constructAuthHeader(
req,
reqJSON,
purgePath,
core.RandomString(16),
)
if err != nil {
return errFatal(err.Error())
}
req.Header.Set("Authorization", authHeader)
req.Header.Set("Content-Type", "application/json")
rS := cpc.clk.Now()
resp, err := cpc.client.Do(req)
cpc.stats.TimingDuration("CCU.PurgeRequestLatency", time.Since(rS), 1.0)
if err != nil {
return err
}
if resp.Body == nil {
return fmt.Errorf("No response body")
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
// Check purge was successful
var purgeInfo purgeResponse
err = json.Unmarshal(body, &purgeInfo)
if err != nil {
return err
}
if purgeInfo.HTTPStatus != 201 || resp.StatusCode != 201 {
return fmt.Errorf("Incorrect HTTP status code: %s", string(body))
}
cpc.log.Info(fmt.Sprintf(
"Sent successful purge request purgeID: %s, purge expected in: %ds, for URLs: %s",
purgeInfo.PurgeID,
purgeInfo.EstimatedSeconds,
urls,
))
return nil
}
// Purge attempts to send a purge request to the Akamai CCU API cpc.retries number
// of times before giving up and returning ErrAllRetriesFailed
func (cpc *CachePurgeClient) Purge(urls []string) error {
successful := false
for i := 0; i <= cpc.retries; i++ {
core.RetryBackoff(i, cpc.retryBackoff, time.Minute, 1.3)
err := cpc.purge(urls)
if err != nil {
if _, ok := err.(errFatal); ok {
cpc.log.AuditErr(err)
cpc.stats.Inc("CCU.FatalFailures", 1, 1.0)
return err
}
cpc.stats.Inc("CCU.RetryableFailures", 1, 1.0)
continue
}
successful = true
break
}
if !successful {
cpc.stats.Inc("CCU.FatalFailures", 1, 1.0)
return ErrAllRetriesFailed
}
cpc.stats.Inc("CCU.SuccessfulPurges", 1, 1.0)
return nil
}

51
akamai/cache-client_test.go Normal file
View File

@ -0,0 +1,51 @@
package akamai
import (
"bytes"
"fmt"
"net/http"
"testing"
"time"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/test"
)
func TestConstructAuthHeader(t *testing.T) {
stats, _ := statsd.NewNoopClient(nil)
cpc, err := NewCachePurgeClient(
"https://akaa-baseurl-xxxxxxxxxxx-xxxxxxxxxxxxx.luna.akamaiapis.net",
"akab-client-token-xxx-xxxxxxxxxxxxxxxx",
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=",
"akab-access-token-xxx-xxxxxxxxxxxxxxxx",
0,
time.Second,
nil,
stats,
)
test.AssertNotError(t, err, "Failed to create cache purge client")
fc := clock.NewFake()
cpc.clk = fc
wantedTimestamp, err := time.Parse(timestampFormat, "20140321T19:34:21+0000")
test.AssertNotError(t, err, "Failed to parse timestamp")
fc.Add(wantedTimestamp.Sub(fc.Now()))
req, err := http.NewRequest(
"POST",
fmt.Sprintf("%s%s", cpc.apiEndpoint, purgePath),
bytes.NewBuffer([]byte{0}),
)
test.AssertNotError(t, err, "Failed to create request")
expectedHeader := "EG1-HMAC-SHA256 client_token=akab-client-token-xxx-xxxxxxxxxxxxxxxx;access_token=akab-access-token-xxx-xxxxxxxxxxxxxxxx;timestamp=20140321T19:34:21+0000;nonce=nonce-xx-xxxx-xxxx-xxxx-xxxxxxxxxxxx;signature=hXm4iCxtpN22m4cbZb4lVLW5rhX8Ca82vCFqXzSTPe4="
authHeader, err := cpc.constructAuthHeader(
req,
[]byte("datadatadatadatadatadatadatadata"),
"/testapi/v1/t3",
"nonce-xx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
)
test.AssertNotError(t, err, "Failed to create authorization header")
test.AssertEquals(t, authHeader, expectedHeader)
}

87
cmd/akamai-purger/main.go Normal file
View File

@ -0,0 +1,87 @@
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/codegangsta/cli"
"github.com/letsencrypt/boulder/akamai"
"github.com/letsencrypt/boulder/cmd"
blog "github.com/letsencrypt/boulder/log"
)
func main() {
app := cli.NewApp()
app.Name = "akamai-purger"
app.Usage = "Purge a resource from the Akamai CDN cache"
app.Version = cmd.Version()
app.Author = "Boulder contributors"
app.Email = "ca-dev@letsencrypt.org"
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "config",
Value: "config.json",
EnvVar: "BOULDER_CONFIG",
Usage: "Path to Boulder JSON configuration file",
},
cli.StringFlag{
Name: "url",
Usage: "URL to purge from CDN",
},
}
app.Action = func(c *cli.Context) {
configFileName := c.GlobalString("config")
url := c.GlobalString("url")
if url == "" || configFileName == "" {
fmt.Println("Both -url -config (or BOULDER_CONFIG) are required")
return
}
configJSON, err := ioutil.ReadFile(configFileName)
if err != nil {
fmt.Printf("Failed to read config file: %s\n", err)
return
}
var config cmd.Config
err = json.Unmarshal(configJSON, &config)
stats, err := statsd.NewClient(config.Statsd.Server, config.Statsd.Prefix)
cmd.FailOnError(err, "Couldn't connect to statsd")
// Set up logging
auditlogger, err := blog.Dial(config.Syslog.Network, config.Syslog.Server, config.Syslog.Tag, stats)
cmd.FailOnError(err, "Could not connect to Syslog")
auditlogger.Info(app.Version)
// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
defer auditlogger.AuditPanic()
blog.SetAuditLogger(auditlogger)
akamaiClient, err := akamai.NewCachePurgeClient(
config.OCSPUpdater.AkamaiBaseURL,
config.OCSPUpdater.AkamaiClientToken,
config.OCSPUpdater.AkamaiClientSecret,
config.OCSPUpdater.AkamaiAccessToken,
config.OCSPUpdater.AkamaiPurgeRetries,
config.OCSPUpdater.AkamaiPurgeRetryBackoff.Duration,
auditlogger,
stats,
)
cmd.FailOnError(err, "Failed to create Akamai CachePurgeClient")
err = akamaiClient.Purge([]string{url})
cmd.FailOnError(err, "Failed to purge requested resource")
}
err := app.Run(os.Args)
cmd.FailOnError(err, "Failed to run application")
}

78
cmd/ocsp-updater/main.go
View File

@ -8,13 +8,18 @@ package main
import (
"crypto/x509"
"database/sql"
"encoding/base64"
"fmt"
"net/url"
"path"
"time"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/Godeps/_workspace/src/golang.org/x/crypto/ocsp"
gorp "github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1"
"github.com/letsencrypt/boulder/akamai"
"github.com/letsencrypt/boulder/cmd"
"github.com/letsencrypt/boulder/core"
blog "github.com/letsencrypt/boulder/log"
@ -42,6 +47,9 @@ type OCSPUpdater struct {
numLogs int
loops []*looper
ccu *akamai.CachePurgeClient
issuer *x509.Certificate
}
// This is somewhat gross but can be pared down a bit once the publisher and this
@ -55,6 +63,7 @@ func newUpdater(
sac core.StorageAuthority,
config cmd.OCSPUpdaterConfig,
numLogs int,
issuerPath string,
) (*OCSPUpdater, error) {
if config.NewCertificateBatchSize == 0 ||
config.OldOCSPBatchSize == 0 ||
@ -67,14 +76,16 @@ func newUpdater(
return nil, fmt.Errorf("Loop window sizes must be non-zero")
}
log := blog.GetAuditLogger()
updater := OCSPUpdater{
stats: stats,
clk: clk,
dbMap: dbMap,
cac: ca,
log: log,
sac: sac,
pubc: pub,
log: blog.GetAuditLogger(),
numLogs: numLogs,
ocspMinTimeToExpiry: config.OCSPMinTimeToExpiry.Duration,
oldestIssuedSCT: config.OldestIssuedSCT.Duration,
@ -127,11 +138,61 @@ func newUpdater(
})
}
updater.ocspMinTimeToExpiry = config.OCSPMinTimeToExpiry.Duration
// TODO(#1050): Remove this gate and the nil ccu checks below
if config.AkamaiBaseURL != "" {
issuer, err := core.LoadCert(issuerPath)
ccu, err := akamai.NewCachePurgeClient(
config.AkamaiBaseURL,
config.AkamaiClientToken,
config.AkamaiClientSecret,
config.AkamaiAccessToken,
config.AkamaiPurgeRetries,
config.AkamaiPurgeRetryBackoff.Duration,
log,
stats,
)
if err != nil {
return nil, err
}
updater.ccu = ccu
updater.issuer = issuer
}
return &updater, nil
}
// sendPurge should only be called as a Goroutine as it will block until the purge
// request is successful
func (updater *OCSPUpdater) sendPurge(der []byte) {
cert, err := x509.ParseCertificate(der)
if err != nil {
updater.log.AuditErr(fmt.Errorf("Failed to parse certificate for cache purge: %s", err))
return
}
req, err := ocsp.CreateRequest(cert, updater.issuer, nil)
if err != nil {
updater.log.AuditErr(fmt.Errorf("Failed to create OCSP request for cache purge: %s", err))
return
}
// Create a GET style OCSP url for each endpoint in cert.OCSPServer (still waiting
// on word from Akamai on how to properly purge cached POST requests, for now just
// do GET)
urls := []string{}
for _, ocspServer := range cert.OCSPServer {
urls = append(
urls,
path.Join(ocspServer, url.QueryEscape(base64.StdEncoding.EncodeToString(req))),
)
}
err = updater.ccu.Purge(urls)
if err != nil {
updater.log.AuditErr(fmt.Errorf("Failed to purge OCSP response from CDN: %s", err))
}
}
func (updater *OCSPUpdater) findStaleOCSPResponses(oldestLastUpdatedTime time.Time, batchSize int) ([]core.CertificateStatus, error) {
var statuses []core.CertificateStatus
_, err := updater.dbMap.Select(
@ -207,6 +268,12 @@ func (updater *OCSPUpdater) generateResponse(status core.CertificateStatus) (*co
status.OCSPLastUpdated = updater.clk.Now()
status.OCSPResponse = ocspResponse
// Purge OCSP response from CDN, gated on client having been initialized
if updater.ccu != nil {
go updater.sendPurge(cert.DER)
}
return &status, nil
}
@ -231,6 +298,12 @@ func (updater *OCSPUpdater) generateRevokedResponse(status core.CertificateStatu
now := updater.clk.Now()
status.OCSPLastUpdated = now
status.OCSPResponse = ocspResponse
// Purge OCSP response from CDN, gated on client having been initialized
if updater.ccu != nil {
go updater.sendPurge(cert.DER)
}
return &status, nil
}
@ -520,6 +593,7 @@ func main() {
// Necessary evil for now
c.OCSPUpdater,
len(c.Common.CT.Logs),
c.Common.IssuerCert,
)
cmd.FailOnError(err, "Failed to create updater")

1
cmd/ocsp-updater/main_test.go
View File

@ -80,6 +80,7 @@ func setup(t *testing.T) (*OCSPUpdater, core.StorageAuthority, *gorp.DbMap, cloc
MissingSCTWindow: cmd.ConfigDuration{Duration: time.Second},
},
0,
"",
)
return updater, sa, dbMap, fc, cleanUp

7
cmd/shell.go
View File

@ -306,6 +306,13 @@ type OCSPUpdaterConfig struct {
OCSPMinTimeToExpiry ConfigDuration
OldestIssuedSCT ConfigDuration
AkamaiBaseURL string
AkamaiClientToken string
AkamaiClientSecret string
AkamaiAccessToken string
AkamaiPurgeRetries int
AkamaiPurgeRetryBackoff ConfigDuration
SignFailureBackoffFactor float64
SignFailureBackoffMax ConfigDuration