letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bad-key-revoker: Report unprocessed keys on each invocation (#6197) 

Add a new query to bad-key-revoker, which counts the number of
unprocessed/queued keys on each run. This gives us a metric by
which we can see if the bad-key-revoker is backed up or running
behind.

Fixes #6063
This commit is contained in:
Samantha 2022-07-01 09:38:06 -07:00 committed by GitHub
parent 124aada357
commit b825594fa4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 43 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
cmd/bad-key-revoker/main.go
View File

@ -29,6 +29,12 @@ import (
"google.golang.org/protobuf/types/known/emptypb"
)
const blockedKeysGaugeLimit = 1000
var keysToProcess = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "bad_keys_to_process",
Help: fmt.Sprintf("A gauge of blockedKeys rows to process (max: %d)", blockedKeysGaugeLimit),
})
var keysProcessed = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "bad_keys_processed",
Help: "A counter of blockedKeys rows processed labelled by processing state",
@ -76,6 +82,19 @@ func (ubk uncheckedBlockedKey) String() string {
ubk.RevokedBy, ubk.KeyHash)
}
func (bkr *badKeyRevoker) countUncheckedKeys() (int, error) {
var count int
err := bkr.dbMap.SelectOne(
&count,
`SELECT COUNT(*)
FROM (SELECT 1 FROM blockedKeys
WHERE extantCertificatesChecked = false
LIMIT ?) AS a`,
blockedKeysGaugeLimit,
)
return count, err
}
func (bkr *badKeyRevoker) selectUncheckedKey() (uncheckedBlockedKey, error) {
var row uncheckedBlockedKey
err := bkr.dbMap.SelectOne(
@ -265,6 +284,22 @@ func (bkr *badKeyRevoker) revokeCerts(revokerEmails []string, emailToCerts map[s
// invoke processes a single key in the blockedKeys table and returns whether
// there were any rows to process or not.
func (bkr *badKeyRevoker) invoke() (bool, error) {
// Gather a count of rows to be processed.
uncheckedCount, err := bkr.countUncheckedKeys()
if err != nil {
return false, err
}
// Set the gauge to the number of rows to be processed (max:
// blockedKeysGaugeLimit).
keysToProcess.Set(float64(uncheckedCount))
if uncheckedCount >= blockedKeysGaugeLimit {
bkr.logger.AuditInfof("found >= %d unchecked blocked keys left to process", uncheckedCount)
} else {
bkr.logger.AuditInfof("found %d unchecked blocked keys left to process", uncheckedCount)
}
// select a row to process
unchecked, err := bkr.selectUncheckedKey()
if err != nil {

8
cmd/bad-key-revoker/main_test.go
View File

@ -19,6 +19,7 @@ import (
"github.com/letsencrypt/boulder/sa"
"github.com/letsencrypt/boulder/test"
"github.com/letsencrypt/boulder/test/vars"
"github.com/prometheus/client_golang/prometheus"
"google.golang.org/grpc"
"google.golang.org/protobuf/types/known/emptypb"
)
@ -61,11 +62,17 @@ func TestSelectUncheckedRows(t *testing.T) {
hashA, hashB, hashC := randHash(t), randHash(t), randHash(t)
insertBlockedRow(t, dbMap, fc, hashA, 1, true)
count, err := bkr.countUncheckedKeys()
test.AssertNotError(t, err, "countUncheckedKeys failed")
test.AssertEquals(t, count, 0)
_, err = bkr.selectUncheckedKey()
test.AssertError(t, err, "selectUncheckedKey didn't fail with no rows to process")
test.Assert(t, db.IsNoRows(err), "returned error is not sql.ErrNoRows")
insertBlockedRow(t, dbMap, fc, hashB, 1, false)
insertBlockedRow(t, dbMap, fc, hashC, 1, false)
count, err = bkr.countUncheckedKeys()
test.AssertNotError(t, err, "countUncheckedKeys failed")
test.AssertEquals(t, count, 2)
row, err := bkr.selectUncheckedKey()
test.AssertNotError(t, err, "selectUncheckKey failed")
test.AssertByteEquals(t, row.KeyHash, hashB)
@ -372,6 +379,7 @@ func TestInvoke(t *testing.T) {
test.AssertEquals(t, mr.revoked, 4)
test.AssertEquals(t, len(mm.Messages), 1)
test.AssertEquals(t, mm.Messages[0].To, "example.com")
test.AssertMetricWithLabelsEquals(t, keysToProcess, prometheus.Labels{}, 1)
var checked struct {
ExtantCertificatesChecked bool