letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ratelimits: Provide verbose user-facing rate limit errors (#7653) 

- Instruct callers to call *Decision.Result() to check the result of
rate limit transactions
- Preserve the Transaction within the resulting *Decision
- Generate consistently formatted verbose errors using the metadata
found in the *Decision
- Fix broken key-value rate limits integration test in
TestDuplicateFQDNRateLimit

Fixes #7577
This commit is contained in:
Samantha Frank 2024-08-12 16:14:15 -04:00 committed by GitHub
parent 9e286918f8
commit 6a3e9d725b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 551 additions and 340 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
ra/ra_test.go
View File

@ -874,19 +874,6 @@ func TestPerformValidationSuccess(t *testing.T) {
Problems: nil,
}
var remainingFailedValidations int64
var rlTxns []ratelimits.Transaction
if strings.Contains(os.Getenv("BOULDER_CONFIG_DIR"), "test/config-next") {
// Gather a baseline for the rate limit.
var err error
rlTxns, err = ra.txnBuilder.FailedAuthorizationsPerDomainPerAccountCheckOnlyTransactions(authzPB.RegistrationID, []string{Identifier}, 100)
test.AssertNotError(t, err, "FailedAuthorizationsPerDomainPerAccountCheckOnlyTransactions failed")
d, err := ra.limiter.BatchSpend(ctx, rlTxns)
test.AssertNotError(t, err, "BatchSpend failed")
remainingFailedValidations = d.Remaining
}
now := fc.Now()
challIdx := dnsChallIdx(t, authzPB.Challenges)
authzPB, err := ra.PerformValidation(ctx, &rapb.PerformValidationRequest{
@ -928,13 +915,6 @@ func TestPerformValidationSuccess(t *testing.T) {
// Check that validated timestamp was recorded, stored, and retrieved
expectedValidated := fc.Now()
test.Assert(t, *challenge.Validated == expectedValidated, "Validated timestamp incorrect or missing")
if strings.Contains(os.Getenv("BOULDER_CONFIG_DIR"), "test/config-next") {
// The failed validations bucket should be identical to the baseline.
d, err := ra.limiter.BatchSpend(ctx, rlTxns)
test.AssertNotError(t, err, "BatchSpend failed")
test.AssertEquals(t, d.Remaining, remainingFailedValidations)
}
}
func TestPerformValidationVAError(t *testing.T) {
@ -943,19 +923,6 @@ func TestPerformValidationVAError(t *testing.T) {
authzPB := createPendingAuthorization(t, sa, Identifier, fc.Now().Add(12*time.Hour))
var remainingFailedValidations int64
var rlTxns []ratelimits.Transaction
if strings.Contains(os.Getenv("BOULDER_CONFIG_DIR"), "test/config-next") {
// Gather a baseline for the rate limit.
var err error
rlTxns, err = ra.txnBuilder.FailedAuthorizationsPerDomainPerAccountCheckOnlyTransactions(authzPB.RegistrationID, []string{Identifier}, 100)
test.AssertNotError(t, err, "FailedAuthorizationsPerDomainPerAccountCheckOnlyTransactions failed")
d, err := ra.limiter.BatchSpend(ctx, rlTxns)
test.AssertNotError(t, err, "BatchSpend failed")
remainingFailedValidations = d.Remaining
}
va.PerformValidationRequestResultError = fmt.Errorf("Something went wrong")
challIdx := dnsChallIdx(t, authzPB.Challenges)
@ -995,13 +962,6 @@ func TestPerformValidationVAError(t *testing.T) {
// Check that validated timestamp was recorded, stored, and retrieved
expectedValidated := fc.Now()
test.Assert(t, *challenge.Validated == expectedValidated, "Validated timestamp incorrect or missing")
if strings.Contains(os.Getenv("BOULDER_CONFIG_DIR"), "test/config-next") {
// The failed validations bucket should have been decremented by 1.
d, err := ra.limiter.BatchSpend(ctx, rlTxns)
test.AssertNotError(t, err, "BatchSpend failed")
test.AssertEquals(t, d.Remaining, remainingFailedValidations-1)
}
}
func TestCertificateKeyNotEqualAccountKey(t *testing.T) {

66
ratelimits/gcra.go
View File

@ -9,8 +9,8 @@ import (
// maybeSpend uses the GCRA algorithm to decide whether to allow a request. It
// returns a Decision struct with the result of the decision and the updated
// TAT. The cost must be 0 or greater and <= the burst capacity of the limit.
func maybeSpend(clk clock.Clock, rl limit, tat time.Time, cost int64) *Decision {
if cost < 0 || cost > rl.Burst {
func maybeSpend(clk clock.Clock, txn Transaction, tat time.Time) *Decision {
if txn.cost < 0 || txn.cost > txn.limit.Burst {
// The condition above is the union of the conditions checked in Check
// and Spend methods of Limiter. If this panic is reached, it means that
// the caller has introduced a bug.
@ -27,36 +27,38 @@ func maybeSpend(clk clock.Clock, rl limit, tat time.Time, cost int64) *Decision
}
// Compute the cost increment.
costIncrement := rl.emissionInterval * cost
costIncrement := txn.limit.emissionInterval * txn.cost
// Deduct the cost to find the new TAT and residual capacity.
newTAT := tatUnix + costIncrement
difference := nowUnix - (newTAT - rl.burstOffset)
difference := nowUnix - (newTAT - txn.limit.burstOffset)
if difference < 0 {
// Too little capacity to satisfy the cost, deny the request.
residual := (nowUnix - (tatUnix - rl.burstOffset)) / rl.emissionInterval
residual := (nowUnix - (tatUnix - txn.limit.burstOffset)) / txn.limit.emissionInterval
return &Decision{
Allowed: false,
Remaining: residual,
RetryIn: -time.Duration(difference),
ResetIn: time.Duration(tatUnix - nowUnix),
newTAT: time.Unix(0, tatUnix).UTC(),
allowed: false,
remaining: residual,
retryIn: -time.Duration(difference),
resetIn: time.Duration(tatUnix - nowUnix),
newTAT: time.Unix(0, tatUnix).UTC(),
transaction: txn,
}
}
// There is enough capacity to satisfy the cost, allow the request.
var retryIn time.Duration
residual := difference / rl.emissionInterval
residual := difference / txn.limit.emissionInterval
if difference < costIncrement {
retryIn = time.Duration(costIncrement - difference)
}
return &Decision{
Allowed: true,
Remaining: residual,
RetryIn: retryIn,
ResetIn: time.Duration(newTAT - nowUnix),
newTAT: time.Unix(0, newTAT).UTC(),
allowed: true,
remaining: residual,
retryIn: retryIn,
resetIn: time.Duration(newTAT - nowUnix),
newTAT: time.Unix(0, newTAT).UTC(),
transaction: txn,
}
}
@ -64,8 +66,8 @@ func maybeSpend(clk clock.Clock, rl limit, tat time.Time, cost int64) *Decision
// the cost of a request which was previously spent. The refund cost must be 0
// or greater. A cost will only be refunded up to the burst capacity of the
// limit. A partial refund is still considered successful.
func maybeRefund(clk clock.Clock, rl limit, tat time.Time, cost int64) *Decision {
if cost < 0 || cost > rl.Burst {
func maybeRefund(clk clock.Clock, txn Transaction, tat time.Time) *Decision {
if txn.cost < 0 || txn.cost > txn.limit.Burst {
// The condition above is checked in the Refund method of Limiter. If
// this panic is reached, it means that the caller has introduced a bug.
panic("invalid cost for maybeRefund")
@ -77,16 +79,17 @@ func maybeRefund(clk clock.Clock, rl limit, tat time.Time, cost int64) *Decision
if nowUnix > tatUnix {
// The TAT is in the past, therefore the bucket is full.
return &Decision{
Allowed: false,
Remaining: rl.Burst,
RetryIn: time.Duration(0),
ResetIn: time.Duration(0),
newTAT: tat,
allowed: false,
remaining: txn.limit.Burst,
retryIn: time.Duration(0),
resetIn: time.Duration(0),
newTAT: tat,
transaction: txn,
}
}
// Compute the refund increment.
refundIncrement := rl.emissionInterval * cost
refundIncrement := txn.limit.emissionInterval * txn.cost
// Subtract the refund increment from the TAT to find the new TAT.
newTAT := tatUnix - refundIncrement
@ -97,14 +100,15 @@ func maybeRefund(clk clock.Clock, rl limit, tat time.Time, cost int64) *Decision
}
// Calculate the new capacity.
difference := nowUnix - (newTAT - rl.burstOffset)
residual := difference / rl.emissionInterval
difference := nowUnix - (newTAT - txn.limit.burstOffset)
residual := difference / txn.limit.emissionInterval
return &Decision{
Allowed: (newTAT != tatUnix),
Remaining: residual,
RetryIn: time.Duration(0),
ResetIn: time.Duration(newTAT - nowUnix),
newTAT: time.Unix(0, newTAT).UTC(),
allowed: (newTAT != tatUnix),
remaining: residual,
retryIn: time.Duration(0),
resetIn: time.Duration(newTAT - nowUnix),
newTAT: time.Unix(0, newTAT).UTC(),
transaction: txn,
}
}

248
ratelimits/gcra_test.go
View File

@ -15,121 +15,125 @@ func TestDecide(t *testing.T) {
limit.precompute()
// Begin by using 1 of our 10 requests.
d := maybeSpend(clk, limit, clk.Now(), 1)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(9))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Second)
d := maybeSpend(clk, Transaction{"test", limit, 1, true, true}, clk.Now())
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(9))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Transaction is set when we're allowed.
test.AssertEquals(t, d.transaction, Transaction{"test", limit, 1, true, true})
// Immediately use another 9 of our remaining requests.
d = maybeSpend(clk, limit, d.newTAT, 9)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
d = maybeSpend(clk, Transaction{"test", limit, 9, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
// We should have to wait 1 second before we can use another request but we
// used 9 so we should have to wait 9 seconds to make an identical request.
test.AssertEquals(t, d.RetryIn, time.Second*9)
test.AssertEquals(t, d.ResetIn, time.Second*10)
test.AssertEquals(t, d.retryIn, time.Second*9)
test.AssertEquals(t, d.resetIn, time.Second*10)
// Our new TAT should be 10 seconds (limit.Burst) in the future.
test.AssertEquals(t, d.newTAT, clk.Now().Add(time.Second*10))
// Let's try using just 1 more request without waiting.
d = maybeSpend(clk, limit, d.newTAT, 1)
test.Assert(t, !d.Allowed, "should not be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.RetryIn, time.Second)
test.AssertEquals(t, d.ResetIn, time.Second*10)
d = maybeSpend(clk, Transaction{"test", limit, 1, true, true}, d.newTAT)
test.Assert(t, !d.allowed, "should not be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.retryIn, time.Second)
test.AssertEquals(t, d.resetIn, time.Second*10)
// Transaction is set when we're denied.
test.AssertEquals(t, d.transaction, Transaction{"test", limit, 1, true, true})
// Let's try being exactly as patient as we're told to be.
clk.Add(d.RetryIn)
d = maybeSpend(clk, limit, d.newTAT, 0)
test.AssertEquals(t, d.Remaining, int64(1))
clk.Add(d.retryIn)
d = maybeSpend(clk, Transaction{"test", limit, 0, true, true}, d.newTAT)
test.AssertEquals(t, d.remaining, int64(1))
// We are 1 second in the future, we should have 1 new request.
d = maybeSpend(clk, limit, d.newTAT, 1)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.RetryIn, time.Second)
test.AssertEquals(t, d.ResetIn, time.Second*10)
d = maybeSpend(clk, Transaction{"test", limit, 1, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.retryIn, time.Second)
test.AssertEquals(t, d.resetIn, time.Second*10)
// Let's try waiting (10 seconds) for our whole bucket to refill.
clk.Add(d.ResetIn)
clk.Add(d.resetIn)
// We should have 10 new requests. If we use 1 we should have 9 remaining.
d = maybeSpend(clk, limit, d.newTAT, 1)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(9))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Second)
d = maybeSpend(clk, Transaction{"test", limit, 1, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(9))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Wait just shy of how long we're told to wait for refilling.
clk.Add(d.ResetIn - time.Millisecond)
clk.Add(d.resetIn - time.Millisecond)
// We should still have 9 remaining because we're still 1ms shy of the
// refill time.
d = maybeSpend(clk, limit, d.newTAT, 0)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(9))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond)
d = maybeSpend(clk, Transaction{"test", limit, 0, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(9))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond)
// Spending 0 simply informed us that we still have 9 remaining, let's see
// what we have after waiting 20 hours.
clk.Add(20 * time.Hour)
// C'mon, big money, no whammies, no whammies, STOP!
d = maybeSpend(clk, limit, d.newTAT, 0)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(10))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Duration(0))
d = maybeSpend(clk, Transaction{"test", limit, 0, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(10))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Duration(0))
// Turns out that the most we can accrue is 10 (limit.Burst). Let's empty
// this bucket out so we can try something else.
d = maybeSpend(clk, limit, d.newTAT, 10)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
d = maybeSpend(clk, Transaction{"test", limit, 10, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
// We should have to wait 1 second before we can use another request but we
// used 10 so we should have to wait 10 seconds to make an identical
// request.
test.AssertEquals(t, d.RetryIn, time.Second*10)
test.AssertEquals(t, d.ResetIn, time.Second*10)
test.AssertEquals(t, d.retryIn, time.Second*10)
test.AssertEquals(t, d.resetIn, time.Second*10)
// If you spend 0 while you have 0 you should get 0.
d = maybeSpend(clk, limit, d.newTAT, 0)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Second*10)
d = maybeSpend(clk, Transaction{"test", limit, 0, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Second*10)
// We don't play by the rules, we spend 1 when we have 0.
d = maybeSpend(clk, limit, d.newTAT, 1)
test.Assert(t, !d.Allowed, "should not be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.RetryIn, time.Second)
test.AssertEquals(t, d.ResetIn, time.Second*10)
d = maybeSpend(clk, Transaction{"test", limit, 1, true, true}, d.newTAT)
test.Assert(t, !d.allowed, "should not be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.retryIn, time.Second)
test.AssertEquals(t, d.resetIn, time.Second*10)
// Okay, maybe we should play by the rules if we want to get anywhere.
clk.Add(d.RetryIn)
clk.Add(d.retryIn)
// Our patience pays off, we should have 1 new request. Let's use it.
d = maybeSpend(clk, limit, d.newTAT, 1)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.RetryIn, time.Second)
test.AssertEquals(t, d.ResetIn, time.Second*10)
d = maybeSpend(clk, Transaction{"test", limit, 1, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.retryIn, time.Second)
test.AssertEquals(t, d.resetIn, time.Second*10)
// Refill from empty to 5.
clk.Add(d.ResetIn / 2)
clk.Add(d.resetIn / 2)
// Attempt to spend 7 when we only have 5. We should be denied but the
// decision should reflect a retry of 2 seconds, the time it would take to
// refill from 5 to 7.
d = maybeSpend(clk, limit, d.newTAT, 7)
test.Assert(t, !d.Allowed, "should not be allowed")
test.AssertEquals(t, d.Remaining, int64(5))
test.AssertEquals(t, d.RetryIn, time.Second*2)
test.AssertEquals(t, d.ResetIn, time.Second*5)
d = maybeSpend(clk, Transaction{"test", limit, 7, true, true}, d.newTAT)
test.Assert(t, !d.allowed, "should not be allowed")
test.AssertEquals(t, d.remaining, int64(5))
test.AssertEquals(t, d.retryIn, time.Second*2)
test.AssertEquals(t, d.resetIn, time.Second*5)
}
func TestMaybeRefund(t *testing.T) {
@ -138,88 +142,94 @@ func TestMaybeRefund(t *testing.T) {
limit.precompute()
// Begin by using 1 of our 10 requests.
d := maybeSpend(clk, limit, clk.Now(), 1)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(9))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Second)
d := maybeSpend(clk, Transaction{"test", limit, 1, true, true}, clk.Now())
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(9))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Transaction is set when we're refunding.
test.AssertEquals(t, d.transaction, Transaction{"test", limit, 1, true, true})
// Refund back to 10.
d = maybeRefund(clk, limit, d.newTAT, 1)
test.AssertEquals(t, d.Remaining, int64(10))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Duration(0))
d = maybeRefund(clk, Transaction{"test", limit, 1, true, true}, d.newTAT)
test.AssertEquals(t, d.remaining, int64(10))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Duration(0))
// Refund 0, we should still have 10.
d = maybeRefund(clk, limit, d.newTAT, 0)
test.AssertEquals(t, d.Remaining, int64(10))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Duration(0))
d = maybeRefund(clk, Transaction{"test", limit, 0, true, true}, d.newTAT)
test.AssertEquals(t, d.remaining, int64(10))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Duration(0))
// Spend 1 more of our 10 requests.
d = maybeSpend(clk, limit, d.newTAT, 1)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(9))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Second)
d = maybeSpend(clk, Transaction{"test", limit, 1, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(9))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Wait for our bucket to refill.
clk.Add(d.ResetIn)
clk.Add(d.resetIn)
// Attempt to refund from 10 to 11.
d = maybeRefund(clk, limit, d.newTAT, 1)
test.Assert(t, !d.Allowed, "should not be allowed")
test.AssertEquals(t, d.Remaining, int64(10))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Duration(0))
d = maybeRefund(clk, Transaction{"test", limit, 1, true, true}, d.newTAT)
test.Assert(t, !d.allowed, "should not be allowed")
test.AssertEquals(t, d.remaining, int64(10))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Duration(0))
// Transaction is set when our bucket is full.
test.AssertEquals(t, d.transaction, Transaction{"test", limit, 1, true, true})
// Spend 10 all 10 of our requests.
d = maybeSpend(clk, limit, d.newTAT, 10)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
d = maybeSpend(clk, Transaction{"test", limit, 10, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
// We should have to wait 1 second before we can use another request but we
// used 10 so we should have to wait 10 seconds to make an identical
// request.
test.AssertEquals(t, d.RetryIn, time.Second*10)
test.AssertEquals(t, d.ResetIn, time.Second*10)
test.AssertEquals(t, d.retryIn, time.Second*10)
test.AssertEquals(t, d.resetIn, time.Second*10)
// Attempt a refund of 10.
d = maybeRefund(clk, limit, d.newTAT, 10)
test.AssertEquals(t, d.Remaining, int64(10))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Duration(0))
d = maybeRefund(clk, Transaction{"test", limit, 10, true, true}, d.newTAT)
test.AssertEquals(t, d.remaining, int64(10))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Duration(0))
// Wait 11 seconds to catching up to TAT.
clk.Add(11 * time.Second)
// Attempt to refund to 11, then ensure it's still 10.
d = maybeRefund(clk, limit, d.newTAT, 1)
test.Assert(t, !d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(10))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Duration(0))
d = maybeRefund(clk, Transaction{"test", limit, 1, true, true}, d.newTAT)
test.Assert(t, !d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(10))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Duration(0))
// Transaction is set when our TAT is in the past.
test.AssertEquals(t, d.transaction, Transaction{"test", limit, 1, true, true})
// Spend 5 of our 10 requests, then refund 1.
d = maybeSpend(clk, limit, d.newTAT, 5)
d = maybeRefund(clk, limit, d.newTAT, 1)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(6))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
d = maybeSpend(clk, Transaction{"test", limit, 5, true, true}, d.newTAT)
d = maybeRefund(clk, Transaction{"test", limit, 1, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(6))
test.AssertEquals(t, d.retryIn, time.Duration(0))
// Wait, a 2.5 seconds to refill to 8.5 requests.
clk.Add(time.Millisecond * 2500)
// Ensure we have 8.5 requests.
d = maybeSpend(clk, limit, d.newTAT, 0)
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(8))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
d = maybeSpend(clk, Transaction{"test", limit, 0, true, true}, d.newTAT)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(8))
test.AssertEquals(t, d.retryIn, time.Duration(0))
// Check that ResetIn represents the fractional earned request.
test.AssertEquals(t, d.ResetIn, time.Millisecond*1500)
test.AssertEquals(t, d.resetIn, time.Millisecond*1500)
// Refund 2 requests, we should only have 10, not 10.5.
d = maybeRefund(clk, limit, d.newTAT, 2)
test.AssertEquals(t, d.Remaining, int64(10))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Duration(0))
d = maybeRefund(clk, Transaction{"test", limit, 2, true, true}, d.newTAT)
test.AssertEquals(t, d.remaining, int64(10))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Duration(0))
}

155
ratelimits/limiter.go
View File

@ -5,11 +5,15 @@ import (
"errors"
"fmt"
"math"
"math/rand/v2"
"slices"
"strings"
"time"
"github.com/jmhodges/clock"
"github.com/prometheus/client_golang/prometheus"
berrors "github.com/letsencrypt/boulder/errors"
)
const (
@ -24,7 +28,7 @@ const (
// allowedDecision is an "allowed" *Decision that should be returned when a
// checked limit is found to be disabled.
var allowedDecision = &Decision{Allowed: true, Remaining: math.MaxInt64}
var allowedDecision = &Decision{allowed: true, remaining: math.MaxInt64}
// Limiter provides a high-level interface for rate limiting requests by
// utilizing a leaky bucket-style approach.
@ -62,27 +66,123 @@ func NewLimiter(clk clock.Clock, source source, stats prometheus.Registerer) (*L
}, nil
}
// Decision represents the result of a rate limit check or spend operation. To
// check the result of a *Decision, call the Result() method.
type Decision struct {
// Allowed is true if the bucket possessed enough capacity to allow the
// allowed is true if the bucket possessed enough capacity to allow the
// request given the cost.
Allowed bool
allowed bool
// Remaining is the number of requests the client is allowed to make before
// remaining is the number of requests the client is allowed to make before
// they're rate limited.
Remaining int64
remaining int64
// RetryIn is the duration the client MUST wait before they're allowed to
// retryIn is the duration the client MUST wait before they're allowed to
// make a request.
RetryIn time.Duration
retryIn time.Duration
// ResetIn is the duration the bucket will take to refill to its maximum
// resetIn is the duration the bucket will take to refill to its maximum
// capacity, assuming no further requests are made.
ResetIn time.Duration
resetIn time.Duration
// newTAT indicates the time at which the bucket will be full. It is the
// theoretical arrival time (TAT) of next request. It must be no more than
// (burst * (period / count)) in the future at any single point in time.
newTAT time.Time
// transaction is the Transaction that resulted in this Decision. It is
// included for the production of verbose Subscriber-facing errors. It is
// set by the Limiter before returning the Decision.
transaction Transaction
}
// Result translates a denied *Decision into a berrors.RateLimitError for the
// Subscriber, or returns nil if the *Decision allows the request. The error
// message includes a human-readable description of the exceeded rate limit and
// a retry-after timestamp.
func (d *Decision) Result(now time.Time) error {
if d.allowed {
return nil
}
fmt.Printf("\n\n%#v\n\n", d.transaction)
// Add 0-3% jitter to the RetryIn duration to prevent thundering herd.
jitter := time.Duration(float64(d.retryIn) * 0.03 * rand.Float64())
retryAfter := d.retryIn + jitter
retryAfterTs := now.UTC().Add(retryAfter).Format("2006-01-02 15:04:05 MST")
switch d.transaction.limit.name {
case NewRegistrationsPerIPAddress:
return berrors.RegistrationsPerIPError(
retryAfter,
"too many new registrations (%d) from this IP address in the last %s, retry after %s",
d.transaction.limit.Burst,
d.transaction.limit.Period.Duration,
retryAfterTs,
)
case NewRegistrationsPerIPv6Range:
return berrors.RateLimitError(
retryAfter,
"too many new registrations (%d) from this /48 block of IPv6 addresses in the last %s, retry after %s",
d.transaction.limit.Burst,
d.transaction.limit.Period.Duration,
retryAfterTs,
)
case NewOrdersPerAccount:
return berrors.RateLimitError(
retryAfter,
"too many new orders (%d) from this account in the last %s, retry after %s",
d.transaction.limit.Burst,
d.transaction.limit.Period.Duration,
retryAfterTs,
)
case FailedAuthorizationsPerDomainPerAccount:
// Uses bucket key 'enum:regId:domain'.
idx := strings.LastIndex(d.transaction.bucketKey, ":")
if idx == -1 {
return berrors.InternalServerError("unrecognized bucket key while generating error")
}
domain := d.transaction.bucketKey[idx+1:]
return berrors.FailedValidationError(
retryAfter,
"too many failed authorizations (%d) for %q in the last %s, retry after %s",
d.transaction.limit.Burst,
domain,
d.transaction.limit.Period.Duration,
retryAfterTs,
)
case CertificatesPerDomain, CertificatesPerDomainPerAccount:
// Uses bucket key 'enum:domain' or 'enum:regId:domain' respectively.
idx := strings.LastIndex(d.transaction.bucketKey, ":")
if idx == -1 {
return berrors.InternalServerError("unrecognized bucket key while generating error")
}
domain := d.transaction.bucketKey[idx+1:]
return berrors.RateLimitError(
retryAfter,
"too many certificates (%d) already issued for %q in the last %s, retry after %s",
d.transaction.limit.Burst,
domain,
d.transaction.limit.Period.Duration,
retryAfterTs,
)
case CertificatesPerFQDNSet:
return berrors.DuplicateCertificateError(
retryAfter,
"too many certificates (%d) already issued for this exact set of domains in the last %s, retry after %s",
d.transaction.limit.Burst,
d.transaction.limit.Period.Duration,
retryAfterTs,
)
default:
return berrors.InternalServerError("cannot generate error for unknown rate limit")
}
}
// Check DOES NOT deduct the cost of the request from the provided bucket's
@ -105,9 +205,9 @@ func (l *Limiter) Check(ctx context.Context, txn Transaction) (*Decision, error)
// First request from this client. No need to initialize the bucket
// because this is a check, not a spend. A TAT of "now" is equivalent to
// a full bucket.
return maybeSpend(l.clk, txn.limit, l.clk.Now(), txn.cost), nil
return maybeSpend(l.clk, txn, l.clk.Now()), nil
}
return maybeSpend(l.clk, txn.limit, tat, txn.cost), nil
return maybeSpend(l.clk, txn, tat), nil
}
// Spend attempts to deduct the cost from the provided bucket's capacity. The
@ -144,19 +244,20 @@ type batchDecision struct {
func newBatchDecision() *batchDecision {
return &batchDecision{
Decision: &Decision{
Allowed: true,
Remaining: math.MaxInt64,
allowed: true,
remaining: math.MaxInt64,
},
}
}
func (d *batchDecision) merge(in *Decision) {
d.Allowed = d.Allowed && in.Allowed
d.Remaining = min(d.Remaining, in.Remaining)
d.RetryIn = max(d.RetryIn, in.RetryIn)
d.ResetIn = max(d.ResetIn, in.ResetIn)
d.allowed = d.allowed && in.allowed
d.remaining = min(d.remaining, in.remaining)
d.resetIn = max(d.resetIn, in.resetIn)
if in.newTAT.After(d.newTAT) {
d.newTAT = in.newTAT
d.retryIn = in.retryIn
d.transaction = in.transaction
}
}
@ -200,14 +301,14 @@ func (l *Limiter) BatchSpend(ctx context.Context, txns []Transaction) (*Decision
tat = l.clk.Now()
}
d := maybeSpend(l.clk, txn.limit, tat, txn.cost)
d := maybeSpend(l.clk, txn, tat)
if txn.limit.isOverride() {
utilization := float64(txn.limit.Burst-d.Remaining) / float64(txn.limit.Burst)
utilization := float64(txn.limit.Burst-d.remaining) / float64(txn.limit.Burst)
l.overrideUsageGauge.WithLabelValues(txn.limit.name.String(), txn.limit.overrideKey).Set(utilization)
}
if d.Allowed && (tat != d.newTAT) && txn.spend {
if d.allowed && (tat != d.newTAT) && txn.spend {
// New bucket state should be persisted.
newTATs[txn.bucketKey] = d.newTAT
}
@ -217,12 +318,12 @@ func (l *Limiter) BatchSpend(ctx context.Context, txns []Transaction) (*Decision
}
txnOutcomes[txn] = Denied
if d.Allowed {
if d.allowed {
txnOutcomes[txn] = Allowed
}
}
if batchDecision.Allowed && len(newTATs) > 0 {
if batchDecision.allowed && len(newTATs) > 0 {
err = l.source.BatchSet(ctx, newTATs)
if err != nil {
return nil, err
@ -292,13 +393,13 @@ func (l *Limiter) BatchRefund(ctx context.Context, txns []Transaction) (*Decisio
continue
}
var cost int64
if !txn.checkOnly() {
cost = txn.cost
if txn.checkOnly() {
// The cost of check-only transactions are never refunded.
txn.cost = 0
}
d := maybeRefund(l.clk, txn.limit, tat, cost)
d := maybeRefund(l.clk, txn, tat)
batchDecision.merge(d)
if d.Allowed && tat != d.newTAT {
if d.allowed && tat != d.newTAT {
// New bucket state should be persisted.
newTATs[txn.bucketKey] = d.newTAT
}

334
ratelimits/limiter_test.go
View File

@ -10,6 +10,8 @@ import (
"github.com/jmhodges/clock"
"github.com/prometheus/client_golang/prometheus"
"github.com/letsencrypt/boulder/config"
berrors "github.com/letsencrypt/boulder/errors"
"github.com/letsencrypt/boulder/metrics"
"github.com/letsencrypt/boulder/test"
)
@ -74,16 +76,16 @@ func TestLimiter_CheckWithLimitOverrides(t *testing.T) {
test.AssertNotError(t, err, "txn should be valid")
d, err := l.Spend(testCtx, overriddenTxn40)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.Assert(t, d.allowed, "should be allowed")
// Attempting to spend 1 more, this should fail.
overriddenTxn1, err := newTransaction(overriddenLimit, overriddenBucketKey, 1)
test.AssertNotError(t, err, "txn should be valid")
d, err = l.Spend(testCtx, overriddenTxn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, !d.Allowed, "should not be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.ResetIn, time.Second)
test.Assert(t, !d.allowed, "should not be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Verify our overrideUsageGauge is being set correctly. 1.0 == 100%
// of the bucket has been consumed.
@ -93,38 +95,38 @@ func TestLimiter_CheckWithLimitOverrides(t *testing.T) {
// Verify our RetryIn is correct. 1 second == 1000 milliseconds and
// 1000/40 = 25 milliseconds per request.
test.AssertEquals(t, d.RetryIn, time.Millisecond*25)
test.AssertEquals(t, d.retryIn, time.Millisecond*25)
// Wait 50 milliseconds and try again.
clk.Add(d.RetryIn)
clk.Add(d.retryIn)
// We should be allowed to spend 1 more request.
d, err = l.Spend(testCtx, overriddenTxn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.ResetIn, time.Second)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Wait 1 second for a full bucket reset.
clk.Add(d.ResetIn)
clk.Add(d.resetIn)
// Quickly spend 40 requests in a row.
for i := range 40 {
d, err = l.Spend(testCtx, overriddenTxn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(39-i))
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(39-i))
}
// Attempting to spend 1 more, this should fail.
d, err = l.Spend(testCtx, overriddenTxn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, !d.Allowed, "should not be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.ResetIn, time.Second)
test.Assert(t, !d.allowed, "should not be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Wait 1 second for a full bucket reset.
clk.Add(d.ResetIn)
clk.Add(d.resetIn)
testIP := net.ParseIP(testIP)
normalBucketKey, err := newIPAddressBucketKey(NewRegistrationsPerIPAddress, testIP)
@ -139,27 +141,27 @@ func TestLimiter_CheckWithLimitOverrides(t *testing.T) {
test.AssertNotError(t, err, "txn should be valid")
d, err = l.BatchSpend(testCtx, []Transaction{overriddenTxn1, defaultTxn1})
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(19))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond*50)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(19))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*50)
// Refund quota to both buckets. This should succeed, but the
// decision should reflect that of the default bucket.
d, err = l.BatchRefund(testCtx, []Transaction{overriddenTxn1, defaultTxn1})
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(20))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Duration(0))
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(20))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Duration(0))
// Once more.
d, err = l.BatchSpend(testCtx, []Transaction{overriddenTxn1, defaultTxn1})
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(19))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond*50)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(19))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*50)
// Reset between tests.
err = l.Reset(testCtx, overriddenBucketKey)
@ -174,27 +176,27 @@ func TestLimiter_CheckWithLimitOverrides(t *testing.T) {
test.AssertNotError(t, err, "txn should be valid")
d, err = l.BatchSpend(testCtx, []Transaction{overriddenTxn1, defaultCheckOnlyTxn1})
test.AssertNotError(t, err, "should not error")
test.AssertEquals(t, d.Remaining, int64(19))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond*50)
test.AssertEquals(t, d.remaining, int64(19))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*50)
// Check the remaining quota of the overridden bucket.
overriddenCheckOnlyTxn0, err := newCheckOnlyTransaction(overriddenLimit, overriddenBucketKey, 0)
test.AssertNotError(t, err, "txn should be valid")
d, err = l.Check(testCtx, overriddenCheckOnlyTxn0)
test.AssertNotError(t, err, "should not error")
test.AssertEquals(t, d.Remaining, int64(39))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond*25)
test.AssertEquals(t, d.remaining, int64(39))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*25)
// Check the remaining quota of the default bucket.
defaultTxn0, err := newTransaction(normalLimit, normalBucketKey, 0)
test.AssertNotError(t, err, "txn should be valid")
d, err = l.Check(testCtx, defaultTxn0)
test.AssertNotError(t, err, "should not error")
test.AssertEquals(t, d.Remaining, int64(20))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Duration(0))
test.AssertEquals(t, d.remaining, int64(20))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Duration(0))
// Spend the same bucket but in a batch with a Transaction that is
// spend-only. This should succeed, but the decision should reflect
@ -203,23 +205,23 @@ func TestLimiter_CheckWithLimitOverrides(t *testing.T) {
test.AssertNotError(t, err, "txn should be valid")
d, err = l.BatchSpend(testCtx, []Transaction{overriddenTxn1, defaultSpendOnlyTxn1})
test.AssertNotError(t, err, "should not error")
test.AssertEquals(t, d.Remaining, int64(38))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond*50)
test.AssertEquals(t, d.remaining, int64(38))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*50)
// Check the remaining quota of the overridden bucket.
d, err = l.Check(testCtx, overriddenCheckOnlyTxn0)
test.AssertNotError(t, err, "should not error")
test.AssertEquals(t, d.Remaining, int64(38))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond*50)
test.AssertEquals(t, d.remaining, int64(38))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*50)
// Check the remaining quota of the default bucket.
d, err = l.Check(testCtx, defaultTxn0)
test.AssertNotError(t, err, "should not error")
test.AssertEquals(t, d.Remaining, int64(19))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond*50)
test.AssertEquals(t, d.remaining, int64(19))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*50)
// Once more, but in now the spend-only Transaction will attempt to
// spend 20 requests. The spend-only Transaction should fail, but
@ -228,23 +230,23 @@ func TestLimiter_CheckWithLimitOverrides(t *testing.T) {
test.AssertNotError(t, err, "txn should be valid")
d, err = l.BatchSpend(testCtx, []Transaction{overriddenTxn1, defaultSpendOnlyTxn20})
test.AssertNotError(t, err, "should not error")
test.AssertEquals(t, d.Remaining, int64(37))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond*75)
test.AssertEquals(t, d.remaining, int64(37))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*75)
// Check the remaining quota of the overridden bucket.
d, err = l.Check(testCtx, overriddenCheckOnlyTxn0)
test.AssertNotError(t, err, "should not error")
test.AssertEquals(t, d.Remaining, int64(37))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond*75)
test.AssertEquals(t, d.remaining, int64(37))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*75)
// Check the remaining quota of the default bucket.
d, err = l.Check(testCtx, defaultTxn0)
test.AssertNotError(t, err, "should not error")
test.AssertEquals(t, d.Remaining, int64(19))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.ResetIn, time.Millisecond*50)
test.AssertEquals(t, d.remaining, int64(19))
test.AssertEquals(t, d.retryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*50)
// Reset between tests.
err = l.Reset(testCtx, overriddenBucketKey)
@ -269,12 +271,12 @@ func TestLimiter_InitializationViaCheckAndSpend(t *testing.T) {
test.AssertNotError(t, err, "txn should be valid")
d, err := l.Check(testCtx, txn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(19))
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(19))
// Verify our ResetIn timing is correct. 1 second == 1000
// milliseconds and 1000/20 = 50 milliseconds per request.
test.AssertEquals(t, d.ResetIn, time.Millisecond*50)
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*50)
test.AssertEquals(t, d.retryIn, time.Duration(0))
// However, that cost should not be spent yet, a 0 cost check should
// tell us that we actually have 20 remaining.
@ -282,10 +284,10 @@ func TestLimiter_InitializationViaCheckAndSpend(t *testing.T) {
test.AssertNotError(t, err, "txn should be valid")
d, err = l.Check(testCtx, txn0)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(20))
test.AssertEquals(t, d.ResetIn, time.Duration(0))
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(20))
test.AssertEquals(t, d.resetIn, time.Duration(0))
test.AssertEquals(t, d.retryIn, time.Duration(0))
// Reset our bucket.
err = l.Reset(testCtx, bucketKey)
@ -295,23 +297,23 @@ func TestLimiter_InitializationViaCheckAndSpend(t *testing.T) {
// the bucket. Spend should return the same result as Check.
d, err = l.Spend(testCtx, txn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(19))
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(19))
// Verify our ResetIn timing is correct. 1 second == 1000
// milliseconds and 1000/20 = 50 milliseconds per request.
test.AssertEquals(t, d.ResetIn, time.Millisecond*50)
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*50)
test.AssertEquals(t, d.retryIn, time.Duration(0))
// However, that cost should not be spent yet, a 0 cost check should
// tell us that we actually have 19 remaining.
d, err = l.Check(testCtx, txn0)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(19))
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(19))
// Verify our ResetIn is correct. 1 second == 1000 milliseconds and
// 1000/20 = 50 milliseconds per request.
test.AssertEquals(t, d.ResetIn, time.Millisecond*50)
test.AssertEquals(t, d.RetryIn, time.Duration(0))
test.AssertEquals(t, d.resetIn, time.Millisecond*50)
test.AssertEquals(t, d.retryIn, time.Duration(0))
})
}
}
@ -331,50 +333,50 @@ func TestLimiter_DefaultLimits(t *testing.T) {
test.AssertNotError(t, err, "txn should be valid")
d, err := l.Spend(testCtx, txn20)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.ResetIn, time.Second)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Attempting to spend 1 more, this should fail.
txn1, err := newTransaction(limit, bucketKey, 1)
test.AssertNotError(t, err, "txn should be valid")
d, err = l.Spend(testCtx, txn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, !d.Allowed, "should not be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.ResetIn, time.Second)
test.Assert(t, !d.allowed, "should not be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Verify our ResetIn is correct. 1 second == 1000 milliseconds and
// 1000/20 = 50 milliseconds per request.
test.AssertEquals(t, d.RetryIn, time.Millisecond*50)
test.AssertEquals(t, d.retryIn, time.Millisecond*50)
// Wait 50 milliseconds and try again.
clk.Add(d.RetryIn)
clk.Add(d.retryIn)
// We should be allowed to spend 1 more request.
d, err = l.Spend(testCtx, txn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.ResetIn, time.Second)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Wait 1 second for a full bucket reset.
clk.Add(d.ResetIn)
clk.Add(d.resetIn)
// Quickly spend 20 requests in a row.
for i := range 20 {
d, err = l.Spend(testCtx, txn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(19-i))
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(19-i))
}
// Attempting to spend 1 more, this should fail.
d, err = l.Spend(testCtx, txn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, !d.Allowed, "should not be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.ResetIn, time.Second)
test.Assert(t, !d.allowed, "should not be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.resetIn, time.Second)
})
}
}
@ -394,23 +396,23 @@ func TestLimiter_RefundAndReset(t *testing.T) {
test.AssertNotError(t, err, "txn should be valid")
d, err := l.Spend(testCtx, txn20)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.ResetIn, time.Second)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Refund 10 requests.
txn10, err := newTransaction(limit, bucketKey, 10)
test.AssertNotError(t, err, "txn should be valid")
d, err = l.Refund(testCtx, txn10)
test.AssertNotError(t, err, "should not error")
test.AssertEquals(t, d.Remaining, int64(10))
test.AssertEquals(t, d.remaining, int64(10))
// Spend 10 requests, this should succeed.
d, err = l.Spend(testCtx, txn10)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.ResetIn, time.Second)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.resetIn, time.Second)
err = l.Reset(testCtx, bucketKey)
test.AssertNotError(t, err, "should not error")
@ -418,20 +420,20 @@ func TestLimiter_RefundAndReset(t *testing.T) {
// Attempt to spend 20 more requests, this should succeed.
d, err = l.Spend(testCtx, txn20)
test.AssertNotError(t, err, "should not error")
test.Assert(t, d.Allowed, "should be allowed")
test.AssertEquals(t, d.Remaining, int64(0))
test.AssertEquals(t, d.ResetIn, time.Second)
test.Assert(t, d.allowed, "should be allowed")
test.AssertEquals(t, d.remaining, int64(0))
test.AssertEquals(t, d.resetIn, time.Second)
// Reset to full.
clk.Add(d.ResetIn)
clk.Add(d.resetIn)
// Refund 1 requests above our limit, this should fail.
txn1, err := newTransaction(limit, bucketKey, 1)
test.AssertNotError(t, err, "txn should be valid")
d, err = l.Refund(testCtx, txn1)
test.AssertNotError(t, err, "should not error")
test.Assert(t, !d.Allowed, "should not be allowed")
test.AssertEquals(t, d.Remaining, int64(20))
test.Assert(t, !d.allowed, "should not be allowed")
test.AssertEquals(t, d.remaining, int64(20))
// Spend so we can refund.
_, err = l.Spend(testCtx, txn1)
@ -457,3 +459,133 @@ func TestLimiter_RefundAndReset(t *testing.T) {
})
}
}
func TestRateLimitError(t *testing.T) {
t.Parallel()
now := clock.NewFake().Now()
testCases := []struct {
name string
decision *Decision
expectedErr string
expectedErrType berrors.ErrorType
}{
{
name: "Allowed decision",
decision: &Decision{
allowed: true,
},
},
{
name: "RegistrationsPerIP limit reached",
decision: &Decision{
allowed: false,
retryIn: 5 * time.Second,
transaction: Transaction{
limit: limit{
name: NewRegistrationsPerIPAddress,
Burst: 10,
Period: config.Duration{Duration: time.Hour},
},
},
},
expectedErr: "too many new registrations (10) from this IP address in the last 1h0m0s, retry after 1970-01-01 00:00:05 UTC",
expectedErrType: berrors.RateLimit,
},
{
name: "RegistrationsPerIPv6Range limit reached",
decision: &Decision{
allowed: false,
retryIn: 10 * time.Second,
transaction: Transaction{
limit: limit{
name: NewRegistrationsPerIPv6Range,
Burst: 5,
Period: config.Duration{Duration: time.Hour},
},
},
},
expectedErr: "too many new registrations (5) from this /48 block of IPv6 addresses in the last 1h0m0s, retry after 1970-01-01 00:00:10 UTC",
expectedErrType: berrors.RateLimit,
},
{
name: "FailedAuthorizationsPerDomainPerAccount limit reached",
decision: &Decision{
allowed: false,
retryIn: 15 * time.Second,
transaction: Transaction{
limit: limit{
name: FailedAuthorizationsPerDomainPerAccount,
Burst: 7,
Period: config.Duration{Duration: time.Hour},
},
bucketKey: "4:12345:example.com",
},
},
expectedErr: "too many failed authorizations (7) for \"example.com\" in the last 1h0m0s, retry after 1970-01-01 00:00:15 UTC",
expectedErrType: berrors.RateLimit,
},
{
name: "CertificatesPerDomain limit reached",
decision: &Decision{
allowed: false,
retryIn: 20 * time.Second,
transaction: Transaction{
limit: limit{
name: CertificatesPerDomain,
Burst: 3,
Period: config.Duration{Duration: time.Hour},
},
bucketKey: "5:example.org",
},
},
expectedErr: "too many certificates (3) already issued for \"example.org\" in the last 1h0m0s, retry after 1970-01-01 00:00:20 UTC",
expectedErrType: berrors.RateLimit,
},
{
name: "CertificatesPerDomainPerAccount limit reached",
decision: &Decision{
allowed: false,
retryIn: 20 * time.Second,
transaction: Transaction{
limit: limit{
name: CertificatesPerDomainPerAccount,
Burst: 3,
Period: config.Duration{Duration: time.Hour},
},
bucketKey: "6:12345678:example.net",
},
},
expectedErr: "too many certificates (3) already issued for \"example.net\" in the last 1h0m0s, retry after 1970-01-01 00:00:20 UTC",
expectedErrType: berrors.RateLimit,
},
{
name: "Unknown rate limit name",
decision: &Decision{
allowed: false,
retryIn: 30 * time.Second,
transaction: Transaction{
limit: limit{
name: 9999999,
},
},
},
expectedErr: "cannot generate error for unknown rate limit",
expectedErrType: berrors.InternalServer,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
err := tc.decision.Result(now)
if tc.expectedErr == "" {
test.AssertNotError(t, err, "expected no error")
} else {
test.AssertError(t, err, "expected an error")
test.AssertContains(t, err.Error(), tc.expectedErr)
test.AssertErrorIs(t, err, tc.expectedErrType)
}
})
}
}

33
ratelimits/names.go
View File

@ -13,10 +13,11 @@ import (
// limit names as strings and to provide a type-safe way to refer to rate
// limits.
//
// IMPORTANT: If you add a new limit Name, you MUST add:
// - it to the nameToString mapping,
// - an entry for it in the validateIdForName(), and
// - provide the appropriate constructors in bucket.go.
// IMPORTANT: If you add or remove a limit Name, you MUST update:
// - the string representation of the Name in nameToString,
// - the validators for that name in validateIdForName(),
// - the transaction constructors for that name in bucket.go, and
// - the Subscriber facing error message in ErrForDecision().
type Name int
const (
@ -77,6 +78,18 @@ const (
CertificatesPerFQDNSet
)
// nameToString is a map of Name values to string names.
var nameToString = map[Name]string{
Unknown: "Unknown",
NewRegistrationsPerIPAddress: "NewRegistrationsPerIPAddress",
NewRegistrationsPerIPv6Range: "NewRegistrationsPerIPv6Range",
NewOrdersPerAccount: "NewOrdersPerAccount",
FailedAuthorizationsPerDomainPerAccount: "FailedAuthorizationsPerDomainPerAccount",
CertificatesPerDomain: "CertificatesPerDomain",
CertificatesPerDomainPerAccount: "CertificatesPerDomainPerAccount",
CertificatesPerFQDNSet: "CertificatesPerFQDNSet",
}
// isValid returns true if the Name is a valid rate limit name.
func (n Name) isValid() bool {
return n > Unknown && n < Name(len(nameToString))
@ -99,18 +112,6 @@ func (n Name) EnumString() string {
return strconv.Itoa(int(n))
}
// nameToString is a map of Name values to string names.
var nameToString = map[Name]string{
Unknown: "Unknown",
NewRegistrationsPerIPAddress: "NewRegistrationsPerIPAddress",
NewRegistrationsPerIPv6Range: "NewRegistrationsPerIPv6Range",
NewOrdersPerAccount: "NewOrdersPerAccount",
FailedAuthorizationsPerDomainPerAccount: "FailedAuthorizationsPerDomainPerAccount",
CertificatesPerDomain: "CertificatesPerDomain",
CertificatesPerDomainPerAccount: "CertificatesPerDomainPerAccount",
CertificatesPerFQDNSet: "CertificatesPerFQDNSet",
}
// validIPAddress validates that the provided string is a valid IP address.
func validIPAddress(id string) error {
ip := net.ParseIP(id)

6
test/config-next/wfe2-ratelimit-defaults.yml
View File

@ -19,6 +19,6 @@ NewOrdersPerAccount:
burst: 1500
period: 3h
CertificatesPerFQDNSet:
count: 6
burst: 6
period: 168h
count: 2
burst: 2
period: 3h

9
test/integration/ratelimit_test.go
View File

@ -11,6 +11,7 @@ import (
"github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/cmd"
berrors "github.com/letsencrypt/boulder/errors"
blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/metrics"
"github.com/letsencrypt/boulder/ratelimits"
@ -52,7 +53,7 @@ func TestDuplicateFQDNRateLimit(t *testing.T) {
PasswordFile: "test/secrets/ratelimits_redis_password",
}
fc := clock.NewFake()
fc := clock.New()
stats := metrics.NoopRegisterer
log := blog.NewMock()
ring, err := bredis.NewRingFromConfig(rc, stats, log)
@ -67,8 +68,10 @@ func TestDuplicateFQDNRateLimit(t *testing.T) {
// Check that the CertificatesPerFQDNSet limit is reached.
txns, err := txnBuilder.NewOrderLimitTransactions(1, []string{domain}, 100, false)
test.AssertNotError(t, err, "making transaction")
result, err := limiter.BatchSpend(context.Background(), txns)
decision, err := limiter.BatchSpend(context.Background(), txns)
test.AssertNotError(t, err, "checking transaction")
test.Assert(t, !result.Allowed, "should not be allowed")
err = decision.Result(fc.Now())
test.AssertErrorIs(t, err, berrors.RateLimit)
test.AssertContains(t, err.Error(), "too many certificates (2) already issued for this exact set of domains in the last 3h0m0s")
}
}