letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Honeycomb: don't transmit gRPC client spans (#5446) 

Add a test to the Honeycomb SamplerHook which never sends
spans which have a "meta.type" of "grpc_client". 

This field and value are set automatically by the Honeycomb gRPC
client interceptor, and can't be set by application code (any fields
set by application code have "app." prepended to their name).

Never sending these spans reduces our visibility into in-datacenter
network latency, but also reduces the number of spans sent by
roughly 50%.
This commit is contained in:
Aaron Gable 2021-05-28 09:36:01 -07:00 committed by GitHub
parent e3d194f4b0
commit 02b6ea1489
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 38 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/config.go
View File

@ -283,6 +283,7 @@ type BeelineConfig struct {
// makeSampler constructs a SamplerHook which will deterministically decide if
// any given span should be sampled based on its TraceID, which is shared by all
// spans within a trace. If a trace_id can't be found, the span will be sampled.
// If the span is an autogenerated grpc_client span, it will never be sampled.
// A sample rate of 0 defaults to a sample rate of 1 (i.e. all events are sent).
func makeSampler(rate uint32) func(fields map[string]interface{}) (bool, int) {
if rate == 0 {
@ -291,15 +292,16 @@ func makeSampler(rate uint32) func(fields map[string]interface{}) (bool, int) {
upperBound := math.MaxUint32 / rate
return func(fields map[string]interface{}) (bool, int) {
kind, ok := fields["meta.type"].(string)
if ok && kind == "grpc_client" {
return false, 0
}
id, ok := fields["trace.trace_id"].(string)
if !ok {
return true, 0
}
h := fnv.New32()
_, err := h.Write([]byte(id))
if err != nil {
return true, 0
}
h.Write([]byte(id))
return h.Sum32() < upperBound, int(rate)
}
}

74
cmd/config_test.go
View File

@ -99,52 +99,42 @@ func TestTLSConfigLoad(t *testing.T) {
}
func TestSampler(t *testing.T) {
type subcase struct {
span map[string]interface{}
sampled bool
rate int
}
testCases := []struct {
rate uint32
cases []subcase
samplerate uint32
span map[string]interface{}
sampled bool
rate int
}{
// At sample rate 1, both of the well-formed spans should get sampled.
{1, []subcase{
{map[string]interface{}{"trace.trace_id": "foo"}, true, 1},
{map[string]interface{}{"trace.trace_id": ""}, true, 1},
{map[string]interface{}{"trace.trace_id": 1}, true, 0},
{map[string]interface{}{}, true, 0},
}},
// At sample rate 1, both of these should get sampled.
{1, map[string]interface{}{"trace.trace_id": "foo"}, true, 1},
{1, map[string]interface{}{"trace.trace_id": ""}, true, 1},
// At sample rate 0, it should behave the same as sample rate 1.
{0, []subcase{
{map[string]interface{}{"trace.trace_id": "foo"}, true, 1},
{map[string]interface{}{"trace.trace_id": ""}, true, 1},
{map[string]interface{}{"trace.trace_id": 1}, true, 0},
{map[string]interface{}{}, true, 0},
}},
// At sample rate 2, only one of the well-formed spans should be sampled.
{2, []subcase{
{map[string]interface{}{"trace.trace_id": "foo"}, true, 2},
{map[string]interface{}{"trace.trace_id": ""}, false, 2},
{map[string]interface{}{"trace.trace_id": 1}, true, 0},
{map[string]interface{}{}, true, 0},
}},
// At sample rate 100, neither of the well-formed spans should be sampled.
{100, []subcase{
{map[string]interface{}{"trace.trace_id": "foo"}, false, 100},
{map[string]interface{}{"trace.trace_id": ""}, false, 100},
{map[string]interface{}{"trace.trace_id": 1}, true, 0},
{map[string]interface{}{}, true, 0},
}},
{0, map[string]interface{}{"trace.trace_id": "foo"}, true, 1},
{0, map[string]interface{}{"trace.trace_id": ""}, true, 1},
// At sample rate 2, only one of these should be sampled.
{2, map[string]interface{}{"trace.trace_id": "foo"}, true, 2},
{2, map[string]interface{}{"trace.trace_id": ""}, false, 2},
// At sample rate 100, neither of these should be sampled.
{100, map[string]interface{}{"trace.trace_id": "foo"}, false, 100},
{100, map[string]interface{}{"trace.trace_id": ""}, false, 100},
// A span with meta.type grpc_client should never be sampled.
{1, map[string]interface{}{"trace.trace_id": "foo", "meta.type": "grpc_client"}, false, 0},
{100, map[string]interface{}{"trace.trace_id": "foo", "meta.type": "grpc_client"}, false, 0},
// Any other meta.type should not affect sampling.
{1, map[string]interface{}{"trace.trace_id": "foo", "meta.type": "grpc_server"}, true, 1},
{1, map[string]interface{}{"trace.trace_id": "foo", "meta.type": 123}, true, 1},
{100, map[string]interface{}{"trace.trace_id": "foo", "meta.type": "grpc_server"}, false, 100},
{100, map[string]interface{}{"trace.trace_id": "foo", "meta.type": 123}, false, 100},
// A missing or non-string trace_id should result in sampling.
{100, map[string]interface{}{}, true, 0},
{100, map[string]interface{}{"trace.trace_id": 123}, true, 0},
}
for _, tc := range testCases {
s := makeSampler(tc.rate)
for _, c := range tc.cases {
t.Run(fmt.Sprintf("Rate(%d) Id(%s)", tc.rate, c.span["trace.trace_id"]), func(t *testing.T) {
b, i := s(c.span)
test.AssertEquals(t, b, c.sampled)
test.AssertEquals(t, i, c.rate)
})
}
t.Run(fmt.Sprintf("Rate(%d) Span(%s)", tc.samplerate, tc.span), func(t *testing.T) {
s := makeSampler(tc.samplerate)
b, i := s(tc.span)
test.AssertEquals(t, b, tc.sampled)
test.AssertEquals(t, i, tc.rate)
})
}
}