Cache agent response and discard default sampling rates (#902)

* Cache agent response and discard default sampling rates

src/Datadog.Trace/Agent/Api.cs

@@ -23,6 +23,7 @@ namespace Datadog.Trace.Agent
         private readonly string _containerId;
         private readonly FrameworkDescription _frameworkDescription;
         private Uri _tracesEndpoint; // The Uri may be reassigned dynamically so that retry attempts may attempt updated Agent ports
+        private string _cachedResponse;
 
         public Api(Uri baseEndpoint, IApiRequestFactory apiRequestFactory, IStatsd statsd)
         {
@@ -205,8 +206,15 @@ namespace Datadog.Trace.Agent
                     if (response.ContentLength > 0 && Tracer.Instance.Sampler != null)
                     {
                         var responseContent = await response.ReadAsStringAsync().ConfigureAwait(false);
-                        var apiResponse = JsonConvert.DeserializeObject<ApiResponse>(responseContent);
-                        Tracer.Instance.Sampler.SetDefaultSampleRates(apiResponse?.RateByService);
+
+                        if (responseContent != _cachedResponse)
+                        {
+                            var apiResponse = JsonConvert.DeserializeObject<ApiResponse>(responseContent);
+
+                            Tracer.Instance.Sampler.SetDefaultSampleRates(apiResponse?.RateByService);
+
+                            _cachedResponse = responseContent;
+                        }
                     }
                 }
                 catch (Exception ex)

src/Datadog.Trace/Sampling/DefaultSamplingRule.cs

@@ -8,7 +8,7 @@ namespace Datadog.Trace.Sampling
     {
         private static readonly Vendors.Serilog.ILogger Log = DatadogLogging.For<DefaultSamplingRule>();
 
-        private static Dictionary<string, float> _sampleRates = new Dictionary<string, float>();
+        private static Dictionary<SampleRateKey, float> _sampleRates = new Dictionary<SampleRateKey, float>();
 
         public string RuleName => "default-rule";
 
@@ -26,10 +26,15 @@ namespace Datadog.Trace.Sampling
         {
             Log.Debug("Using the default sampling logic");
 
+            if (_sampleRates.Count == 0)
+            {
+                return 1;
+            }
+
             var env = span.GetTag(Tags.Env);
             var service = span.ServiceName;
 
-            var key = $"service:{service},env:{env}";
+            var key = new SampleRateKey(service, env);
 
             if (_sampleRates.TryGetValue(key, out var sampleRate))
             {
@@ -46,17 +51,92 @@ namespace Datadog.Trace.Sampling
         {
             // to avoid locking if writers and readers can access the dictionary at the same time,
             // build the new dictionary first, then replace the old one
-            var rates = new Dictionary<string, float>(StringComparer.OrdinalIgnoreCase);
+            var rates = new Dictionary<SampleRateKey, float>();
 
             if (sampleRates != null)
             {
                 foreach (var pair in sampleRates)
                 {
-                    rates.Add(pair.Key, pair.Value);
+                    // No point in adding default rates
+                    if (pair.Value == 1.0f)
+                    {
+                        continue;
+                    }
+
+                    var key = SampleRateKey.Parse(pair.Key);
+
+                    if (key == null)
+                    {
+                        Log.Warning("Could not parse sample rate key {0}", pair.Key);
+                        continue;
+                    }
+
+                    rates.Add(key.Value, pair.Value);
                 }
             }
 
             _sampleRates = rates;
         }
+
+        private readonly struct SampleRateKey : IEquatable<SampleRateKey>
+        {
+            private static readonly char[] PartSeparator = new[] { ',' };
+            private static readonly char[] ValueSeparator = new[] { ':' };
+
+            private readonly string _service;
+            private readonly string _env;
+
+            public SampleRateKey(string service, string env)
+            {
+                _service = service;
+                _env = env;
+            }
+
+            public static SampleRateKey? Parse(string key)
+            {
+                // Expected format:
+                // service:{service},env:{env}
+                var parts = key.Split(PartSeparator);
+
+                if (parts.Length != 2)
+                {
+                    return null;
+                }
+
+                var serviceParts = parts[0].Split(ValueSeparator, 2);
+
+                if (serviceParts.Length != 2)
+                {
+                    return null;
+                }
+
+                var envParts = parts[1].Split(ValueSeparator, 2);
+
+                if (envParts.Length != 2)
+                {
+                    return null;
+                }
+
+                return new SampleRateKey(serviceParts[1], envParts[1]);
+            }
+
+            public bool Equals(SampleRateKey other)
+            {
+                return _service == other._service && _env == other._env;
+            }
+
+            public override bool Equals(object obj)
+            {
+                return obj is SampleRateKey other && Equals(other);
+            }
+
+            public override int GetHashCode()
+            {
+                unchecked
+                {
+                    return ((_service != null ? _service.GetHashCode() : 0) * 397) ^ (_env != null ? _env.GetHashCode() : 0);
+                }
+            }
+        }
     }
 }

test/Datadog.Trace.Tests/Sampling/DefaultSamplingRuleTests.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using Datadog.Trace.Sampling;
+using Xunit;
+
+namespace Datadog.Trace.Tests.Sampling
+{
+    public class DefaultSamplingRuleTests
+    {
+        [Theory]
+        // Value returned by the agent per default
+        [InlineData("service:,env:", "hello", "world", 1f)]
+        // Does not match
+        [InlineData("service:nope,env:nope", "hello", "world", 1f)]
+        // Nominal case
+        [InlineData("service:hello,env:world", "hello", "world", .5f)]
+        // Too many values
+        [InlineData("service:hello,env:world,xxxx", "hello", "world", 1f)]
+        // ':' in service name
+        [InlineData("service:hello:1,env:world", "hello:1", "world", .5f)]
+        // ':' in env name
+        [InlineData("service:hello,env:world:1", "hello", "world:1", .5f)]
+        public void KeyParsing(string key, string expectedService, string expectedEnv, float expectedRate)
+        {
+            var rule = new DefaultSamplingRule();
+
+            rule.SetDefaultSampleRates(new[] { new KeyValuePair<string, float>(key, .5f) });
+
+            var span = new Span(new SpanContext(1, 1, null, serviceName: expectedService), DateTimeOffset.Now);
+            span.SetTag(Tags.Env, expectedEnv);
+
+            var samplingRate = rule.GetSamplingRate(span);
+
+            Assert.Equal(expectedRate, samplingRate);
+        }
+    }
+}