Revive scatterplot: re-add scatterplot to Deployments page (#144)

* Revive scatterplot: re-add scatterplot to Deployments page

Tried to make some UI improvements to address previous problems:

* added a hover bar and tooltip that displays all of the nodes under the bar, 
in descending order of successRate (to correspond with their order in the chart)
* the tooltip looked weird in the empty state so I also added the max/min latencies 
observed there

Also cleans up the Deployments page a little when there are not any "least healthy deployments".

* Previously, the sidebar tooltip would still render the last
highlighted nodes' information when the dots updated. Fix that
by selecting a datapoint to highlight when the dots update.

* Add overlay tooltip with names of highlighted nodes

* Align the node labels with the node, except in cases of label overlap


Signed-off-by: Risha Mars <mars@buoyant.io>

web/app/css/deployments.css

@@ -5,22 +5,22 @@
   width: 50%;
 }
 
-.deployments-list, .scatterplot {
-  padding-top: 30px;
-}
-
 .line-graph {
   margin-top: var(--base-width);
 }
 
-.scatterplot-tooltip .chart-label {
+.scatterplot-display {
-  float: left;
+  font-size: 12px;
-}
 
-.scatterplot-tooltip .tooltip {
+  & .title {
-  float: right;
+    font-weight: var(--font-weight-bold);
-  color: black;
+    margin-top: var(--base-width);
-  font-weight: bold;
+  }
+
+  & .extremal-latencies {
+    padding-bottom: var(--base-width);
+    border-bottom: 1px solid #BDBDBD;
+  }
 }
 
 & .border-container {

web/app/css/scatterplot.css

@@ -20,9 +20,21 @@
 
 circle.dot {
   stroke-width: 2px;
-  opacity: 0.8;
+  fill-opacity: 0.7;
+  stroke-opacity: 0.9;
 }
 
-.dot-label {
+.overlay {
-  font-size: 11px;
+  fill: none;
+  pointer-events: all;
+}
+
+.overlay-tooltip {
+  fill: #777;
+  font-size: 12px;
+}
+
+.vertical-highlight {
+  fill: steelblue;
+  opacity: 0.1;
 }

web/app/js/components/Deployments.jsx

@@ -4,15 +4,25 @@ import ConduitSpinner from "./ConduitSpinner.jsx";
 import DeploymentSummary from './DeploymentSummary.jsx';
 import ErrorBanner from './ErrorBanner.jsx';
 import React from 'react';
-import { rowGutter } from './util/Utils.js';
+import ScatterPlot from './ScatterPlot.jsx';
 import TabbedMetricsTable from './TabbedMetricsTable.jsx';
 import { ApiHelpers, urlsForResource } from './util/ApiHelpers.js';
 import { Col, Row } from 'antd';
 import { emptyMetric, getPodsByDeployment, processRollupMetrics, processTimeseriesMetrics } from './util/MetricUtils.js';
+import { metricToFormatter, rowGutter } from './util/Utils.js';
 import './../../css/deployments.css';
 import 'whatwg-fetch';
 
 const maxTsToFetch = 15; // Beyond this, stop showing sparklines in table
+let nodeStats = (description, node) => (
+  <div>
+    <div className="title">{description}:</div>
+    <div>
+      {node.name} ({metricToFormatter["LATENCY"](_.get(node, ["latency", "P99"]))})
+    </div>
+  </div>
+);
+
 export default class Deployments extends React.Component {
   constructor(props) {
     super(props);
@@ -125,11 +135,19 @@ export default class Deployments extends React.Component {
 
   renderPageContents() {
     let leastHealthyDeployments = this.getLeastHealthyDeployments(this.state.metrics);
+    let scatterplotData = _.reduce(this.state.metrics, (mem, datum) => {
+      if (!_.isNil(datum.successRate) && !_.isNil(datum.latency)) {
+        mem.push(datum);
+      }
+      return mem;
+    }, []);
+
+    let slowestNode = _.maxBy(scatterplotData, 'latency.P99');
+    let fastestNode = _.minBy(scatterplotData, 'latency.P99');
 
     return (
       <div className="clearfix">
-        <div className="subsection-header">Least-healthy deployments</div>
+        {_.isEmpty(leastHealthyDeployments) ? null : <div className="subsection-header">Least-healthy deployments</div>}
-        {_.isEmpty(this.state.metrics) ? <div className="no-data-msg">No data</div> : null}
         <Row gutter={rowGutter}>
           {
             _.map(leastHealthyDeployments, deployment => {
@@ -144,6 +162,31 @@ export default class Deployments extends React.Component {
             })
           }
         </Row>
+        <Row gutter={rowGutter}>
+          { _.isEmpty(scatterplotData) ? null :
+            <div className="deployments-scatterplot">
+              <div className="scatterplot-info">
+                <div className="subsection-header">Success rate vs p99 latency</div>
+              </div>
+              <Row gutter={rowGutter}>
+                <Col span={8}>
+                  <div className="scatterplot-display">
+                    <div className="extremal-latencies">
+                      { !fastestNode ? null : nodeStats("Least latency", fastestNode) }
+                      { !slowestNode ? null : nodeStats("Most latency", slowestNode) }
+                    </div>
+                  </div>
+                </Col>
+                <Col span={16}><div className="scatterplot-chart">
+                  <ScatterPlot
+                    data={scatterplotData}
+                    lastUpdated={this.state.lastUpdated}
+                    containerClassName="scatterplot-chart" />
+                </div></Col>
+              </Row>
+            </div>
+          }
+        </Row>
         <div className="deployments-list">
           <TabbedMetricsTable
             resource="deployment"

web/app/js/components/LatencyOverview.jsx

@@ -5,7 +5,7 @@ import * as d3 from 'd3';
 import './../../css/latency-overview.css';
 import './../../css/line-graph.css';
 
-const defaultSvgWidth = 900;
+const defaultSvgWidth = 874;
 const defaultSvgHeight = 350;
 const margin = { top: 20, right: 0, bottom: 30, left: 0 };
 const dataDefaults = { P50: [], P95: [], P99: [] };

web/app/js/components/ScatterPlot.jsx

@@ -1,27 +1,57 @@
+import _ from 'lodash';
+import { metricToFormatter } from './util/Utils.js';
 import React from 'react';
 import * as d3 from 'd3';
-import { metricToFormatter } from './util/Utils.js';
+import './../../css/scatterplot.css';
-import styles from './../../css/scatterplot.css';
 
-const defaultSvgWidth = 906;
+const defaultSvgWidth = 574;
 const defaultSvgHeight = 375;
-const margin = { top: 10, right: 0, bottom: 10, left: 10 };
+const margin = { top: 0, right: 0, bottom: 10, left: 0 };
-const circleRadius = 16;
+const baseWidth = 8;
+const circleRadius = 2 * baseWidth;
 const graphPadding = 3 * circleRadius;
-const successRateColorScale = d3.scaleLinear()
+const highlightBarWidth = 3 * circleRadius;
+const successRateColorScale = d3.scaleQuantize()
   .domain([0, 1])
-  .range(["#FF9292", "#addd8e"]);
+  .range(["#8B0000", "#FF6347", "#FF4500", "#FFA500","#008000"]);
-const successRateStrokeColorScale = d3.scaleLinear()
-  .domain([0, 1])
-  .range(["#EB5757", "#31a354"]);
 
 export default class ScatterPlot extends React.Component {
   constructor(props) {
     super(props);
+    this.renderNodeTooltipDatum = this.renderNodeTooltipDatum.bind(this);
     this.state = this.getChartDimensions();
   }
 
-  shouldComponentUpdate(nextProps, nextState) {
+  componentWillMount() {
+    this.initializeScales();
+  }
+
+  componentDidMount() {
+    this.svg = d3.select("." + this.props.containerClassName)
+      .append("svg")
+      .attr("class", "scatterplot")
+      .attr("width", defaultSvgWidth)
+      .attr("height", defaultSvgHeight)
+      .append("g")
+      .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+    this.xAxis = this.svg.append("g")
+      .attr("class", "x-axis")
+      .attr("transform", "translate(0," + (this.state.height - graphPadding) + ")");
+
+    this.yAxis = this.svg.append("g")
+      .attr("class", "y-axis")
+      .attr("transform", "translate(" + this.state.width + ",0)");
+
+    this.sidebar = d3.select(".scatterplot-display")
+      .append("div").attr("class", "sidebar-tooltip");
+
+    this.initializeVerticalHighlight();
+    this.renderAxisLabels();
+    this.updateGraph();
+  }
+
+  shouldComponentUpdate(nextProps) {
     if (nextProps.lastUpdated === this.props.lastUpdated) {
       // control whether react re-renders the component
       // only rerender if the input data has changed
@@ -30,14 +60,50 @@ export default class ScatterPlot extends React.Component {
     return true;
   }
 
-  componentWillMount() {
-    this.initializeScales();
-  }
-
   componentDidUpdate() {
     this.updateGraph();
   }
 
+  initializeVerticalHighlight() {
+    this.updateScales(this.props.data);
+
+    // highlight bar to show x position
+    this.verticalHighlight = this.svg.append("rect")
+      .attr("class", "vertical-highlight")
+      .attr("width", highlightBarWidth)
+      .attr("height", this.state.height);
+
+    // overlay on which to attach mouse events
+    // attach this after all other items are attached otherwise they block mouse events
+    this.overlay = this.svg.append("rect")
+      .attr("transform", "translate(" + margin.left + "," + margin.top + ")")
+      .attr("class", "overlay")
+      .attr("width", this.state.width)
+      .attr("height", this.state.height);
+    this.overlayNode = d3.select(".overlay").node();
+
+    this.overlayTooltip = this.svg.append("g")
+      .attr("class", "overlay-tooltip")
+      .attr("height", this.state.height);
+
+    this.highlightFirstDatapoint();
+  }
+
+  highlightFirstDatapoint() {
+    // when graph is initially loaded / reloaded, set highlight and sidebar to first datapoint
+    let firstDatapoint = _.first(this.props.data);
+    if (firstDatapoint) {
+      let firstLatency = _.get(firstDatapoint, ["latency", "P99"]);
+      let firstLatencyX = this.xScale(firstLatency);
+      let nearestDatapoints = this.getNearbyDatapoints(firstLatencyX, this.props.data);
+
+      this.verticalHighlight
+        .attr("transform", "translate(" + (firstLatencyX - (highlightBarWidth/2)) + ", 0)");
+      this.renderSidebarTooltip(nearestDatapoints);
+      this.overlayTooltip.text('');
+    }
+  }
+
   getChartDimensions() {
     let svgWidth = this.props.width || defaultSvgWidth;
     let svgHeight = this.props.height || defaultSvgHeight;
@@ -51,7 +117,7 @@ export default class ScatterPlot extends React.Component {
       width: width,
       height: height,
       margin: margin
-    }
+    };
   }
 
   initializeScales() {
@@ -60,7 +126,7 @@ export default class ScatterPlot extends React.Component {
   }
 
   updateScales(data) {
-    this.xScale.domain(d3.extent(data, d => _.get(d, ["latency", "P99", 0, "value"])));
+    this.xScale.domain(d3.extent(data, d => d.latency.P99));
     this.yScale.domain([0, 1]);
 
     this.updateAxes();
@@ -85,32 +151,10 @@ export default class ScatterPlot extends React.Component {
         .attr("x", 4)
         .attr("dx", -10)
         .attr("dy", -4);
-    }
+    };
     this.yAxis.call(customYAxis);
   }
 
-  componentDidMount() {
-    this.svg = d3.select("." + this.props.containerClassName)
-      .append("svg")
-        .attr("width", defaultSvgWidth)
-        .attr("height", defaultSvgHeight)
-      .append("g")
-        .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
-
-    this.xAxis = this.svg.append("g")
-      .attr("class", "x-axis")
-      .attr("transform", "translate(0," + (this.state.height - graphPadding) + ")");
-
-    this.yAxis = this.svg.append("g")
-      .attr("class", "y-axis")
-      .attr("transform", "translate(" + this.state.width + ",0)");
-
-    this.tooltip = d3.select("." + this.props.containerClassName + " .scatterplot-tooltip")
-      .append("div").attr("class", "tooltip");
-
-    this.renderAxisLabels();
-  }
-
   renderAxisLabels() {
     // text label for the x axis
     this.svg.append("text")
@@ -128,19 +172,35 @@ export default class ScatterPlot extends React.Component {
       .text("Success rate");
   }
 
-  updateGraph() {
+  getNearbyDatapoints(x, data) {
-    let plotData = _.reduce(this.props.data, (mem, datum) => {
+    // return nodes that have nearby x-coordinates
-      if(!_.isNil(datum.scatterplot.success) && !_.isNil(datum.scatterplot.latency)) {
+    let x0 = this.xScale.invert(x - highlightBarWidth);
-        mem.push(datum.scatterplot);
+    let x1 = this.xScale.invert(x + highlightBarWidth);
-      }
-      return mem;
-    }, []);
-    this.updateScales(plotData);
-    this.scatterPlot = this.svg.selectAll(".dot")
-      .data(plotData)
 
-    this.labels = this.svg.selectAll(".dot-label")
+    if (x0 === x1) {
-      .data(plotData)
+      // handle case where all the x points are in one column
+      let datapointsX = this.xScale(_.first(data).latency.P99);
+      if (Math.abs(x - datapointsX < highlightBarWidth)) {
+        return data;
+      } else {
+        return [];
+      }
+    } else {
+      return _(data).filter(d => {
+        return d.latency.P99 <= x1 && d.latency.P99 >= x0;
+      }).orderBy('successRate', 'desc').value();
+    }
+  }
+
+  updateGraph() {
+    this.updateScales(this.props.data);
+
+    this.scatterPlot = this.svg.selectAll(".dot")
+      .data(this.props.data);
+
+    this.scatterPlot.exit().remove();
+
+    let spNode = this.scatterPlot.node();
 
     this.scatterPlot
       .enter()
@@ -148,29 +208,111 @@ export default class ScatterPlot extends React.Component {
       .attr("class", "dot")
       .attr("r", circleRadius)
       .merge(this.scatterPlot) // newfangled d3 'update' selection
-        .attr("cx", d => this.xScale(_.get(d, ["latency", "P99", 0, "value"])))
+      .attr("cx", d => this.xScale(d.latency.P99))
       .attr("cy", d => this.yScale(d.successRate))
       .style("fill", d => successRateColorScale(d.successRate))
-        .style("stroke", d => successRateStrokeColorScale(d.successRate))
+      .style("stroke", d => successRateColorScale(d.successRate))
-        .on("mousemove", d => {
+      .on("mousemove", () => {
-          let sr = metricToFormatter["SUCCESS_RATE"](d.successRate);
+        if (spNode) {
-          let latency = metricToFormatter["LATENCY"](_.get(d, ["latency", "P99", 0, "value"]));
+          let currXPos = d3.mouse(spNode)[0];
-          this.tooltip
+          this.positionOverlayHighlightAndTooltip(currXPos);
-            .style("left", d3.event.pageX - 50 + "px")
+        }
-            .style("top", d3.event.pageY - 70 + "px")
+      });
-            .style("display", "inline-block") // show tooltip
-            .text(`${d.name}: (${latency}, ${sr})`);
-        })
-        .on("mouseout", () => this.tooltip.style("display", "none"));
 
-    this.labels
+    this.highlightFirstDatapoint();
-      .enter()
+    this.overlay
-        .append("text")
+      .on("mousemove", () => {
-        .attr("class", "dot-label")
+        let currXPos = d3.mouse(this.overlayNode)[0];
-      .merge(this.labels)
+        this.positionOverlayHighlightAndTooltip(currXPos);
-        .text(d => d.name)
+      });
-        .attr("x", d => this.xScale(_.get(d, ["latency", "P99", 0, "value"])) - circleRadius)
+  }
-        .attr("y", d => this.yScale(d.successRate) - 2 * circleRadius)
+
+  positionOverlayHighlightAndTooltip(currXPos) {
+    let nearestDatapoints = this.getNearbyDatapoints(currXPos, this.props.data);
+    this.renderOverlayTooltip(nearestDatapoints);
+    this.renderSidebarTooltip(nearestDatapoints);
+    this.verticalHighlight.attr("transform", "translate(" + (currXPos - highlightBarWidth / 2) + ", 0)");
+
+    let bbox = this.overlayTooltip.node().getBBox();
+
+    let overlayTooltipYPos = 0;
+    let firstLabelPosition = _.isEmpty(nearestDatapoints) ? null : this.getTooltipLabelY(nearestDatapoints[0]);
+    if (firstLabelPosition + bbox.height > this.state.height - 50) {
+      // if there are a bunch of nodes at 0, the labels could extend below the chart
+      // translate upward if this is the case
+      overlayTooltipYPos -= bbox.height;
+
+      // re-render tooltip labels, squished together
+      this.renderOverlayTooltip(nearestDatapoints, true);
+    }
+
+    let overlayTooltipXPos = currXPos + highlightBarWidth / 2 + baseWidth;
+    if (currXPos > defaultSvgWidth / 2) {
+      // display tooltip to the left if we're on the RH side of the graph
+      overlayTooltipXPos = currXPos - bbox.width - baseWidth - highlightBarWidth / 2;
+    }
+
+    this.overlayTooltip
+      .attr("transform", "translate(" + overlayTooltipXPos + ", " + overlayTooltipYPos + ")")
+      .raise();
+  }
+
+  renderSidebarTooltip(data) {
+    let innerHtml = _.map(data, d => this.renderNodeTooltipDatum(d));
+    this.sidebar.html(innerHtml.join(''));
+  }
+
+  renderOverlayTooltip(data, ignoreNodeSpacing = false) {
+    this.overlayTooltip.text('');
+    let labelWithPosition = this.computeTooltipLabelPositions(data, ignoreNodeSpacing);
+
+    _.each(labelWithPosition, d => {
+      this.overlayTooltip
+        .append("text").text(d.name)
+        .attr("x", 0)
+        .attr("y", d.computedY);
+    });
+  }
+
+  getTooltipLabelY(datum) {
+    // position the tooltip label roughly aligned with the center of the node
+    return this.yScale(datum.successRate) + circleRadius / 2 - 5;
+  }
+
+  computeTooltipLabelPositions(data, ignoreNodeSpacing = false) {
+    // in the case that there are multiple nodes in the highlighted area,
+    // try to position each label next to its corresponding node
+    // if the nodes are too close together, simply list the node labels
+    let positions = _.map(data, d => {
+      return {
+        name: d.name,
+        computedY: this.getTooltipLabelY(d)
+      };
+    });
+
+    if (_.size(positions) > 1) {
+      _.each(positions, (_d, i) => {
+        if (i > 0) {
+          if (positions[i].computedY - positions[i - 1].computedY < 10) {
+            // labels are too close together, don't label at the node Y values
+            ignoreNodeSpacing = true;
+          }
+        }
+      });
+      if (ignoreNodeSpacing) {
+        let basePos = positions[0].computedY;
+        _.each(positions, (d, i) => {
+          d.computedY = basePos + i * 15;
+        });
+      }
+    }
+    return positions;
+  }
+
+  renderNodeTooltipDatum(d) {
+    let latency = metricToFormatter["LATENCY"](d.latency.P99);
+    let sr = metricToFormatter["SUCCESS_RATE"](d.successRate);
+    return `<div class="title">${d.name}</div><div>${latency}, ${sr}</div>`;
   }
 
   render() {

web/app/js/components/TabbedMetricsTable.jsx

@@ -146,18 +146,15 @@ export default class TabbedMetricsTable extends React.Component {
   }
 
   preprocessMetrics() {
-    let tableData = this.props.metrics;
+    let tableData = _.cloneDeep(this.props.metrics);
     let totalRequestRate = _.sumBy(this.props.metrics, "requestRate") || 0;
 
     _.each(tableData, datum => {
       datum.totalRequests = totalRequestRate;
       datum.requestDistribution = new Percentage(datum.requestRate, datum.totalRequests);
 
-      _.each(datum.latency, (d, quantile) => {
+      _.each(datum.latency, (value, quantile) => {
-        _.each(d, datapoint => {
+        datum[quantile] = value;
-          let latencyValue = _.isNil(datapoint.value) ? null : parseInt(datapoint.value, 10);
-          datum[quantile] = latencyValue;
-        });
       });
     });
 

web/app/js/components/util/MetricUtils.js

@@ -72,7 +72,10 @@ export const processRollupMetrics = (rawMetrics, targetEntity) => {
         successRate = _.get(datum, gaugeAccessor);
       } else if (datum.name === "LATENCY") {
         let latencies = _.get(datum, latencyAccessor);
-        latency = _.groupBy(latencies, 'label');
+        latency = _.reduce(latencies, (mem, ea) => {
+          mem[ea.label] = _.isNil(ea.value) ? null : parseInt(ea.value, 10);
+          return mem;
+        }, {});
       }
     });
 

web/app/test/MetricUtilsTest.js

@@ -41,9 +41,9 @@ describe('MetricUtils', () => {
           requestRate: 6.1,
           successRate: 0.3770491803278688,
           latency: {
-            P95: [ { label: 'P95', value: '953' } ],
+            P95: 953,
-            P99: [ { label: 'P99', value: '990' } ],
+            P99: 990,
-            P50: [ { label: 'P50', value: '537' } ],
+            P50: 537
           },
           added: true
         }