Use Tap data on Resource Detail page to display unmeshed resources (#1596)

* Use Tap data on Resource Detail page to display unmeshed resources
that send traffic to the specified resource.

* Don't update neighbors on every websocket recv; this causes too much rendering.
Instead, store in internal variable and update with the api results.

This branch uses the src data from tap to discern which unmeshed resources are
sending traffic to the specified resource. We then show this resource in the
octopus graph.

Note that tap is sampled data, so it's possible for an unmeshed resource to not
show up. Also, because we won't know about the resource until it appears in the
Tap results, results could pop into the chart at any time.

web/app/js/components/Octopus.jsx

@@ -35,47 +35,57 @@ ArrowCol.propTypes = PropTypes.bool.isRequired;
 export default class Octopus extends React.Component {
   static defaultProps = {
     neighbors: {},
-    resource: {}
+    resource: {},
+    unmeshedSources: []
   }
 
   static propTypes = {
     neighbors: PropTypes.shape({}),
-    resource: PropTypes.shape({})
+    resource: PropTypes.shape({}),
+    unmeshedSources: PropTypes.arrayOf(PropTypes.shape({})),
   }
 
-  renderResourceSummary(resource, type) {
+  renderResourceSummary(resource, type, unmeshed) {
     return (
       <div key={resource.name} className={`octopus-body ${type}`}>
         <div className={`octopus-title ${type}`}>
+          { _.isNil(resource.namespace) ? displayName(resource) :
           <this.props.api.ResourceLink
             resource={resource}
             linkText={displayName(resource)} />
+          }
         </div>
-        <div className="octopus-sr-gauge">
-          <Progress
-            className={getSrClassification(resource.successRate)}
-            type="dashboard"
-            format={() => metricToFormatter["SUCCESS_RATE"](resource.successRate)}
-            width={type === "main" ? 132 : 64}
-            percent={resource.successRate * 100}
-            gapDegree={180} />
-        </div>
-        <Metric title="RPS" value={metricToFormatter["REQUEST_RATE"](resource.requestRate)} />
-        <Metric title="P99" value={metricToFormatter["LATENCY"](_.get(resource, "latency.P99"))} />
+        {
+          unmeshed ? <div>Unmeshed</div> :
+          <div>
+            <div className="octopus-sr-gauge">
+              <Progress
+                className={getSrClassification(resource.successRate)}
+                type="dashboard"
+                format={() => metricToFormatter["SUCCESS_RATE"](resource.successRate)}
+                width={type === "main" ? 132 : 64}
+                percent={resource.successRate * 100}
+                gapDegree={180} />
+            </div>
+            <Metric title="RPS" value={metricToFormatter["REQUEST_RATE"](resource.requestRate)} />
+            <Metric title="P99" value={metricToFormatter["LATENCY"](_.get(resource, "latency.P99"))} />
+          </div>
+        }
       </div>
     );
   }
 
   render() {
-    let { resource, neighbors } = this.props;
-    let hasUpstreams = _.size(neighbors.upstream) > 0;
+    let { resource, neighbors, unmeshedSources } = this.props;
+    let hasUpstreams = _.size(neighbors.upstream) > 0 || _.size(unmeshedSources) > 0;
     let hasDownstreams = _.size(neighbors.downstream) > 0;
 
     return (
       <div className="octopus-graph">
         <Row type="flex" justify="center" gutter={32} align="middle">
           <Col span={6} className={`octopus-col ${hasUpstreams ? "resource-col" : ""}`}>
-            {_.map(neighbors.upstream, n => this.renderResourceSummary(n, "neighbor"))}
+            {_.map(_.sortBy(neighbors.upstream, "resource.name"), n => this.renderResourceSummary(n, "neighbor"))}
+            {_.map(_.sortBy(unmeshedSources, "name"), n => this.renderResourceSummary(n, "neighbor", true))}
           </Col>
 
           <ArrowCol showArrow={hasUpstreams} />
@@ -87,7 +97,7 @@ export default class Octopus extends React.Component {
           <ArrowCol showArrow={hasDownstreams} />
 
           <Col span={6} className={`octopus-col ${hasDownstreams ? "resource-col" : ""}`}>
-            {_.map(neighbors.downstream, n => this.renderResourceSummary(n, "neighbor"))}
+            {_.map(_.sortBy(neighbors.downstream, "resource.name"), n => this.renderResourceSummary(n, "neighbor"))}
           </Col>
         </Row>
       </div>

web/app/js/components/ResourceDetail.jsx

@@ -4,6 +4,7 @@ import ErrorBanner from './ErrorBanner.jsx';
 import MetricsTable from './MetricsTable.jsx';
 import Octopus from './Octopus.jsx';
 import PageHeader from './PageHeader.jsx';
+import { processNeighborData } from './util/TapUtils.jsx';
 import { processSingleResourceRollup } from './util/MetricUtils.js';
 import PropTypes from 'prop-types';
 import React from 'react';
@@ -41,6 +42,7 @@ export class ResourceDetailBase extends React.Component {
   constructor(props) {
     super(props);
     this.api = this.props.api;
+    this.unmeshedSources = {};
     this.handleApiError = this.handleApiError.bind(this);
     this.loadFromServer = this.loadFromServer.bind(this);
     this.state = this.getInitialState(props.match, props.pathPrefix);
@@ -60,6 +62,7 @@ export class ResourceDetailBase extends React.Component {
         upstream: {},
         downstream: {}
       },
+      unmeshedSources: {},
       resourceIsMeshed: true,
       pendingRequests: false,
       loaded: false,
@@ -75,6 +78,7 @@ export class ResourceDetailBase extends React.Component {
   componentWillReceiveProps(newProps) {
     // React won't unmount this component when switching resource pages so we need to clear state
     this.api.cancelCurrentRequests();
+    this.unmeshedSources = {};
     this.setState(this.getInitialState(newProps.match, newProps.pathPrefix));
   }
 
@@ -147,7 +151,8 @@ export class ResourceDetailBase extends React.Component {
           },
           loaded: true,
           pendingRequests: false,
-          error: null
+          error: null,
+          unmeshedSources: this.unmeshedSources // in place of debouncing, just update this when we update the rest of the state
         });
       })
       .catch(this.handleApiError);
@@ -165,6 +170,12 @@ export class ResourceDetailBase extends React.Component {
     });
   }
 
+  updateNeighborsFromTapData = (source, sourceLabels) => {
+    // store this outside of state, as updating the state upon every websocket event received
+    // is very costly and causes the page to freeze up
+    this.unmeshedSources = processNeighborData(source, sourceLabels, this.unmeshedSources, this.state.resource.type);
+  }
+
   banner = () => {
     if (!this.state.error) {
       return;
@@ -198,6 +209,7 @@ export class ResourceDetailBase extends React.Component {
           <Octopus
             resource={this.state.resourceMetrics[0]}
             neighbors={this.state.neighborMetrics}
+            unmeshedSources={_.values(this.state.unmeshedSources)}
             api={this.api} />
         </div>
 
@@ -208,6 +220,7 @@ export class ResourceDetailBase extends React.Component {
               pathPrefix={this.props.pathPrefix}
               query={topQuery}
               startTap={true}
+              updateNeighbors={this.updateNeighborsFromTapData}
               maxRowsToDisplay={10} />
           </div>
         }

web/app/js/components/TopEventTable.jsx

@@ -1,6 +1,7 @@
-import BaseTable from './BaseTable.jsx';
+import PropTypes from 'prop-types';
 import React from 'react';
 import { srcDstColumn } from './util/TapUtils.jsx';
+import { Table } from 'antd';
 import { withContext } from './util/AppContext.jsx';
 import { formatLatencySec, numericSort } from './util/Utils.js';
 
@@ -58,10 +59,21 @@ const topColumns = ResourceLink => [
   }
 ];
 
-class TopEventTable extends BaseTable {
+class TopEventTable extends React.Component {
+  static propTypes = {
+    api: PropTypes.shape({
+      ResourceLink: PropTypes.func.isRequired,
+    }).isRequired,
+    tableRows: PropTypes.arrayOf(PropTypes.shape({})),
+  }
+
+  static defaultProps = {
+    tableRows: []
+  }
+
   render() {
     return (
-      <BaseTable
+      <Table
         dataSource={this.props.tableRows}
         columns={topColumns(this.props.api.ResourceLink)}
         rowKey="key"

web/app/js/components/TopModule.jsx

@@ -12,11 +12,13 @@ class TopModule extends React.Component {
     maxRowsToDisplay: PropTypes.number,
     pathPrefix: PropTypes.string.isRequired,
     query: PropTypes.shape({}).isRequired,
-    startTap: PropTypes.bool.isRequired
+    startTap: PropTypes.bool.isRequired,
+    updateNeighbors: PropTypes.func
   }
 
   static defaultProps = {
-    maxRowsToDisplay: 40 // max aggregated top rows to index and display in table
+    maxRowsToDisplay: 40, // max aggregated top rows to index and display in table
+    updateNeighbors: _.noop
   }
 
   constructor(props) {
@@ -67,6 +69,7 @@ class TopModule extends React.Component {
 
   onWebsocketRecv = e => {
     this.indexTapResult(e.data);
+    this.props.updateNeighbors(e.data);
     this.debouncedWebsocketRecvHandler();
   }
 
@@ -158,6 +161,10 @@ class TopModule extends React.Component {
       this.deleteOldestIndexedResult(topResults);
     }
 
+    if (d.base.proxyDirection === "INBOUND") {
+      this.props.updateNeighbors(d.requestInit.source, _.get(d, "requestInit.sourceMeta.labels"), null);
+    }
+
     return topResults;
   }
 
@@ -203,7 +210,7 @@ class TopModule extends React.Component {
   addSuccessCount = d => {
     // cope with the fact that gRPC failures are returned with HTTP status 200
     // and correctly classify gRPC failures as failures
-    let success = parseInt(d.responseInit.http.responseInit.httpStatus, 10) < 500;
+    let success = parseInt(_.get(d, "responseInit.http.responseInit.httpStatus"), 10) < 500;
     if (success) {
       let grpcStatusCode = _.get(d, "responseEnd.http.responseEnd.eos.grpcStatusCode");
       if (!_.isNil(grpcStatusCode)) {

web/app/js/components/util/TapUtils.jsx

@@ -28,6 +28,45 @@ export const tapQueryProps = {
 
 export const tapQueryPropType = PropTypes.shape(tapQueryProps);
 
+
+/*
+  Use tap data to figure out a resource's unmeshed upstreams/downstreams
+*/
+export const processNeighborData = (source, labels, resourceAgg, resourceType) => {
+  if (_.isEmpty(labels)) {
+    return resourceAgg;
+  }
+
+  let neighb = {};
+  if (_.has(labels, resourceType)) {
+    neighb = {
+      type: resourceType,
+      name: labels[resourceType],
+      namespace: labels.namespace
+    };
+  } else if (_.has(labels, "pod")) {
+    neighb = {
+      type: "pod",
+      name: labels.pod,
+      namespace: labels.namespace
+    };
+  } else {
+    neighb = {
+      type: "ip",
+      name: source.str
+    };
+  }
+
+  let key = neighb.type + "/" + neighb.name;
+  if (_.has(labels, "control_plane_ns")) {
+    delete resourceAgg[key];
+  } else {
+    resourceAgg[key] = neighb;
+  }
+
+  return resourceAgg;
+};
+
 export const processTapEvent = jsonString => {
   let d = JSON.parse(jsonString);
   d.source.str = publicAddressToString(_.get(d, "source.ip.ipv4"));