[release-1.19] [controller] fix depth metrics (#3254)

* fix depth metric calculation by decrementing when we have something in the queue

* fix var name to reflect it's true purpose

---------

Co-authored-by: Dave Protasowski <dprotaso@gmail.com>

controller/queue_metrics.go

@@ -88,13 +88,15 @@ func (m *queueMetrics) get(item any) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
-	m.depth.Dec()
-	m.processingStartTimes[item] = m.clock.Now()
-
 	if startTime, exists := m.addTimes[item]; exists {
+		m.depth.Dec()
 		m.latency.Observe(m.sinceInSeconds(startTime))
 		delete(m.addTimes, item)
 	}
+
+	if _, exists := m.processingStartTimes[item]; !exists {
+		m.processingStartTimes[item] = m.clock.Now()
+	}
 }
 
 func (m *queueMetrics) done(item any) {

controller/two_lane_queue.go

@@ -223,9 +223,9 @@ func (q *twoLaneRateLimitingQueue) slowLane() workqueue.TypedInterface[any] {
 // It gets the item from fast lane if it has anything, alternatively
 // the slow lane.
 func (tlq *twoLaneQueue) Get() (any, bool) {
-	item, ok := tlq.consumerQueue.Get()
+	item, shutdown := tlq.consumerQueue.Get()
 	tlq.metrics.get(item)
-	return item, ok
+	return item, shutdown
 }
 
 // Len returns the sum of lengths.