diff --git a/pkg/server/healthz.go b/pkg/server/healthz.go index 27032b140..61da5873c 100644 --- a/pkg/server/healthz.go +++ b/pkg/server/healthz.go @@ -103,7 +103,10 @@ func (s *GenericAPIServer) installReadyz() { s.readyzLock.Lock() defer s.readyzLock.Unlock() s.readyzChecksInstalled = true - healthz.InstallReadyzHandler(s.Handler.NonGoRestfulMux, s.readyzChecks...) + healthz.InstallReadyzHandlerWithHealthyFunc(s.Handler.NonGoRestfulMux, func() { + // note: InstallReadyzHandlerWithHealthyFunc guarantees that this is called only once + s.lifecycleSignals.HasBeenReady.Signal() + }, s.readyzChecks...) } // installLivez creates the livez endpoint for this server. diff --git a/pkg/server/healthz/healthz_test.go b/pkg/server/healthz/healthz_test.go index 0fb67c845..89937987e 100644 --- a/pkg/server/healthz/healthz_test.go +++ b/pkg/server/healthz/healthz_test.go @@ -17,6 +17,7 @@ limitations under the License. package healthz import ( + "context" "errors" "fmt" "net/http" @@ -311,3 +312,66 @@ type cacheSyncWaiterStub struct { func (s cacheSyncWaiterStub) WaitForCacheSync(_ <-chan struct{}) map[reflect.Type]bool { return s.startedByInformerType } + +func TestInstallReadyzHandlerWithHealthyFunc(t *testing.T) { + mux := http.NewServeMux() + readyzCh := make(chan struct{}) + + hasBeenReadyCounter := 0 + hasBeenReadyFn := func() { + hasBeenReadyCounter++ + } + InstallReadyzHandlerWithHealthyFunc(mux, hasBeenReadyFn, readyOnChanClose{readyzCh}) + + // scenario 1: expect the check to fail since the channel hasn't been closed + req, err := http.NewRequest("GET", fmt.Sprintf("http://example.com%s", "/readyz"), nil) + if err != nil { + t.Errorf("%v", err) + } + rr := httptest.NewRecorder() + mux.ServeHTTP(rr, req) + if rr.Code != http.StatusInternalServerError { + t.Errorf("scenario 1: unexpected status code returned, expected %d, got %d", http.StatusInternalServerError, rr.Code) + } + + // scenario 2: close the channel that will cause the readyz checker to report success, + // verify that hasBeenReadyFn was called + close(readyzCh) + rr = httptest.NewRecorder() + req = req.Clone(context.TODO()) + mux.ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Errorf("scenario 2: unexpected status code returned, expected %d, got %d", http.StatusOK, rr.Code) + } + if hasBeenReadyCounter != 1 { + t.Errorf("scenario 2: unexpected value of hasBeenReadyCounter, expected 1, got %d", hasBeenReadyCounter) + } + + // scenario 3: checks if hasBeenReadyFn hasn't been called again. + rr = httptest.NewRecorder() + req = req.Clone(context.TODO()) + mux.ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Errorf("scenario 3: unexpected status code returned, expected %d, got %d", http.StatusOK, rr.Code) + } + if hasBeenReadyCounter != 1 { + t.Errorf("scenario 3: unexpected value of hasBeenReadyCounter, expected 1, got %d", hasBeenReadyCounter) + } +} + +type readyOnChanClose struct { + ch <-chan struct{} +} + +func (readyOnChanClose) Name() string { + return "readyOnChanClose" +} + +func (c readyOnChanClose) Check(_ *http.Request) error { + select { + case <-c.ch: + return nil + default: + } + return fmt.Errorf("the provided channel hasn't been closed") +} diff --git a/pkg/server/lifecycle_signals.go b/pkg/server/lifecycle_signals.go index 2297a776e..fda4f0951 100644 --- a/pkg/server/lifecycle_signals.go +++ b/pkg/server/lifecycle_signals.go @@ -26,6 +26,7 @@ Events: - ShutdownInitiated: KILL signal received - AfterShutdownDelayDuration: shutdown delay duration has passed - InFlightRequestsDrained: all in flight request(s) have been drained +- HasBeenReady is signaled when the readyz endpoint succeeds for the first time The following is a sequence of shutdown events that we expect to see during termination: T0: ShutdownInitiated: KILL signal received @@ -95,6 +96,9 @@ type lifecycleSignals struct { // HTTPServerStoppedListening termination event is signaled when the // HTTP Server has stopped listening to the underlying socket. HTTPServerStoppedListening lifecycleSignal + + // HasBeenReady is signaled when the readyz endpoint succeeds for the first time. + HasBeenReady lifecycleSignal } // newLifecycleSignals returns an instance of lifecycleSignals interface to be used @@ -105,6 +109,7 @@ func newLifecycleSignals() lifecycleSignals { AfterShutdownDelayDuration: newNamedChannelWrapper("AfterShutdownDelayDuration"), InFlightRequestsDrained: newNamedChannelWrapper("InFlightRequestsDrained"), HTTPServerStoppedListening: newNamedChannelWrapper("HTTPServerStoppedListening"), + HasBeenReady: newNamedChannelWrapper("HasBeenReady"), } }