From 86ef841256108ede138f6556bf2590b5c5514e06 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 10 Aug 2017 10:34:25 +0200 Subject: [PATCH] apiservers: add synchronous shutdown mechanism on SIGTERM+INT Kubernetes-commit: 11b25366bc7bfe2ad273c8bf9c332fd9d233bffc --- pkg/audit/types.go | 4 +++ pkg/audit/union.go | 6 ++++ pkg/audit/union_test.go | 4 +++ pkg/server/BUILD | 9 +++++- pkg/server/genericapiserver.go | 5 ++++ pkg/server/signal.go | 43 +++++++++++++++++++++++++++++ pkg/server/signal_posix.go | 26 +++++++++++++++++ pkg/server/signal_windows.go | 23 +++++++++++++++ plugin/pkg/audit/log/backend.go | 4 +++ plugin/pkg/audit/webhook/webhook.go | 8 ++++++ 10 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 pkg/server/signal.go create mode 100644 pkg/server/signal_posix.go create mode 100644 pkg/server/signal_windows.go diff --git a/pkg/audit/types.go b/pkg/audit/types.go index a7c10cf03..0b27b0536 100644 --- a/pkg/audit/types.go +++ b/pkg/audit/types.go @@ -34,4 +34,8 @@ type Backend interface { // Run will initialize the backend. It must not block, but may run go routines in the background. If // stopCh is closed, it is supposed to stop them. Run will be called before the first call to ProcessEvents. Run(stopCh <-chan struct{}) error + + // Shutdown will synchronously shut down the backend while making sure that all pending + // events are delivered. + Shutdown() } diff --git a/pkg/audit/union.go b/pkg/audit/union.go index ba969cec9..856b5c125 100644 --- a/pkg/audit/union.go +++ b/pkg/audit/union.go @@ -49,3 +49,9 @@ func (u union) Run(stopCh <-chan struct{}) error { } return errors.AggregateGoroutines(funcs...) } + +func (u union) Shutdown() { + for _, backend := range u.backends { + backend.Shutdown() + } +} diff --git a/pkg/audit/union_test.go b/pkg/audit/union_test.go index c016f3d07..70d33b03f 100644 --- a/pkg/audit/union_test.go +++ b/pkg/audit/union_test.go @@ -36,6 +36,10 @@ func (f *fakeBackend) Run(stopCh <-chan struct{}) error { return nil } +func (u *fakeBackend) Shutdown() { + // nothing to do here +} + func TestUnion(t *testing.T) { backends := []Backend{ new(fakeBackend), diff --git a/pkg/server/BUILD b/pkg/server/BUILD index 7e3433955..e4ae0d178 100644 --- a/pkg/server/BUILD +++ b/pkg/server/BUILD @@ -52,7 +52,14 @@ go_library( "hooks.go", "plugins.go", "serve.go", - ], + "signal.go", + "signal_posix.go", + ] + select({ + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "signal_windows.go", + ], + "//conditions:default": [], + }), deps = [ "//vendor/github.com/coreos/go-systemd/daemon:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", diff --git a/pkg/server/genericapiserver.go b/pkg/server/genericapiserver.go index 806f5dc9e..419597927 100644 --- a/pkg/server/genericapiserver.go +++ b/pkg/server/genericapiserver.go @@ -246,6 +246,11 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error { } <-stopCh + + if s.GenericAPIServer.AuditBackend != nil { + s.GenericAPIServer.AuditBackend.Shutdown() + } + return nil } diff --git a/pkg/server/signal.go b/pkg/server/signal.go new file mode 100644 index 000000000..1cd8cefaa --- /dev/null +++ b/pkg/server/signal.go @@ -0,0 +1,43 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "os" + "os/signal" +) + +var onlyOneSignalHandler = make(chan struct{}) + +// SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned +// which is closed on one of these signals. If a second signal is caught, the program +// is terminated with exit code 1. +func SetupSignalHandler() (stopCh <-chan struct{}) { + close(onlyOneSignalHandler) // panics when called twice + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + <-c + close(stop) + <-c + os.Exit(1) // second signal. Exit directly. + }() + + return stop +} diff --git a/pkg/server/signal_posix.go b/pkg/server/signal_posix.go new file mode 100644 index 000000000..11b3bba65 --- /dev/null +++ b/pkg/server/signal_posix.go @@ -0,0 +1,26 @@ +// +build !windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "os" + "syscall" +) + +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} diff --git a/pkg/server/signal_windows.go b/pkg/server/signal_windows.go new file mode 100644 index 000000000..e7645a208 --- /dev/null +++ b/pkg/server/signal_windows.go @@ -0,0 +1,23 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "os" +) + +var shutdownSignals = []os.Signal{os.Interrupt} diff --git a/plugin/pkg/audit/log/backend.go b/plugin/pkg/audit/log/backend.go index 2794e76eb..ff1c12881 100644 --- a/plugin/pkg/audit/log/backend.go +++ b/plugin/pkg/audit/log/backend.go @@ -85,3 +85,7 @@ func (b *backend) logEvent(ev *auditinternal.Event) { func (b *backend) Run(stopCh <-chan struct{}) error { return nil } + +func (u *backend) Shutdown() { + // nothing to do here +} diff --git a/plugin/pkg/audit/webhook/webhook.go b/plugin/pkg/audit/webhook/webhook.go index e20afd2e0..52e21855e 100644 --- a/plugin/pkg/audit/webhook/webhook.go +++ b/plugin/pkg/audit/webhook/webhook.go @@ -117,6 +117,10 @@ func (b *blockingBackend) Run(stopCh <-chan struct{}) error { return nil } +func (b *blockingBackend) Shutdown() { + // nothing to do here +} + func (b *blockingBackend) ProcessEvents(ev ...*auditinternal.Event) { if err := b.processEvents(ev...); err != nil { audit.HandlePluginError(pluginName, err, ev...) @@ -203,6 +207,10 @@ func (b *batchBackend) Run(stopCh <-chan struct{}) error { return nil } +func (b *batchBackend) Shutdown() { + // TODO: send out batched events +} + // sendBatchEvents attempts to batch some number of events to the backend. It POSTs events // in a goroutine and logging any error encountered during the POST. //