Skip to content

Commit bad9104

Browse files
tkashembertinatto
tkashem
authored and
bertinatto
committed
UPSTREAM: <carry>: add conditional shutdown response header
1 parent 18e8d63 commit bad9104

File tree

3 files changed

+85
-172
lines changed

3 files changed

+85
-172
lines changed

‎staging/src/k8s.io/apiserver/pkg/server/config.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1024,7 +1024,6 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
10241024
handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator, c.LongRunningFunc)
10251025
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "audit")
10261026

1027-
handler = genericfilters.WithShutdownLateAnnotation(handler, c.lifecycleSignals.ShutdownInitiated, c.ShutdownDelayDuration)
10281027
handler = genericfilters.WithStartupEarlyAnnotation(handler, c.lifecycleSignals.HasBeenReady)
10291028

10301029
failedHandler := genericapifilters.Unauthorized(c.Serializer)
@@ -1059,6 +1058,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
10591058
handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
10601059
}
10611060
handler = genericfilters.WithOptInRetryAfter(handler, c.newServerFullyInitializedFunc())
1061+
handler = genericfilters.WithShutdownResponseHeader(handler, c.lifecycleSignals.ShutdownInitiated, c.ShutdownDelayDuration, c.APIServerID)
10621062
handler = genericfilters.WithHTTPLogging(handler, c.newIsTerminatingFunc())
10631063
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
10641064
handler = genericapifilters.WithTracing(handler, c.TracerProvider)

‎staging/src/k8s.io/apiserver/pkg/server/filters/with_early_late_annotations.go‎

Lines changed: 28 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -64,16 +64,20 @@ func exemptIfHealthProbe(r *http.Request) bool {
6464
return false
6565
}
6666

67-
// WithShutdownLateAnnotation, if added to the handler chain, tracks the
68-
// incoming request(s) after the apiserver has initiated the graceful
69-
// shutdown, and annoates the audit event for these request(s) with
70-
// diagnostic information.
71-
// This enables us to identify the actor(s)/load balancer(s) that are sending
72-
// requests to the apiserver late during the server termination.
73-
// It should be placed after (in order of execution) the
74-
// 'WithAuthentication' filter.
75-
func WithShutdownLateAnnotation(handler http.Handler, shutdownInitiated lifecycleEvent, delayDuration time.Duration) http.Handler {
76-
return withShutdownLateAnnotation(handler, shutdownInitiated, delayDuration, exemptIfHealthProbe, clockutils.RealClock{})
67+
// WithShutdownResponseHeader, if added to the handler chain, adds a header
68+
// 'X-OpenShift-Disruption' to the response with the following information:
69+
//
70+
// shutdown={true|false} shutdown-delay-duration=%s elapsed=%s host=%s
71+
// shutdown: whether the server is currently shutting down gracefully.
72+
// shutdown-delay-duration: value of --shutdown-delay-duration server run option
73+
// elapsed: how much time has elapsed since the server received a TERM signal
74+
// host: host name of the server, it is used to identify the server instance
75+
// from the others.
76+
//
77+
// This handler will add the response header only if the client opts in by
78+
// adding the 'X-Openshift-If-Disruption' header to the request.
79+
func WithShutdownResponseHeader(handler http.Handler, shutdownInitiated lifecycleEvent, delayDuration time.Duration, apiServerID string) http.Handler {
80+
return withShutdownResponseHeader(handler, shutdownInitiated, delayDuration, apiServerID, clockutils.RealClock{})
7781
}
7882

7983
// WithStartupEarlyAnnotation annotates the request with an annotation keyed as
@@ -84,59 +88,38 @@ func WithStartupEarlyAnnotation(handler http.Handler, hasBeenReady lifecycleEven
8488
return withStartupEarlyAnnotation(handler, hasBeenReady, exemptIfHealthProbe)
8589
}
8690

87-
func withShutdownLateAnnotation(handler http.Handler, shutdownInitiated lifecycleEvent, delayDuration time.Duration, shouldExemptFn shouldExemptFunc, clock clockutils.PassiveClock) http.Handler {
91+
func withShutdownResponseHeader(handler http.Handler, shutdownInitiated lifecycleEvent, delayDuration time.Duration, apiServerID string, clock clockutils.PassiveClock) http.Handler {
8892
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
89-
select {
90-
case <-shutdownInitiated.Signaled():
91-
default:
93+
if len(req.Header.Get("X-Openshift-If-Disruption")) == 0 {
9294
handler.ServeHTTP(w, req)
9395
return
9496
}
9597

96-
if shouldExemptFn(req) {
98+
msgFn := func(shutdown bool, elapsed time.Duration) string {
99+
return fmt.Sprintf("shutdown=%t shutdown-delay-duration=%s elapsed=%s host=%s",
100+
shutdown, delayDuration.Round(time.Second).String(), elapsed.Round(time.Second).String(), apiServerID)
101+
}
102+
103+
select {
104+
case <-shutdownInitiated.Signaled():
105+
default:
106+
w.Header().Set("X-OpenShift-Disruption", msgFn(false, time.Duration(0)))
97107
handler.ServeHTTP(w, req)
98108
return
99109
}
110+
100111
shutdownInitiatedAt := shutdownInitiated.SignaledAt()
101112
if shutdownInitiatedAt == nil {
113+
w.Header().Set("X-OpenShift-Disruption", msgFn(true, time.Duration(0)))
102114
handler.ServeHTTP(w, req)
103115
return
104116
}
105117

106-
elapsedSince := clock.Since(*shutdownInitiatedAt)
107-
// TODO: 80% is the threshold, if requests arrive after 80% of
108-
// shutdown-delay-duration elapses we annotate the request as late=true.
109-
late := lateMsg(delayDuration, elapsedSince, 80)
110-
111-
// NOTE: some upstream unit tests have authentication disabled and will
112-
// fail if we require the requestor to be present in the request
113-
// context. Fixing those unit tests will increase the chance of merge
114-
// conflict during rebase.
115-
// This also implies that this filter must be placed after (in order of
116-
// execution) the 'WithAuthentication' filter.
117-
self := "self="
118-
if requestor, exists := request.UserFrom(req.Context()); exists && requestor != nil {
119-
self = fmt.Sprintf("%s%t", self, requestor.GetName() == user.APIServerUser)
120-
}
121-
122-
message := fmt.Sprintf("%s %s loopback=%t", late, self, isLoopback(req.RemoteAddr))
123-
audit.AddAuditAnnotation(req.Context(), "apiserver.k8s.io/shutdown", message)
124-
118+
w.Header().Set("X-OpenShift-Disruption", msgFn(true, clock.Since(*shutdownInitiatedAt)))
125119
handler.ServeHTTP(w, req)
126-
w.Header().Set("X-OpenShift-Shutdown", message)
127120
})
128121
}
129122

130-
func lateMsg(delayDuration, elapsedSince time.Duration, threshold float64) string {
131-
if delayDuration == time.Duration(0) {
132-
return fmt.Sprintf("elapsed=%s threshold= late=%t", elapsedSince.Round(time.Second).String(), true)
133-
}
134-
135-
percentElapsed := (float64(elapsedSince) / float64(delayDuration)) * 100
136-
return fmt.Sprintf("elapsed=%s threshold=%.2f%% late=%t",
137-
elapsedSince.Round(time.Second).String(), percentElapsed, percentElapsed > threshold)
138-
}
139-
140123
func withStartupEarlyAnnotation(handler http.Handler, hasBeenReady lifecycleEvent, shouldExemptFn shouldExemptFunc) http.Handler {
141124
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
142125
select {

‎staging/src/k8s.io/apiserver/pkg/server/filters/with_early_late_annotations_test.go‎

Lines changed: 56 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package filters
1919
import (
2020
"net/http"
2121
"net/http/httptest"
22-
"strings"
2322
"testing"
2423
"time"
2524

@@ -32,137 +31,90 @@ import (
3231
clocktesting "k8s.io/utils/clock/testing"
3332
)
3433

35-
func TestWithShutdownLateAnnotation(t *testing.T) {
34+
func TestWithShutdownResponseHeader(t *testing.T) {
3635
var (
37-
shutdownDelayDuration = 100 * time.Second
38-
signaledAt = time.Now()
39-
elapsedAtWithingThreshold = signaledAt.Add(shutdownDelayDuration - 21*time.Second)
40-
elapsedAtBeyondThreshold = signaledAt.Add(shutdownDelayDuration - 19*time.Second)
36+
signaledAt = time.Now()
37+
elapsedAt = signaledAt.Add(20 * time.Second)
4138
)
4239

4340
tests := []struct {
44-
name string
45-
shutdownInitiated func() lifecycleEvent
46-
delayDuration time.Duration
47-
user authenticationuser.Info
48-
clock func() utilsclock.PassiveClock
49-
url string
50-
remoteAddr string
51-
handlerInvoked int
52-
statusCodeExpected int
53-
annotationShouldContain string
41+
name string
42+
optIn bool
43+
shutdownInitiated func() lifecycleEvent
44+
delayDuration time.Duration
45+
clock func() utilsclock.PassiveClock
46+
handlerInvoked int
47+
statusCodeExpected int
48+
responseHeader string
5449
}{
5550
{
56-
name: "shutdown is not initiated",
51+
name: "client did not opt in",
5752
shutdownInitiated: func() lifecycleEvent {
58-
return fakeLifecycleSignal{ch: make(chan struct{})}
53+
return nil
5954
},
6055
handlerInvoked: 1,
6156
statusCodeExpected: http.StatusOK,
6257
},
6358
{
64-
name: "shutdown initiated, health probes are not annotated",
59+
name: "client opted in, shutdown not initiated",
60+
optIn: true,
6561
shutdownInitiated: func() lifecycleEvent {
66-
return fakeLifecycleSignal{ch: newClosedChannel()}
62+
return fakeLifecycleSignal{ch: make(chan struct{})}
6763
},
68-
url: "/readyz?verbos=1",
69-
user: &authenticationuser.DefaultInfo{Name: authenticationuser.Anonymous},
64+
delayDuration: 10 * time.Second,
7065
handlerInvoked: 1,
7166
statusCodeExpected: http.StatusOK,
72-
},
73-
// use cases where the request will be annotated
74-
{
75-
name: "shutdown initiated, no user in request context",
76-
shutdownInitiated: func() lifecycleEvent {
77-
return fakeLifecycleSignal{ch: newClosedChannel(), at: &signaledAt}
78-
},
79-
handlerInvoked: 1,
80-
statusCodeExpected: http.StatusOK,
81-
annotationShouldContain: "self= loopback=",
82-
},
83-
{
84-
name: "shutdown initiated, self=true",
85-
shutdownInitiated: func() lifecycleEvent {
86-
return fakeLifecycleSignal{ch: newClosedChannel(), at: &signaledAt}
87-
},
88-
user: &authenticationuser.DefaultInfo{Name: authenticationuser.APIServerUser},
89-
handlerInvoked: 1,
90-
statusCodeExpected: http.StatusOK,
91-
annotationShouldContain: "self=true",
92-
},
93-
{
94-
name: "shutdown initiated, self=false",
95-
shutdownInitiated: func() lifecycleEvent {
96-
return fakeLifecycleSignal{ch: newClosedChannel(), at: &signaledAt}
97-
},
98-
user: &authenticationuser.DefaultInfo{Name: authenticationuser.Anonymous},
99-
handlerInvoked: 1,
100-
statusCodeExpected: http.StatusOK,
101-
annotationShouldContain: "self=false",
67+
responseHeader: "shutdown=false shutdown-delay-duration=10s elapsed=0s host=foo",
10268
},
10369
{
104-
name: "shutdown initiated, loopback=true",
70+
name: "client opted in, shutdown initiated, signaled at is nil",
71+
optIn: true,
72+
delayDuration: 10 * time.Second,
10573
shutdownInitiated: func() lifecycleEvent {
106-
return fakeLifecycleSignal{ch: newClosedChannel(), at: &signaledAt}
74+
return fakeLifecycleSignal{ch: newClosedChannel(), at: nil}
10775
},
108-
user: &authenticationuser.DefaultInfo{Name: authenticationuser.Anonymous},
109-
remoteAddr: "127.0.0.1:80",
110-
handlerInvoked: 1,
111-
statusCodeExpected: http.StatusOK,
112-
annotationShouldContain: "loopback=true",
113-
},
114-
{
115-
name: "shutdown initiated, loopback=false",
116-
shutdownInitiated: func() lifecycleEvent {
117-
return fakeLifecycleSignal{ch: newClosedChannel(), at: &signaledAt}
118-
},
119-
user: &authenticationuser.DefaultInfo{Name: authenticationuser.Anonymous},
120-
remoteAddr: "www.foo.bar:80",
121-
handlerInvoked: 1,
122-
statusCodeExpected: http.StatusOK,
123-
annotationShouldContain: "loopback=false",
76+
handlerInvoked: 1,
77+
statusCodeExpected: http.StatusOK,
78+
responseHeader: "shutdown=true shutdown-delay-duration=10s elapsed=0s host=foo",
12479
},
12580
{
126-
name: "shutdown initiated, shutdown delay duration is zero",
81+
name: "client opted in, shutdown initiated, signaled at is nil",
82+
optIn: true,
83+
delayDuration: 10 * time.Second,
12784
shutdownInitiated: func() lifecycleEvent {
128-
return fakeLifecycleSignal{ch: newClosedChannel(), at: &signaledAt}
129-
},
130-
delayDuration: time.Duration(0),
131-
clock: func() utilsclock.PassiveClock {
132-
return clocktesting.NewFakeClock(elapsedAtWithingThreshold)
85+
return fakeLifecycleSignal{ch: newClosedChannel(), at: nil}
13386
},
134-
user: &authenticationuser.DefaultInfo{Name: authenticationuser.Anonymous},
135-
handlerInvoked: 1,
136-
statusCodeExpected: http.StatusOK,
137-
annotationShouldContain: "elapsed=1m19s threshold= late=true",
87+
handlerInvoked: 1,
88+
statusCodeExpected: http.StatusOK,
89+
responseHeader: "shutdown=true shutdown-delay-duration=10s elapsed=0s host=foo",
13890
},
13991
{
140-
name: "shutdown initiated, within 80%",
92+
name: "client opted in, shutdown delay duration is zero",
93+
optIn: true,
94+
delayDuration: 0,
14195
shutdownInitiated: func() lifecycleEvent {
14296
return fakeLifecycleSignal{ch: newClosedChannel(), at: &signaledAt}
14397
},
144-
delayDuration: shutdownDelayDuration,
14598
clock: func() utilsclock.PassiveClock {
146-
return clocktesting.NewFakeClock(elapsedAtWithingThreshold)
99+
return clocktesting.NewFakeClock(elapsedAt)
147100
},
148-
user: &authenticationuser.DefaultInfo{Name: authenticationuser.Anonymous},
149-
handlerInvoked: 1,
150-
statusCodeExpected: http.StatusOK,
151-
annotationShouldContain: "elapsed=1m19s threshold=79.00% late=false self=false loopback=false",
101+
handlerInvoked: 1,
102+
statusCodeExpected: http.StatusOK,
103+
responseHeader: "shutdown=true shutdown-delay-duration=0s elapsed=20s host=foo",
152104
},
153105
{
154-
name: "shutdown initiated, outside 80%",
106+
name: "client opted in, shutdown initiated, signaled at is valied",
107+
optIn: true,
108+
delayDuration: 10 * time.Second,
155109
shutdownInitiated: func() lifecycleEvent {
156110
return fakeLifecycleSignal{ch: newClosedChannel(), at: &signaledAt}
157111
},
158-
delayDuration: shutdownDelayDuration,
159112
clock: func() utilsclock.PassiveClock {
160-
return clocktesting.NewFakeClock(elapsedAtBeyondThreshold)
113+
return clocktesting.NewFakeClock(elapsedAt)
161114
},
162-
user: &authenticationuser.DefaultInfo{Name: authenticationuser.Anonymous},
163-
handlerInvoked: 1,
164-
statusCodeExpected: http.StatusOK,
165-
annotationShouldContain: "elapsed=1m21s threshold=81.00% late=true self=false loopback=false",
115+
handlerInvoked: 1,
116+
statusCodeExpected: http.StatusOK,
117+
responseHeader: "shutdown=true shutdown-delay-duration=10s elapsed=20s host=foo",
166118
},
167119
}
168120

@@ -179,33 +131,14 @@ func TestWithShutdownLateAnnotation(t *testing.T) {
179131
if test.clock != nil {
180132
clock = test.clock()
181133
}
182-
target := withShutdownLateAnnotation(handler, event, test.delayDuration, exemptIfHealthProbe, clock)
134+
target := withShutdownResponseHeader(handler, event, test.delayDuration, "foo", clock)
183135

184-
url := "/api/v1/namespaces"
185-
if test.url != "" {
186-
url = test.url
187-
}
188-
req, err := http.NewRequest(http.MethodGet, url, nil)
136+
req, err := http.NewRequest(http.MethodGet, "/api/v1/namespaces", nil)
189137
if err != nil {
190138
t.Fatalf("failed to create new http request - %v", err)
191139
}
192-
if test.remoteAddr != "" {
193-
req.RemoteAddr = test.remoteAddr
194-
}
195-
196-
ctx := req.Context()
197-
if test.user != nil {
198-
ctx = apirequest.WithUser(ctx, test.user)
199-
}
200-
ctx = audit.WithAuditContext(ctx)
201-
req = req.WithContext(ctx)
202-
203-
ac := audit.AuditContextFrom(req.Context())
204-
if ac == nil {
205-
t.Fatalf("expected audit context inside the request context")
206-
}
207-
ac.Event = &auditinternal.Event{
208-
Level: auditinternal.LevelMetadata,
140+
if test.optIn {
141+
req.Header.Set("X-Openshift-If-Disruption", "true")
209142
}
210143

211144
w := httptest.NewRecorder()
@@ -219,19 +152,16 @@ func TestWithShutdownLateAnnotation(t *testing.T) {
219152
t.Errorf("expected status code: %d, but got: %d", test.statusCodeExpected, w.Result().StatusCode)
220153
}
221154

222-
key := "apiserver.k8s.io/shutdown"
155+
key := "X-OpenShift-Disruption"
223156
switch {
224-
case len(test.annotationShouldContain) == 0:
225-
if valueGot, ok := ac.Event.Annotations[key]; ok {
226-
t.Errorf("did not expect annotation to be added, but got: %s", valueGot)
157+
case len(test.responseHeader) == 0:
158+
if valueGot := w.Header().Get(key); len(valueGot) > 0 {
159+
t.Errorf("did not expect header to be added to the response, but got: %s", valueGot)
227160
}
228161
default:
229-
if valueGot, ok := ac.Event.Annotations[key]; !ok || !strings.Contains(valueGot, test.annotationShouldContain) {
162+
if valueGot := w.Header().Get(key); len(valueGot) == 0 || test.responseHeader != valueGot {
230163
t.Logf("got: %s", valueGot)
231-
t.Errorf("expected annotation to match, diff: %s", cmp.Diff(test.annotationShouldContain, valueGot))
232-
}
233-
if header := w.Header().Get("X-OpenShift-Shutdown"); !strings.Contains(header, test.annotationShouldContain) {
234-
t.Errorf("expected response header to match, diff: %s", cmp.Diff(test.annotationShouldContain, header))
164+
t.Errorf("expected response header to match, diff: %s", cmp.Diff(test.responseHeader, valueGot))
235165
}
236166
}
237167
})

0 commit comments

Comments
 (0)