Skip to content

Commit abfd6a7

Browse files
tkashematiratree
tkashem
authored and
atiratree
committed
UPSTREAM: <carry>: optionally enable retry after until apiserver is ready
OpenShift-Rebase-Source: fc3523f
1 parent c4dd26e commit abfd6a7

File tree

4 files changed

+272
-0
lines changed

4 files changed

+272
-0
lines changed

‎staging/src/k8s.io/apiserver/pkg/server/config.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,18 @@ type Config struct {
319319
// This grace period is orthogonal to other grace periods, and
320320
// it is not overridden by any other grace period.
321321
ShutdownWatchTerminationGracePeriod time.Duration
322+
323+
// SendRetryAfterWhileNotReadyOnce, if enabled, the apiserver will
324+
// reject all incoming requests with a 503 status code and a
325+
// 'Retry-After' response header until the apiserver has fully
326+
// initialized, except for requests from a designated debugger group.
327+
// This option ensures that the system stays consistent even when
328+
// requests are received before the server has been initialized.
329+
// In particular, it prevents child deletion in case of GC or/and
330+
// orphaned content in case of the namespaces controller.
331+
// NOTE: this option is applicable to Microshift only,
332+
// this should never be enabled for OCP.
333+
SendRetryAfterWhileNotReadyOnce bool
322334
}
323335

324336
type RecommendedConfig struct {
@@ -1055,6 +1067,10 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
10551067
handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
10561068
}
10571069

1070+
if c.SendRetryAfterWhileNotReadyOnce {
1071+
handler = genericfilters.WithNotReady(handler, c.lifecycleSignals.HasBeenReady.Signaled())
1072+
}
1073+
10581074
handler = filterlatency.TrackCompleted(handler)
10591075
handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
10601076
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "impersonation")

‎staging/src/k8s.io/apiserver/pkg/server/filters/with_not_ready_patch.go‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package filters
18+
19+
import (
20+
"errors"
21+
"k8s.io/apiserver/pkg/warning"
22+
"net/http"
23+
24+
"k8s.io/apiserver/pkg/authentication/user"
25+
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
26+
"k8s.io/apiserver/pkg/endpoints/request"
27+
)
28+
29+
const (
30+
// notReadyDebuggerGroup facilitates debugging if the apiserver takes longer
31+
// to initilize. All request(s) from this designated group will be allowed
32+
// while the apiserver is being initialized.
33+
// The apiserver will reject all incoming requests with a 'Retry-After'
34+
// response header until it has fully initialized, except for
35+
// requests from this special debugger group.
36+
notReadyDebuggerGroup = "system:openshift:risky-not-ready-microshift-debugging-group"
37+
)
38+
39+
// WithNotReady rejects any incoming new request(s) with a 'Retry-After'
40+
// response if the specified hasBeenReadyCh channel is still open, with
41+
// the following exceptions:
42+
// - all request(s) from the designated debugger group is exempt, this
43+
// helps debug the apiserver if it takes longer to initialize.
44+
// - local loopback requests (this exempts system:apiserver)
45+
// - /healthz, /livez, /readyz, /metrics are exempt
46+
//
47+
// It includes new request(s) on a new or an existing TCP connection
48+
// Any new request(s) arriving before hasBeenreadyCh is closed
49+
// are replied with a 503 and the following response headers:
50+
// - 'Retry-After: N` (so client can retry after N seconds)
51+
func WithNotReady(handler http.Handler, hasBeenReadyCh <-chan struct{}) http.Handler {
52+
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
53+
select {
54+
case <-hasBeenReadyCh:
55+
handler.ServeHTTP(w, req)
56+
return
57+
default:
58+
}
59+
60+
requestor, exists := request.UserFrom(req.Context())
61+
if !exists {
62+
responsewriters.InternalError(w, req, errors.New("no user found for request"))
63+
return
64+
}
65+
66+
// make sure we exempt:
67+
// - local loopback requests (this exempts system:apiserver)
68+
// - health probes and metric scraping
69+
// - requests from the exempt debugger group.
70+
if requestor.GetName() == user.APIServerUser ||
71+
hasExemptPathPrefix(req) ||
72+
matchesDebuggerGroup(requestor, notReadyDebuggerGroup) {
73+
warning.AddWarning(req.Context(), "", "The apiserver was still initializing, while this request was being served")
74+
handler.ServeHTTP(w, req)
75+
return
76+
}
77+
78+
// Return a 503 status asking the client to try again after 5 seconds
79+
w.Header().Set("Retry-After", "5")
80+
http.Error(w, "The apiserver hasn't been fully initialized yet, please try again later.",
81+
http.StatusServiceUnavailable)
82+
})
83+
}
84+
85+
func matchesDebuggerGroup(requestor user.Info, debugger string) bool {
86+
for _, group := range requestor.GetGroups() {
87+
if group == debugger {
88+
return true
89+
}
90+
}
91+
return false
92+
}

‎staging/src/k8s.io/apiserver/pkg/server/filters/with_not_ready_patch_test.go‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package filters
2+
3+
import (
4+
"net/http"
5+
"net/http/httptest"
6+
"testing"
7+
8+
"k8s.io/apiserver/pkg/authentication/user"
9+
genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
10+
"k8s.io/apiserver/pkg/endpoints/request"
11+
)
12+
13+
func TestWithNotReady(t *testing.T) {
14+
const warning = `299 - "The apiserver was still initializing, while this request was being served"`
15+
16+
tests := []struct {
17+
name string
18+
requestURL string
19+
hasBeenReady bool
20+
user *user.DefaultInfo
21+
handlerInvoked int
22+
retryAfterExpected string
23+
warningExpected string
24+
statusCodeexpected int
25+
}{
26+
{
27+
name: "the apiserver is fully initialized",
28+
hasBeenReady: true,
29+
handlerInvoked: 1,
30+
statusCodeexpected: http.StatusOK,
31+
},
32+
{
33+
name: "the apiserver is initializing, local loopback",
34+
hasBeenReady: false,
35+
user: &user.DefaultInfo{Name: user.APIServerUser},
36+
handlerInvoked: 1,
37+
statusCodeexpected: http.StatusOK,
38+
warningExpected: warning,
39+
},
40+
{
41+
name: "the apiserver is initializing, exempt debugger group",
42+
hasBeenReady: false,
43+
user: &user.DefaultInfo{Groups: []string{"system:authenticated", notReadyDebuggerGroup}},
44+
handlerInvoked: 1,
45+
statusCodeexpected: http.StatusOK,
46+
warningExpected: warning,
47+
},
48+
{
49+
name: "the apiserver is initializing, readyz",
50+
requestURL: "/readyz?verbose=1",
51+
user: &user.DefaultInfo{},
52+
hasBeenReady: false,
53+
handlerInvoked: 1,
54+
statusCodeexpected: http.StatusOK,
55+
warningExpected: warning,
56+
},
57+
{
58+
name: "the apiserver is initializing, healthz",
59+
requestURL: "/healthz?verbose=1",
60+
user: &user.DefaultInfo{},
61+
hasBeenReady: false,
62+
handlerInvoked: 1,
63+
statusCodeexpected: http.StatusOK,
64+
warningExpected: warning,
65+
},
66+
{
67+
name: "the apiserver is initializing, livez",
68+
requestURL: "/livez?verbose=1",
69+
user: &user.DefaultInfo{},
70+
hasBeenReady: false,
71+
handlerInvoked: 1,
72+
statusCodeexpected: http.StatusOK,
73+
warningExpected: warning,
74+
},
75+
{
76+
name: "the apiserver is initializing, metrics",
77+
requestURL: "/metrics",
78+
user: &user.DefaultInfo{},
79+
hasBeenReady: false,
80+
handlerInvoked: 1,
81+
statusCodeexpected: http.StatusOK,
82+
warningExpected: warning,
83+
},
84+
{
85+
name: "the apiserver is initializing, non-exempt request",
86+
hasBeenReady: false,
87+
user: &user.DefaultInfo{Groups: []string{"system:authenticated", "system:masters"}},
88+
statusCodeexpected: http.StatusServiceUnavailable,
89+
retryAfterExpected: "5",
90+
},
91+
}
92+
93+
for _, test := range tests {
94+
t.Run(test.name, func(t *testing.T) {
95+
hasBeenReadyCh := make(chan struct{})
96+
if test.hasBeenReady {
97+
close(hasBeenReadyCh)
98+
} else {
99+
defer close(hasBeenReadyCh)
100+
}
101+
102+
var handlerInvoked int
103+
handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
104+
handlerInvoked++
105+
w.WriteHeader(http.StatusOK)
106+
})
107+
108+
if len(test.requestURL) == 0 {
109+
test.requestURL = "/api/v1/namespaces"
110+
}
111+
req, err := http.NewRequest(http.MethodGet, test.requestURL, nil)
112+
if err != nil {
113+
t.Fatalf("failed to create new http request - %v", err)
114+
}
115+
if test.user != nil {
116+
req = req.WithContext(request.WithUser(req.Context(), test.user))
117+
}
118+
w := httptest.NewRecorder()
119+
120+
withNotReady := WithNotReady(handler, hasBeenReadyCh)
121+
withNotReady = genericapifilters.WithWarningRecorder(withNotReady)
122+
withNotReady.ServeHTTP(w, req)
123+
124+
if test.handlerInvoked != handlerInvoked {
125+
t.Errorf("expected the handler to be invoked: %d times, but got: %d", test.handlerInvoked, handlerInvoked)
126+
}
127+
if test.statusCodeexpected != w.Code {
128+
t.Errorf("expected Response Status Code: %d, but got: %d", test.statusCodeexpected, w.Code)
129+
}
130+
131+
retryAfterGot := w.Header().Get("Retry-After")
132+
if test.retryAfterExpected != retryAfterGot {
133+
t.Errorf("expected Retry-After: %q, but got: %q", test.retryAfterExpected, retryAfterGot)
134+
}
135+
136+
warningGot := w.Header().Get("Warning")
137+
if test.warningExpected != warningGot {
138+
t.Errorf("expected Warning: %s, but got: %s", test.warningExpected, warningGot)
139+
}
140+
141+
})
142+
}
143+
}

‎staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,18 @@ type ServerRunOptions struct {
9797
ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry
9898
// ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry.
9999
ComponentName string
100+
101+
// SendRetryAfterWhileNotReadyOnce, if enabled, the apiserver will
102+
// reject all incoming requests with a 503 status code and a
103+
// 'Retry-After' response header until the apiserver has fully
104+
// initialized, except for requests from a designated debugger group.
105+
// This option ensures that the system stays consistent even when
106+
// requests are received before the server has been initialized.
107+
// In particular, it prevents child deletion in case of GC or/and
108+
// orphaned content in case of the namespaces controller.
109+
// NOTE: this option is applicable to Microshift only,
110+
// this should never be enabled for OCP.
111+
SendRetryAfterWhileNotReadyOnce bool
100112
}
101113

102114
func NewServerRunOptions() *ServerRunOptions {
@@ -125,6 +137,7 @@ func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegis
125137
ShutdownSendRetryAfter: false,
126138
ComponentName: componentName,
127139
ComponentGlobalsRegistry: componentGlobalsRegistry,
140+
SendRetryAfterWhileNotReadyOnce: false,
128141
}
129142
}
130143

@@ -151,6 +164,7 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
151164
c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
152165
c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName)
153166
c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName)
167+
c.SendRetryAfterWhileNotReadyOnce = s.SendRetryAfterWhileNotReadyOnce
154168

155169
return nil
156170
}
@@ -374,6 +388,13 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
374388
"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
375389
"for active watch request(s) to drain during the graceful server shutdown window.")
376390

391+
// NOTE: this option is applicable to Microshift only, this should never be enabled for OCP.
392+
fs.BoolVar(&s.SendRetryAfterWhileNotReadyOnce, "send-retry-after-while-not-ready-once", s.SendRetryAfterWhileNotReadyOnce, ""+
393+
"If true, incoming request(s) will be rejected with a '503' status code and a 'Retry-After' response header "+
394+
"until the apiserver has initialized, except for requests from a certain group. This option ensures that the system stays "+
395+
"consistent even when requests arrive at the server before it has been initialized. "+
396+
"This option is applicable to Microshift only, this should never be enabled for OCP")
397+
377398
s.ComponentGlobalsRegistry.AddFlags(fs)
378399
}
379400

0 commit comments

Comments
 (0)