Skip to content

Commit 06a81d1

Browse files
sohankunkerkaryuqi-zhang
sohankunkerkar
and
yuqi-zhang
committed
cmd/kubelet: implement drop-in configuration directory for kubelet
This implements a drop-in configuration directory for the kubelet by introducing a "--config-dir" flag. Users can provide individual kubelet config snippets in separate files, formatted similarly to kubelet.conf. The kubelet will process the files in alphanumeric order, appending configurations if subfield(s) doesn't exist, overwriting them if they do, and handling lists by overwriting instead of merging. Co-authored-by: Yu Qi Zhang <[email protected]>
1 parent a9b3ca3 commit 06a81d1

File tree

4 files changed

+257
-4
lines changed

4 files changed

+257
-4
lines changed

‎cmd/kubelet/app/options/options.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ type KubeletFlags struct {
8686
// Omit this flag to use the combination of built-in default configuration values and flags.
8787
KubeletConfigFile string
8888

89+
// kubeletDropinConfigDirectory is a path to a directory to specify dropins allows the user to optionally specify
90+
// additional configs to overwrite what is provided by default and in the KubeletConfigFile flag
91+
KubeletDropinConfigDirectory string
92+
8993
// WindowsService should be set to true if kubelet is running as a service on Windows.
9094
// Its corresponding flag only gets registered in Windows builds.
9195
WindowsService bool
@@ -281,6 +285,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
281285
f.addOSFlags(fs)
282286

283287
fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.")
288+
fs.StringVar(&f.KubeletDropinConfigDirectory, "config-dir", "", "Path to a directory to specify drop-ins, allows the user to optionally specify additional configs to overwrite what is provided by default and in the KubeletConfigFile flag. Note: Set the 'KUBELET_CONFIG_DROPIN_DIR_ALPHA' environment variable to specify the directory. [default='']")
284289
fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.")
285290

286291
fs.StringVar(&f.BootstrapKubeconfig, "bootstrap-kubeconfig", f.BootstrapKubeconfig, "Path to a kubeconfig file that will be used to get client certificate for kubelet. "+

‎cmd/kubelet/app/server.go‎

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"errors"
2424
"fmt"
2525
"io"
26+
"io/fs"
2627
"math"
2728
"net"
2829
"net/http"
@@ -33,6 +34,7 @@ import (
3334
"time"
3435

3536
"github.com/coreos/go-systemd/v22/daemon"
37+
"github.com/imdario/mergo"
3638
"github.com/spf13/cobra"
3739
"github.com/spf13/pflag"
3840
"google.golang.org/grpc/codes"
@@ -202,11 +204,24 @@ is checked every 20 seconds (also configurable with a flag).`,
202204
}
203205

204206
// load kubelet config file, if provided
205-
if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 {
206-
kubeletConfig, err = loadConfigFile(configFile)
207+
if len(kubeletFlags.KubeletConfigFile) > 0 {
208+
kubeletConfig, err = loadConfigFile(kubeletFlags.KubeletConfigFile)
207209
if err != nil {
208-
return fmt.Errorf("failed to load kubelet config file, error: %w, path: %s", err, configFile)
210+
return fmt.Errorf("failed to load kubelet config file, path: %s, error: %w", kubeletFlags.KubeletConfigFile, err)
209211
}
212+
}
213+
// Merge the kubelet configurations if --config-dir is set
214+
if len(kubeletFlags.KubeletDropinConfigDirectory) > 0 {
215+
_, ok := os.LookupEnv("KUBELET_CONFIG_DROPIN_DIR_ALPHA")
216+
if !ok {
217+
return fmt.Errorf("flag %s specified but environment variable KUBELET_CONFIG_DROPIN_DIR_ALPHA not set, cannot start kubelet", kubeletFlags.KubeletDropinConfigDirectory)
218+
}
219+
if err := mergeKubeletConfigurations(kubeletConfig, kubeletFlags.KubeletDropinConfigDirectory); err != nil {
220+
return fmt.Errorf("failed to merge kubelet configs: %w", err)
221+
}
222+
}
223+
224+
if len(kubeletFlags.KubeletConfigFile) > 0 || len(kubeletFlags.KubeletDropinConfigDirectory) > 0 {
210225
// We must enforce flag precedence by re-parsing the command line into the new object.
211226
// This is necessary to preserve backwards-compatibility across binary upgrades.
212227
// See issue #56171 for more details.
@@ -288,6 +303,41 @@ is checked every 20 seconds (also configurable with a flag).`,
288303
return cmd
289304
}
290305

306+
// mergeKubeletConfigurations merges the provided drop-in configurations with the base kubelet configuration.
307+
// The drop-in configurations are processed in lexical order based on the file names. This means that the
308+
// configurations in files with lower numeric prefixes are applied first, followed by higher numeric prefixes.
309+
// For example, if the drop-in directory contains files named "10-config.conf" and "20-config.conf",
310+
// the configurations in "10-config.conf" will be applied first, and then the configurations in "20-config.conf" will be applied,
311+
// potentially overriding the previous values.
312+
func mergeKubeletConfigurations(kubeletConfig *kubeletconfiginternal.KubeletConfiguration, kubeletDropInConfigDir string) error {
313+
const dropinFileExtension = ".conf"
314+
315+
// Walk through the drop-in directory and update the configuration for each file
316+
err := filepath.WalkDir(kubeletDropInConfigDir, func(path string, info fs.DirEntry, err error) error {
317+
if err != nil {
318+
return err
319+
}
320+
if !info.IsDir() && filepath.Ext(info.Name()) == dropinFileExtension {
321+
dropinConfig, err := loadConfigFile(path)
322+
if err != nil {
323+
return fmt.Errorf("failed to load kubelet dropin file, path: %s, error: %w", path, err)
324+
}
325+
326+
// Merge dropinConfig with kubeletConfig
327+
if err := mergo.Merge(kubeletConfig, dropinConfig, mergo.WithOverride); err != nil {
328+
return fmt.Errorf("failed to merge kubelet drop-in config, path: %s, error: %w", path, err)
329+
}
330+
}
331+
return nil
332+
})
333+
334+
if err != nil {
335+
return fmt.Errorf("failed to walk through kubelet dropin directory %q: %w", kubeletDropInConfigDir, err)
336+
}
337+
338+
return nil
339+
}
340+
291341
// newFlagSetWithGlobals constructs a new pflag.FlagSet with global flags registered
292342
// on it.
293343
func newFlagSetWithGlobals() *pflag.FlagSet {

‎cmd/kubelet/app/server_test.go‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,14 @@ limitations under the License.
1717
package app
1818

1919
import (
20+
"os"
21+
"path/filepath"
22+
"reflect"
2023
"testing"
24+
25+
"github.com/stretchr/testify/require"
26+
"k8s.io/kubernetes/cmd/kubelet/app/options"
27+
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
2128
)
2229

2330
func TestValueOfAllocatableResources(t *testing.T) {
@@ -61,3 +68,194 @@ func TestValueOfAllocatableResources(t *testing.T) {
6168
}
6269
}
6370
}
71+
72+
func TestMergeKubeletConfigurations(t *testing.T) {
73+
testCases := []struct {
74+
kubeletConfig string
75+
dropin1 string
76+
dropin2 string
77+
overwrittenConfigFields map[string]interface{}
78+
cliArgs []string
79+
name string
80+
}{
81+
{
82+
kubeletConfig: `
83+
apiVersion: kubelet.config.k8s.io/v1beta1
84+
kind: KubeletConfiguration
85+
port: 9080
86+
readOnlyPort: 10257
87+
`,
88+
dropin1: `
89+
apiVersion: kubelet.config.k8s.io/v1beta1
90+
kind: KubeletConfiguration
91+
port: 9090
92+
`,
93+
dropin2: `
94+
apiVersion: kubelet.config.k8s.io/v1beta1
95+
kind: KubeletConfiguration
96+
port: 8080
97+
readOnlyPort: 10255
98+
`,
99+
overwrittenConfigFields: map[string]interface{}{
100+
"Port": int32(8080),
101+
"ReadOnlyPort": int32(10255),
102+
},
103+
name: "kubelet.conf.d overrides kubelet.conf",
104+
},
105+
{
106+
kubeletConfig: `
107+
apiVersion: kubelet.config.k8s.io/v1beta1
108+
kind: KubeletConfiguration
109+
readOnlyPort: 10256
110+
kubeReserved:
111+
memory: 70Mi
112+
`,
113+
dropin1: `
114+
apiVersion: kubelet.config.k8s.io/v1beta1
115+
kind: KubeletConfiguration
116+
readOnlyPort: 10255
117+
kubeReserved:
118+
memory: 150Mi
119+
cpu: 200m
120+
`,
121+
dropin2: `
122+
apiVersion: kubelet.config.k8s.io/v1beta1
123+
kind: KubeletConfiguration
124+
readOnlyPort: 10257
125+
kubeReserved:
126+
memory: 100Mi
127+
`,
128+
overwrittenConfigFields: map[string]interface{}{
129+
"ReadOnlyPort": int32(10257),
130+
"KubeReserved": map[string]string{
131+
"cpu": "200m",
132+
"memory": "100Mi",
133+
},
134+
},
135+
name: "kubelet.conf.d overrides kubelet.conf with subfield override",
136+
},
137+
{
138+
kubeletConfig: `
139+
apiVersion: kubelet.config.k8s.io/v1beta1
140+
kind: KubeletConfiguration
141+
port: 9090
142+
clusterDNS:
143+
- 192.168.1.3
144+
- 192.168.1.4
145+
`,
146+
dropin1: `
147+
apiVersion: kubelet.config.k8s.io/v1beta1
148+
kind: KubeletConfiguration
149+
port: 9090
150+
systemReserved:
151+
memory: 1Gi
152+
`,
153+
dropin2: `
154+
apiVersion: kubelet.config.k8s.io/v1beta1
155+
kind: KubeletConfiguration
156+
port: 8080
157+
readOnlyPort: 10255
158+
systemReserved:
159+
memory: 2Gi
160+
clusterDNS:
161+
- 192.168.1.1
162+
- 192.168.1.5
163+
- 192.168.1.8
164+
`,
165+
overwrittenConfigFields: map[string]interface{}{
166+
"Port": int32(8080),
167+
"ReadOnlyPort": int32(10255),
168+
"SystemReserved": map[string]string{
169+
"memory": "2Gi",
170+
},
171+
"ClusterDNS": []string{"192.168.1.1", "192.168.1.5", "192.168.1.8"},
172+
},
173+
name: "kubelet.conf.d overrides kubelet.conf with slices/lists",
174+
},
175+
{
176+
dropin1: `
177+
apiVersion: kubelet.config.k8s.io/v1beta1
178+
kind: KubeletConfiguration
179+
port: 9090
180+
`,
181+
dropin2: `
182+
apiVersion: kubelet.config.k8s.io/v1beta1
183+
kind: KubeletConfiguration
184+
port: 8080
185+
readOnlyPort: 10255
186+
`,
187+
overwrittenConfigFields: map[string]interface{}{
188+
"Port": int32(8081),
189+
"ReadOnlyPort": int32(10256),
190+
},
191+
cliArgs: []string{
192+
"--port=8081",
193+
"--read-only-port=10256",
194+
},
195+
name: "cli args override kubelet.conf.d",
196+
},
197+
{
198+
kubeletConfig: `
199+
apiVersion: kubelet.config.k8s.io/v1beta1
200+
kind: KubeletConfiguration
201+
port: 9090
202+
clusterDNS:
203+
- 192.168.1.3
204+
`,
205+
overwrittenConfigFields: map[string]interface{}{
206+
"Port": int32(9090),
207+
"ClusterDNS": []string{"192.168.1.2"},
208+
},
209+
cliArgs: []string{
210+
"--port=9090",
211+
"--cluster-dns=192.168.1.2",
212+
},
213+
name: "cli args override kubelet.conf",
214+
},
215+
}
216+
217+
for _, test := range testCases {
218+
t.Run(test.name, func(t *testing.T) {
219+
// Prepare a temporary directory for testing
220+
tempDir := t.TempDir()
221+
222+
kubeletConfig := &kubeletconfiginternal.KubeletConfiguration{}
223+
kubeletFlags := &options.KubeletFlags{}
224+
225+
if len(test.kubeletConfig) > 0 {
226+
// Create the Kubeletconfig
227+
kubeletConfFile := filepath.Join(tempDir, "kubelet.conf")
228+
err := os.WriteFile(kubeletConfFile, []byte(test.kubeletConfig), 0644)
229+
require.NoError(t, err, "failed to create config from a yaml file")
230+
kubeletFlags.KubeletConfigFile = kubeletConfFile
231+
}
232+
if len(test.dropin1) > 0 || len(test.dropin2) > 0 {
233+
// Create kubelet.conf.d directory and drop-in configuration files
234+
kubeletConfDir := filepath.Join(tempDir, "kubelet.conf.d")
235+
err := os.Mkdir(kubeletConfDir, 0755)
236+
require.NoError(t, err, "Failed to create kubelet.conf.d directory")
237+
238+
err = os.WriteFile(filepath.Join(kubeletConfDir, "10-kubelet.conf"), []byte(test.dropin1), 0644)
239+
require.NoError(t, err, "failed to create config from a yaml file")
240+
241+
err = os.WriteFile(filepath.Join(kubeletConfDir, "20-kubelet.conf"), []byte(test.dropin2), 0644)
242+
require.NoError(t, err, "failed to create config from a yaml file")
243+
244+
// Merge the kubelet configurations
245+
err = mergeKubeletConfigurations(kubeletConfig, kubeletConfDir)
246+
require.NoError(t, err, "failed to merge kubelet drop-in configs")
247+
}
248+
249+
// Use kubelet config flag precedence
250+
err := kubeletConfigFlagPrecedence(kubeletConfig, test.cliArgs)
251+
require.NoError(t, err, "failed to set the kubelet config flag precedence")
252+
253+
// Verify the merged configuration fields
254+
for fieldName, expectedValue := range test.overwrittenConfigFields {
255+
value := reflect.ValueOf(kubeletConfig).Elem()
256+
field := value.FieldByName(fieldName)
257+
require.Equal(t, expectedValue, field.Interface(), "Field mismatch: "+fieldName)
258+
}
259+
})
260+
}
261+
}

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ require (
4343
github.com/google/go-cmp v0.5.9
4444
github.com/google/gofuzz v1.2.0
4545
github.com/google/uuid v1.3.0
46+
github.com/imdario/mergo v0.3.6
4647
github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5
4748
github.com/libopenstorage/openstorage v1.0.0
4849
github.com/lithammer/dedent v1.1.0
@@ -183,7 +184,6 @@ require (
183184
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
184185
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
185186
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
186-
github.com/imdario/mergo v0.3.6 // indirect
187187
github.com/inconshreveable/mousetrap v1.1.0 // indirect
188188
github.com/jonboulle/clockwork v0.2.2 // indirect
189189
github.com/josharian/intern v1.0.0 // indirect

0 commit comments

Comments
 (0)