Skip to content

Commit a2106b5

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#126407 from Jefftree/fake-clock
Fix unit flake in leaderelection/TestReconcileElectionStep
2 parents 2aa468c + f173f0c commit a2106b5

File tree

5 files changed

+55
-42
lines changed

5 files changed

+55
-42
lines changed

‎pkg/controlplane/controller/leaderelection/election.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
v1 "k8s.io/api/coordination/v1"
2525
v1alpha1 "k8s.io/api/coordination/v1alpha1"
2626
"k8s.io/klog/v2"
27+
"k8s.io/utils/clock"
2728
)
2829

2930
func pickBestLeaderOldestEmulationVersion(candidates []*v1alpha1.LeaseCandidate) *v1alpha1.LeaseCandidate {
@@ -155,15 +156,15 @@ func compare(lhs, rhs *v1alpha1.LeaseCandidate) int {
155156
return result
156157
}
157158

158-
func isLeaseExpired(lease *v1.Lease) bool {
159-
currentTime := time.Now()
159+
func isLeaseExpired(clock clock.Clock, lease *v1.Lease) bool {
160+
currentTime := clock.Now()
160161
return lease.Spec.RenewTime == nil ||
161162
lease.Spec.LeaseDurationSeconds == nil ||
162163
lease.Spec.RenewTime.Add(time.Duration(*lease.Spec.LeaseDurationSeconds)*time.Second).Before(currentTime)
163164
}
164165

165-
func isLeaseCandidateExpired(lease *v1alpha1.LeaseCandidate) bool {
166-
currentTime := time.Now()
166+
func isLeaseCandidateExpired(clock clock.Clock, lease *v1alpha1.LeaseCandidate) bool {
167+
currentTime := clock.Now()
167168
return lease.Spec.RenewTime == nil ||
168169
lease.Spec.RenewTime.Add(leaseCandidateValidDuration).Before(currentTime)
169170
}

‎pkg/controlplane/controller/leaderelection/leaderelection_controller.go‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import (
3737
"k8s.io/client-go/tools/cache"
3838
"k8s.io/client-go/util/workqueue"
3939
"k8s.io/klog/v2"
40+
"k8s.io/utils/clock"
4041
"k8s.io/utils/ptr"
4142
)
4243

@@ -68,6 +69,8 @@ type Controller struct {
6869
leaseCandidateRegistration cache.ResourceEventHandlerRegistration
6970

7071
queue workqueue.TypedRateLimitingInterface[types.NamespacedName]
72+
73+
clock clock.Clock
7174
}
7275

7376
func (c *Controller) Run(ctx context.Context, workers int) {
@@ -114,6 +117,8 @@ func NewController(leaseInformer coordinationv1informers.LeaseInformer, leaseCan
114117
leaseCandidateClient: leaseCandidateClient,
115118

116119
queue: workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[types.NamespacedName](), workqueue.TypedRateLimitingQueueConfig[types.NamespacedName]{Name: controllerName}),
120+
121+
clock: clock.RealClock{},
117122
}
118123
leaseSynced, err := leaseInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
119124
AddFunc: func(obj interface{}) {
@@ -199,13 +204,13 @@ func (c *Controller) electionNeeded(candidates []*v1alpha1.LeaseCandidate, lease
199204
return true, nil
200205
}
201206

202-
if isLeaseExpired(lease) || lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity == "" {
207+
if isLeaseExpired(c.clock, lease) || lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity == "" {
203208
return true, nil
204209
}
205210

206211
// every 15min enforce an election to update all candidates. Every 30min we garbage collect.
207212
for _, candidate := range candidates {
208-
if candidate.Spec.RenewTime != nil && candidate.Spec.RenewTime.Add(leaseCandidateValidDuration/2).Before(time.Now()) {
213+
if candidate.Spec.RenewTime != nil && candidate.Spec.RenewTime.Add(leaseCandidateValidDuration/2).Before(c.clock.Now()) {
209214
return true, nil
210215
}
211216
}
@@ -258,7 +263,7 @@ func (c *Controller) reconcileElectionStep(ctx context.Context, leaseNN types.Na
258263
return defaultRequeueInterval, err
259264
}
260265

261-
now := time.Now()
266+
now := c.clock.Now()
262267
canVoteYet := true
263268
for _, candidate := range candidates {
264269
if candidate.Spec.PingTime != nil && candidate.Spec.PingTime.Add(electionDuration).After(now) &&
@@ -331,7 +336,7 @@ func (c *Controller) reconcileElectionStep(ctx context.Context, leaseNN types.Na
331336
Spec: v1.LeaseSpec{
332337
Strategy: &strategy,
333338
LeaseDurationSeconds: ptr.To(defaultLeaseDurationSeconds),
334-
RenewTime: &metav1.MicroTime{Time: time.Now()},
339+
RenewTime: &metav1.MicroTime{Time: c.clock.Now()},
335340
},
336341
}
337342

@@ -368,7 +373,7 @@ func (c *Controller) reconcileElectionStep(ctx context.Context, leaseNN types.Na
368373
}
369374
orig := existing.DeepCopy()
370375

371-
isExpired := isLeaseExpired(existing)
376+
isExpired := isLeaseExpired(c.clock, existing)
372377
noHolderIdentity := leaderLease.Spec.HolderIdentity != nil && existing.Spec.HolderIdentity == nil || *existing.Spec.HolderIdentity == ""
373378
expiredAndNewHolder := isExpired && leaderLease.Spec.HolderIdentity != nil && *existing.Spec.HolderIdentity != *leaderLease.Spec.HolderIdentity
374379
strategyChanged := existing.Spec.Strategy == nil || *existing.Spec.Strategy != strategy
@@ -420,7 +425,7 @@ func (c *Controller) listAdmissableCandidates(leaseNN types.NamespacedName) ([]*
420425
if l.Spec.LeaseName != leaseNN.Name {
421426
continue
422427
}
423-
if !isLeaseCandidateExpired(l) {
428+
if !isLeaseCandidateExpired(c.clock, l) {
424429
results = append(results, l)
425430
} else {
426431
klog.Infof("LeaseCandidate %s is expired", l.Name)

‎pkg/controlplane/controller/leaderelection/leaderelection_controller_test.go‎

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,13 @@ import (
3131
"k8s.io/apimachinery/pkg/util/wait"
3232
"k8s.io/client-go/informers"
3333
"k8s.io/client-go/kubernetes/fake"
34+
testingclock "k8s.io/utils/clock/testing"
3435
"k8s.io/utils/ptr"
3536
)
3637

3738
func TestReconcileElectionStep(t *testing.T) {
39+
fakeClock := testingclock.NewFakeClock(time.Now())
40+
3841
tests := []struct {
3942
name string
4043
leaseNN types.NamespacedName
@@ -83,7 +86,7 @@ func TestReconcileElectionStep(t *testing.T) {
8386
LeaseName: "component-A",
8487
EmulationVersion: "1.19.0",
8588
BinaryVersion: "1.19.0",
86-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
89+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
8790
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
8891
},
8992
},
@@ -108,7 +111,7 @@ func TestReconcileElectionStep(t *testing.T) {
108111
LeaseName: "component-A",
109112
EmulationVersion: "1.19.0",
110113
BinaryVersion: "1.19.0",
111-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
114+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
112115
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
113116
},
114117
},
@@ -121,7 +124,7 @@ func TestReconcileElectionStep(t *testing.T) {
121124
LeaseName: "component-A",
122125
EmulationVersion: "1.18.0",
123126
BinaryVersion: "1.18.0",
124-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
127+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
125128
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
126129
},
127130
},
@@ -134,7 +137,7 @@ func TestReconcileElectionStep(t *testing.T) {
134137
Spec: v1.LeaseSpec{
135138
HolderIdentity: ptr.To("component-identity-1"),
136139
LeaseDurationSeconds: ptr.To(int32(10)),
137-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
140+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
138141
},
139142
},
140143
expectLease: true,
@@ -157,8 +160,8 @@ func TestReconcileElectionStep(t *testing.T) {
157160
LeaseName: "component-A",
158161
EmulationVersion: "1.19.0",
159162
BinaryVersion: "1.19.0",
160-
PingTime: ptr.To(metav1.NewMicroTime(time.Now().Add(-2 * electionDuration))),
161-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now().Add(-4 * electionDuration))),
163+
PingTime: ptr.To(metav1.NewMicroTime(fakeClock.Now().Add(-2 * electionDuration))),
164+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now().Add(-4 * electionDuration))),
162165
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
163166
},
164167
},
@@ -171,8 +174,8 @@ func TestReconcileElectionStep(t *testing.T) {
171174
LeaseName: "component-A",
172175
EmulationVersion: "1.20.0",
173176
BinaryVersion: "1.20.0",
174-
PingTime: ptr.To(metav1.NewMicroTime(time.Now())),
175-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
177+
PingTime: ptr.To(metav1.NewMicroTime(fakeClock.Now().Add(-1 * time.Millisecond))),
178+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
176179
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
177180
},
178181
},
@@ -197,7 +200,7 @@ func TestReconcileElectionStep(t *testing.T) {
197200
LeaseName: "component-A",
198201
EmulationVersion: "1.19.0",
199202
BinaryVersion: "1.19.0",
200-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
203+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
201204
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
202205
},
203206
},
@@ -210,7 +213,7 @@ func TestReconcileElectionStep(t *testing.T) {
210213
Spec: v1.LeaseSpec{
211214
HolderIdentity: ptr.To("component-identity-expired"),
212215
LeaseDurationSeconds: ptr.To(int32(10)),
213-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now().Add(-1 * time.Minute))),
216+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now().Add(-1 * time.Minute))),
214217
},
215218
},
216219
expectLease: true,
@@ -232,8 +235,8 @@ func TestReconcileElectionStep(t *testing.T) {
232235
LeaseName: "component-A",
233236
EmulationVersion: "1.19.0",
234237
BinaryVersion: "1.19.0",
235-
PingTime: ptr.To(metav1.NewMicroTime(time.Now().Add(-1 * time.Minute))),
236-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now().Add(-2 * time.Minute))),
238+
PingTime: ptr.To(metav1.NewMicroTime(fakeClock.Now().Add(-1 * time.Minute))),
239+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now().Add(-2 * time.Minute))),
237240
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
238241
},
239242
},
@@ -257,7 +260,7 @@ func TestReconcileElectionStep(t *testing.T) {
257260
LeaseName: "component-A",
258261
EmulationVersion: "1.19.0",
259262
BinaryVersion: "1.19.0",
260-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now().Add(-2 * electionDuration))),
263+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now().Add(-2 * electionDuration))),
261264
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
262265
},
263266
},
@@ -283,8 +286,8 @@ func TestReconcileElectionStep(t *testing.T) {
283286
LeaseName: "component-A",
284287
EmulationVersion: "1.19.0",
285288
BinaryVersion: "1.19.0",
286-
PingTime: ptr.To(metav1.NewMicroTime(time.Now())),
287-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now().Add(-1 * time.Minute))),
289+
PingTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
290+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now().Add(-1 * time.Minute))),
288291
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
289292
},
290293
},
@@ -308,7 +311,7 @@ func TestReconcileElectionStep(t *testing.T) {
308311
LeaseName: "component-A",
309312
EmulationVersion: "1.19.0",
310313
BinaryVersion: "1.19.0",
311-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
314+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
312315
PreferredStrategies: []v1.CoordinatedLeaseStrategy{"foo.com/bar"},
313316
},
314317
},
@@ -321,7 +324,7 @@ func TestReconcileElectionStep(t *testing.T) {
321324
Spec: v1.LeaseSpec{
322325
HolderIdentity: ptr.To("component-identity-expired"),
323326
LeaseDurationSeconds: ptr.To(int32(10)),
324-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now().Add(-1 * time.Minute))),
327+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now().Add(-1 * time.Minute))),
325328
},
326329
},
327330
expectLease: true,
@@ -344,6 +347,7 @@ func TestReconcileElectionStep(t *testing.T) {
344347
client.CoordinationV1(),
345348
client.CoordinationV1alpha1(),
346349
)
350+
controller.clock = fakeClock
347351
if err != nil {
348352
t.Fatal(err)
349353
}
@@ -435,6 +439,8 @@ func TestReconcileElectionStep(t *testing.T) {
435439
}
436440

437441
func TestController(t *testing.T) {
442+
fakeClock := testingclock.NewFakeClock(time.Now())
443+
438444
cases := []struct {
439445
name string
440446
leaseNN types.NamespacedName
@@ -455,7 +461,7 @@ func TestController(t *testing.T) {
455461
LeaseName: "component-A",
456462
EmulationVersion: "1.19.0",
457463
BinaryVersion: "1.19.0",
458-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
464+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
459465
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
460466
},
461467
},
@@ -485,7 +491,7 @@ func TestController(t *testing.T) {
485491
LeaseName: "component-A",
486492
EmulationVersion: "1.19.0",
487493
BinaryVersion: "1.19.0",
488-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
494+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
489495
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
490496
},
491497
},
@@ -498,7 +504,7 @@ func TestController(t *testing.T) {
498504
LeaseName: "component-A",
499505
EmulationVersion: "1.19.0",
500506
BinaryVersion: "1.20.0",
501-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
507+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
502508
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
503509
},
504510
},
@@ -511,7 +517,7 @@ func TestController(t *testing.T) {
511517
LeaseName: "component-A",
512518
EmulationVersion: "1.20.0",
513519
BinaryVersion: "1.20.0",
514-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
520+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
515521
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
516522
},
517523
},
@@ -549,7 +555,7 @@ func TestController(t *testing.T) {
549555
LeaseName: "component-A",
550556
EmulationVersion: "1.19.0",
551557
BinaryVersion: "1.19.0",
552-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
558+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
553559
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
554560
},
555561
},
@@ -590,7 +596,7 @@ func TestController(t *testing.T) {
590596
LeaseName: "component-A",
591597
EmulationVersion: "1.20.0",
592598
BinaryVersion: "1.20.0",
593-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
599+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
594600
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
595601
},
596602
},
@@ -603,7 +609,7 @@ func TestController(t *testing.T) {
603609
LeaseName: "component-A",
604610
EmulationVersion: "1.19.0",
605611
BinaryVersion: "1.19.0",
606-
RenewTime: ptr.To(metav1.NewMicroTime(time.Now())),
612+
RenewTime: ptr.To(metav1.NewMicroTime(fakeClock.Now())),
607613
PreferredStrategies: []v1.CoordinatedLeaseStrategy{v1.OldestEmulationVersion},
608614
},
609615
},
@@ -680,7 +686,7 @@ func TestController(t *testing.T) {
680686
if err == nil {
681687
if lease.Spec.PingTime != nil {
682688
c := lease.DeepCopy()
683-
c.Spec.RenewTime = &metav1.MicroTime{Time: time.Now()}
689+
c.Spec.RenewTime = &metav1.MicroTime{Time: fakeClock.Now()}
684690
_, err = client.CoordinationV1alpha1().LeaseCandidates(lc.Namespace).Update(ctx, c, metav1.UpdateOptions{})
685691
if err != nil {
686692
runtime.HandleError(err)

‎pkg/controlplane/controller/leaderelection/leasecandidategc_controller.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"k8s.io/client-go/kubernetes"
3131
listers "k8s.io/client-go/listers/coordination/v1alpha1"
3232
"k8s.io/client-go/tools/cache"
33+
"k8s.io/utils/clock"
3334

3435
"k8s.io/klog/v2"
3536
)
@@ -42,6 +43,8 @@ type LeaseCandidateGCController struct {
4243
leaseCandidatesSynced cache.InformerSynced
4344

4445
gcCheckPeriod time.Duration
46+
47+
clock clock.Clock
4548
}
4649

4750
// NewLeaseCandidateGC creates a new LeaseCandidateGCController.
@@ -52,6 +55,7 @@ func NewLeaseCandidateGC(clientset kubernetes.Interface, gcCheckPeriod time.Dura
5255
leaseCandidateInformer: leaseCandidateInformer,
5356
leaseCandidatesSynced: leaseCandidateInformer.Informer().HasSynced,
5457
gcCheckPeriod: gcCheckPeriod,
58+
clock: clock.RealClock{},
5559
}
5660
}
5761

@@ -80,7 +84,7 @@ func (c *LeaseCandidateGCController) gc(ctx context.Context) {
8084
}
8185
for _, leaseCandidate := range lcs {
8286
// evaluate lease from cache
83-
if !isLeaseCandidateExpired(leaseCandidate) {
87+
if !isLeaseCandidateExpired(c.clock, leaseCandidate) {
8488
continue
8589
}
8690
lc, err := c.kubeclientset.CoordinationV1alpha1().LeaseCandidates(leaseCandidate.Namespace).Get(ctx, leaseCandidate.Name, metav1.GetOptions{})
@@ -89,7 +93,7 @@ func (c *LeaseCandidateGCController) gc(ctx context.Context) {
8993
continue
9094
}
9195
// evaluate lease from apiserver
92-
if !isLeaseCandidateExpired(lc) {
96+
if !isLeaseCandidateExpired(c.clock, lc) {
9397
continue
9498
}
9599
if err := c.kubeclientset.CoordinationV1alpha1().LeaseCandidates(lc.Namespace).Delete(

0 commit comments

Comments
 (0)