Leaderlease may cause unnecessary stepping down of leaders in the cluster
Author: zivfutureCreated Feb 20, 2025Updated Feb 20, 2025
Why are LeaderLeadseTimeout and ElectTimeout inconsistent?, the source code here: https://github.com/hashicorp/raft/blob/8f99c150c37d3f7ded1c13f1f5f52b60374632b6/raft.go#L937
// checkLeaderLease is used to check if we can contact a quorum of nodes
// within the last leader lease interval. If not, we need to step down,
// as we may have lost connectivity. Returns the maximum duration without
// contact. This must only be called from the main thread.
func (r *Raft) checkLeaderLease() time.Duration {
// Track contacted nodes, we can always contact ourself
contacted := 0
// Store lease timeout for this one check invocation as we need to refer to it
// in the loop and would be confusing if it ever becomes reloadable and
// changes between iterations below.
leaseTimeout := r.config().LeaderLeaseTimeout
// Check each follower
var maxDiff time.Duration
now := time.Now()
for _, server := range r.configurations.latest.Servers {
if server.Suffrage == Voter {
if server.ID == r.localID {
contacted++
continue
}
f := r.leaderState.replState[server.ID]
diff := now.Sub(f.LastContact())
if diff <= leaseTimeout {
contacted++
if diff > maxDiff {
maxDiff = diff
}
} else {
// Log at least once at high value, then debug. Otherwise it gets very verbose.
if diff <= 3*leaseTimeout {
r.logger.Warn("failed to contact", "server-id", server.ID, "time", diff)
} else {
r.logger.Debug("failed to contact", "server-id", server.ID, "time", diff)
}
}
metrics.AddSample([]string{"raft", "leader", "lastContact"}, float32(diff/time.Millisecond))
}
}
// Verify we can contact a quorum
quorum := r.quorumSize()
if contacted < quorum {
r.logger.Warn("failed to contact quorum of nodes, stepping down")
//some questions here <----------------------
r.setState(Follower)
metrics.IncrCounter([]string{"raft", "transition", "leader_lease_timeout"}, 1)
}
return maxDiff
}As shown in the image below, in some cases, the leader does not need to step down .but here he has stepped down
Source: hashicorp/raft