FileSnapshotSink: failed write leaves state.bin open, and Cancel() leaves the .tmp directory on disk
Author: chahat-101Created Sep 16, 2026Updated Sep 16, 2026
If writing a snapshot fails (e.g. the disk fills up), FileSnapshotSink doesn't clean up properly:
finalize()returns early whenFlush()orSync()fails, sostate.binnever gets closed. This affects bothClose()andCancel()(file_snapshot.go#L470-L478).Cancel()returns that error beforeos.RemoveAll(s.dir), so the partial.tmpdirectory stays on disk (file_snapshot.go#L458-L464). Reaping skips.tmpdirs, so these pile up with each failed attempt.
Raft calls Cancel() on exactly these paths, e.g. when Persist fails (snapshot.go#L192). #224 fixed the directory cleanup for Close(), but Cancel() was missed.
Repro (Linux, v1.8.0): this caps file size so writes past 1 MiB fail, then counts what's left open.
//go:build linux
// Reproduces FileSnapshotSink cleanup problems after a failed state file write.
// RLIMIT_FSIZE makes writes past 1 MiB fail with EFBIG, standing in for a full disk.
// GC is disabled so os.File finalizers don't close leaked files during the run.
package main
import (
"fmt"
"os"
"os/signal"
"path/filepath"
"runtime/debug"
"strings"
"syscall"
"github.com/hashicorp/raft"
)
// openStateFiles counts open file descriptors pointing at a state.bin under dir.
func openStateFiles(dir string) int {
entries, err := os.ReadDir("/proc/self/fd")
if err != nil {
panic(err)
}
n := 0
for _, e := range entries {
target, err := os.Readlink(filepath.Join("/proc/self/fd", e.Name()))
if err == nil && strings.HasPrefix(target, dir) && strings.Contains(target, "state.bin") {
n++
}
}
return n
}
func run(method string) {
dir, err := os.MkdirTemp("", "raft-snapshot-repro")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
store, err := raft.NewFileSnapshotStore(dir, 1, os.Stderr)
if err != nil {
panic(err)
}
_, trans := raft.NewInmemTransport("")
sink, err := store.Create(1, 10, 1, raft.Configuration{}, 0, trans)
if err != nil {
panic(err)
}
_, writeErr := sink.Write(make([]byte, 2<<20))
var endErr error
if method == "Cancel" {
endErr = sink.Cancel()
} else {
endErr = sink.Close()
}
tmpDirs, _ := filepath.Glob(filepath.Join(dir, "snapshots", "*.tmp"))
fmt.Printf("%s: write failed=%t, %s returned error=%t, state.bin still open=%d, .tmp dirs left=%d\n",
method, writeErr != nil, method, endErr != nil, openStateFiles(dir), len(tmpDirs))
}
func main() {
debug.SetGCPercent(-1)
signal.Ignore(syscall.SIGXFSZ)
var lim syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_FSIZE, &lim); err != nil {
panic(err)
}
lim.Cur = 1 << 20
if err := syscall.Setrlimit(syscall.RLIMIT_FSIZE, &lim); err != nil {
panic(err)
}
run("Cancel")
run("Close")
}$ go run . 2>/dev/null
Cancel: write failed=true, Cancel returned error=true, state.bin still open=1, .tmp dirs left=1
Close: write failed=true, Close returned error=true, state.bin still open=1, .tmp dirs left=0Both should show state.bin still open=0 and .tmp dirs left=0.
The fix is small: close stateFile on the error paths in finalize(), and add the same RemoveAll cleanup to Cancel(). I have it ready with a regression test and can open a PR if that approach works for you.
Source: hashicorp/raft