#714·raft

FileSnapshotSink: failed write leaves state.bin open, and Cancel() leaves the .tmp directory on disk

Author: chahat-101Created Sep 16, 2026Updated Sep 16, 2026

If writing a snapshot fails (e.g. the disk fills up), FileSnapshotSink doesn't clean up properly:

  • finalize() returns early when Flush() or Sync() fails, so state.bin never gets closed. This affects both Close() and Cancel() (file_snapshot.go#L470-L478).
  • Cancel() returns that error before os.RemoveAll(s.dir), so the partial .tmp directory stays on disk (file_snapshot.go#L458-L464). Reaping skips .tmp dirs, so these pile up with each failed attempt.

Raft calls Cancel() on exactly these paths, e.g. when Persist fails (snapshot.go#L192). #224 fixed the directory cleanup for Close(), but Cancel() was missed.

Repro (Linux, v1.8.0): this caps file size so writes past 1 MiB fail, then counts what's left open.

go
//go:build linux

// Reproduces FileSnapshotSink cleanup problems after a failed state file write.
// RLIMIT_FSIZE makes writes past 1 MiB fail with EFBIG, standing in for a full disk.
// GC is disabled so os.File finalizers don't close leaked files during the run.
package main

import (
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"runtime/debug"
	"strings"
	"syscall"

	"github.com/hashicorp/raft"
)

// openStateFiles counts open file descriptors pointing at a state.bin under dir.
func openStateFiles(dir string) int {
	entries, err := os.ReadDir("/proc/self/fd")
	if err != nil {
		panic(err)
	}
	n := 0
	for _, e := range entries {
		target, err := os.Readlink(filepath.Join("/proc/self/fd", e.Name()))
		if err == nil && strings.HasPrefix(target, dir) && strings.Contains(target, "state.bin") {
			n++
		}
	}
	return n
}

func run(method string) {
	dir, err := os.MkdirTemp("", "raft-snapshot-repro")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(dir)

	store, err := raft.NewFileSnapshotStore(dir, 1, os.Stderr)
	if err != nil {
		panic(err)
	}
	_, trans := raft.NewInmemTransport("")

	sink, err := store.Create(1, 10, 1, raft.Configuration{}, 0, trans)
	if err != nil {
		panic(err)
	}
	_, writeErr := sink.Write(make([]byte, 2<<20))

	var endErr error
	if method == "Cancel" {
		endErr = sink.Cancel()
	} else {
		endErr = sink.Close()
	}

	tmpDirs, _ := filepath.Glob(filepath.Join(dir, "snapshots", "*.tmp"))
	fmt.Printf("%s: write failed=%t, %s returned error=%t, state.bin still open=%d, .tmp dirs left=%d\n",
		method, writeErr != nil, method, endErr != nil, openStateFiles(dir), len(tmpDirs))
}

func main() {
	debug.SetGCPercent(-1)
	signal.Ignore(syscall.SIGXFSZ)
	var lim syscall.Rlimit
	if err := syscall.Getrlimit(syscall.RLIMIT_FSIZE, &lim); err != nil {
		panic(err)
	}
	lim.Cur = 1 << 20
	if err := syscall.Setrlimit(syscall.RLIMIT_FSIZE, &lim); err != nil {
		panic(err)
	}

	run("Cancel")
	run("Close")
}
$ go run . 2>/dev/null
Cancel: write failed=true, Cancel returned error=true, state.bin still open=1, .tmp dirs left=1
Close: write failed=true, Close returned error=true, state.bin still open=1, .tmp dirs left=0

Both should show state.bin still open=0 and .tmp dirs left=0.

The fix is small: close stateFile on the error paths in finalize(), and add the same RemoveAll cleanup to Cancel(). I have it ready with a regression test and can open a PR if that approach works for you.