Stop() deadlocks when called after GracefulStop() while a handler is still running

// Standalone reproduction of the gRPC-go GracefulStop/Stop mutex deadlock.
//
//	go mod init grpcrepro
//	go get google.golang.org/[email protected]
//	go run .
//
// Expected (buggy) output: "REPRODUCED: Stop() is still blocked after 15s".
package main

import (
	"context"
	"log"
	"net"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
	healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

type blockingHealth struct {
	healthpb.UnimplementedHealthServer
	started chan struct{}
	release chan struct{}
}

// Check ignores its context, standing in for a handler that does not return
// promptly on cancellation (e.g. a subprocess run under context.WithoutCancel).
func (h *blockingHealth) Check(context.Context, *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error) {
	close(h.started)
	<-h.release
	return &healthpb.HealthCheckResponse{}, nil
}

func main() {
	s := grpc.NewServer(grpc.WaitForHandlers(false))
	h := &blockingHealth{started: make(chan struct{}), release: make(chan struct{})}
	healthpb.RegisterHealthServer(s, h)

	lis, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		log.Fatal(err)
	}
	go func() { _ = s.Serve(lis) }()

	conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
	if err != nil {
		log.Fatal(err)
	}
	go func() { _, _ = healthpb.NewHealthClient(conn).Check(context.Background(), &healthpb.HealthCheckRequest{}) }()

	<-h.started // handler is running and counted in handlersWG
	log.Println("handler running; starting GracefulStop()")

	go s.GracefulStop()

	// The client connection drains during the "grace period"; GracefulStop then
	// advances past its conns loop into handlersWG.Wait(), holding the server mutex.
	time.Sleep(200 * time.Millisecond)
	_ = conn.Close()
	time.Sleep(time.Second)

	log.Println("grace period elapsed; calling Stop()")
	done := make(chan struct{})
	start := time.Now()
	go func() { s.Stop(); close(done) }()

	select {
	case <-done:
		log.Printf("NOT reproduced: Stop() returned in %s", time.Since(start))
	case <-time.After(15 * time.Second):
		log.Printf("REPRODUCED: Stop() is still blocked after %s (deadlocked on the server mutex held by GracefulStop)", time.Since(start))
	}

	close(h.release) // let the handler finish so the process can exit cleanly
	<-done
	log.Printf("Stop() finally returned after %s", time.Since(start))
}

2026/09/02 14:17:19 handler running; starting GracefulStop()
2026/09/02 14:17:20 grace period elapsed; calling Stop()
2026/09/02 14:17:35 REPRODUCED: Stop() is still blocked after 15.00107675s (deadlocked on the server mutex held by GracefulStop)
2026/09/02 14:17:35 Stop() finally returned after 15.001543666s