rpc: a stream connection can be destroyed while its loop is still running
Author: bhalevyCreated Aug 30, 2026Updated Aug 31, 2026
Seen with Scylla 2026.2.2 manifested as assertion failure when destroying a connection after error:
(gdb) bt
#0 __GI_abort () at abort.c:95
#1 0x0000ff6eb9a72ffc [PAC] in __gnu_cxx::__verbose_terminate_handler ()
at ../../../../libstdc++-v3/libsupc++/vterminate.cc:95
#2 0x0000ff6eb9a70530 [PAC] in __cxxabiv1::__terminate (handler=<optimized out>)
at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:48
#3 0x0000ff6eb9a66264 [PAC] in std::terminate ()
at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:58
#4 0x0000000001eb69b4 [PAC] in seastar::internal::assert_fail (msg=<optimized out>,
file=<optimized out>, line=<optimized out>, func=<optimized out>)
at ./build/release/seastar/./seastar/src/util/log.cc:122
#5 0x000000000201b0e8 in seastar::output_stream<char>::~output_stream (this=0xe0a00a458550)
at ./seastar/include/seastar/core/iostream.hh:502
#6 seastar::rpc::connection::socket_and_buffers::~socket_and_buffers (this=0xe0a00a458520)
at ./seastar/include/seastar/rpc/rpc.hh:243
#7 std::_Optional_payload_base<seastar::rpc::connection::socket_and_buffers>::_M_destroy (
this=0xe0a00a458520)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/optional:307
#8 std::_Optional_payload_base<seastar::rpc::connection::socket_and_buffers>::_M_reset (
this=0xe0a00a458520)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/optional:338
#9 std::_Optional_payload<seastar::rpc::connection::socket_and_buffers, false, false, false>::~_Optional_payload (this=0xe0a00a458520)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/optional:461
#10 std::_Optional_base<seastar::rpc::connection::socket_and_buffers, false, false>::~_Optional_base (this=0xe0a00a458520)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/optional:485
#11 seastar::rpc::connection::~connection (this=0xe0a00a458510)
at ./seastar/include/seastar/rpc/rpc.hh:334
#12 0x000000000201a860 in seastar::rpc::client::~client (this=0xe0a00a458510)
at ./seastar/include/seastar/rpc/rpc.hh:500
#13 seastar::shared_ptr_count_for<seastar::rpc::client>::~shared_ptr_count_for (
this=0xe0a00a458500) at ./seastar/include/seastar/core/shared_ptr.hh:477
#14 seastar::shared_ptr_count_for<seastar::rpc::client>::~shared_ptr_count_for (
this=0xe0a00a458500) at ./seastar/include/seastar/core/shared_ptr.hh:477
#15 0x000000000201b470 in seastar::shared_ptr<seastar::rpc::connection>::~shared_ptr (
this=<optimized out>) at ./seastar/include/seastar/core/shared_ptr.hh:554
#16 seastar::shared_ptr<seastar::rpc::connection>::operator= (this=<optimized out>, x=...)
at ./seastar/include/seastar/core/shared_ptr.hh:567
#17 seastar::shared_ptr<seastar::rpc::connection>::operator=(decltype(nullptr)) (
this=<optimized out>) at ./seastar/include/seastar/core/shared_ptr.hh:573
#18 seastar::foreign_ptr<seastar::shared_ptr<seastar::rpc::connection> >::destroy_on (
cpu=2902194664, p=...) at ./seastar/include/seastar/core/sharded.hh:960
#19 seastar::foreign_ptr<seastar::shared_ptr<seastar::rpc::connection> >::destroy (
this=0xe0a000dfc648, cpu=2902194664, p=...)
at ./seastar/include/seastar/core/sharded.hh:944
#20 seastar::foreign_ptr<seastar::shared_ptr<seastar::rpc::connection> >::~foreign_ptr (
this=0xe0a000dfc648) at ./seastar/include/seastar/core/sharded.hh:996
#21 seastar::lw_shared_ptr_no_esft<seastar::foreign_ptr<seastar::shared_ptr<seastar::rpc::connection> > >::~lw_shared_ptr_no_esft (this=0xe0a000dfc640)
at ./seastar/include/seastar/core/shared_ptr.hh:165
#22 seastar::internal::lw_shared_ptr_accessors_no_esft<seastar::foreign_ptr<seastar::shared_ptr<seastar::rpc::connection> > >::dispose (counter=0xe0a000dfc640)
at ./seastar/include/seastar/core/shared_ptr.hh:221
#23 seastar::lw_shared_ptr<seastar::foreign_ptr<seastar::shared_ptr<seastar::rpc::connection> > >::~lw_shared_ptr (this=0xe0a00a34d918) at ./seastar/include/seastar/core/shared_ptr.hh:318
#24 seastar::rpc::sink<streaming::stream_blob_cmd_data>::impl::~impl (this=0xe0a00a34d910)
at ./seastar/include/seastar/rpc/rpc_types.hh:330
#25 seastar::rpc::internal::sink_impl<netw::serializer, streaming::stream_blob_cmd_data>::~sink_impl (this=0xe0a00a34d910) at ./seastar/include/seastar/rpc/rpc_impl.hh:930
#26 seastar::shared_ptr_count_for<seastar::rpc::internal::sink_impl<netw::serializer, streaming::stream_blob_cmd_data> >::~shared_ptr_count_for (this=0xe0a00a34d900)
at ./seastar/include/seastar/core/shared_ptr.hh:477
#27 seastar::shared_ptr_count_for<seastar::rpc::internal::sink_impl<netw::serializer, streaming::stream_blob_cmd_data> >::~shared_ptr_count_for (this=0xe0a00a34d900)
at ./seastar/include/seastar/core/shared_ptr.hh:477
#28 0x00000000027356e8 in seastar::shared_ptr<seastar::rpc::sink<streaming::stream_blob_cmd_data>::impl>::~shared_ptr (this=0xe0a00a34d900) at ./seastar/include/seastar/core/shared_ptr.hh:554
#29 seastar::rpc::sink<streaming::stream_blob_cmd_data>::~sink (this=0xe0a00a34d900)
at ./seastar/include/seastar/rpc/rpc_types.hh:321
#30 streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source::~sink_and_source() (this=0xe0a00c6f6c40) at ./streaming/stream_blob.cc:524
#31 std::destroy_at<streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source>(streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source*) (__location=0xe0a00c6f6c40)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/bits/stl_construct.h:88
#32 std::_Destroy<streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source>(streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source*) (__pointer=0xe0a00c6f6c40)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/bits/stl_construct.h:164
#33 std::_Destroy<streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source*>(streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source*, streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source*) (__first=0xe0a00c6f6c40, __last=<optimized out>)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/bits/stl_construct.h:226
#34 std::_Destroy<streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source*, streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source>(streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source*, streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source*, std::allocator<streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source>&) (__first=<optimized out>,
__last=<optimized out>)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/bits/alloc_traits.h:1045
#35 std::vector<streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source, std::allocator<streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool)::sink_and_source> >::~vector (this=<optimized out>)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/bits/stl_vector.h:802
#36 streaming::tablet_stream_files(netw::messaging_service&, std::__cxx11::list<streaming::stream_blob_info, std::allocator<streaming::stream_blob_info> >, std::vector<streaming::node_and_shard, std::allocator<streaming::node_and_shard> >, utils::tagged_uuid<table_id_tag>, utils::tagged_uuid<streaming::file_stream_id_tag>, utils::tagged_uuid<service::session_id_tag>, bool) [clone .resume] (ms=..., sources=..., targets=..., table=..., ops_id=..., topo_guard=...,
inject_errors=<optimized out>) at ./streaming/stream_blob.cc:727
#37 0x0000000003aea050 in std::__n4861::coroutine_handle<void>::resume (this=0xe0a00c6a09c0)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/coroutine:142
(gdb) up
#11 seastar::rpc::connection::~connection (this=0xe0a00a458510)
at ./seastar/include/seastar/rpc/rpc.hh:334
334 virtual ~connection() {}
(gdb) p *this
$3 = {
_vptr$connection = 0x805a40 <vtable for seastar::rpc::connection+16>,
_error = true,
_connected = std::optional [no contained value],
_negotiated = std::optional [no contained value],
_stopped = {
<seastar::internal::promise_base_with_type<void>> = {
<seastar::internal::promise_base> = {
_future = 0x0,
_state = 0xe0a00a85fb08,
_task = 0x0
},
},
members of seastar::promise<void>:
_local_state = {
<seastar::future_state_base> = {
_u = {
st = seastar::future_state_base::state::invalid,
ex = {
_M_exception_object = 0x0
}
}
},
<seastar::internal::uninitialized_wrapper<seastar::internal::monostate>> = {
_v = {<No data fields>}
},
}
},
_stats = {
replied = 0,
pending = 0,
exception_received = 0,
sent_messages = 8,
wait_reply = 0,
timeout = 0,
delay_samples = 0,
delay_total = std::chrono::duration = { 0s }
},
_logger = @0xe0a008a93a48,
_serializer = 0xe0a008a93a40,
_outgoing_queue_ready = {
<seastar::internal::future_base> = {
_promise = 0x0
},
members of seastar::future<void>:
_state = {
<seastar::future_state_base> = {
_u = {
st = seastar::future_state_base::state::invalid,
ex = {
_M_exception_object = 0x0
}
}
},
<seastar::internal::uninitialized_wrapper<seastar::internal::monostate>> = {
_v = {<No data fields>}
},
}
},
_outgoing_queue = boost::intrusive::list of size 0 = [] = [],
_outgoing_queue_size = 0,
_compressor = std::unique_ptr<seastar::rpc::compressor> = {
get() = 0x0
},
_propagate_timeout = false,
_timeout_negotiated = false,
_handler_duration_negotiated = true,
_is_stream = true,
_id = {
_id = 11403291
},
_streams = std::unordered_map with 0 elements,
_stream_queue = {
_q = Traceback (most recent call last):
File "/usr/share/gcc-15/python/libstdcxx/v6/printers.py", line 734, in to_string
self._visualizer.to_string())
^^^^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: 'NoneType' object has no attribute 'to_string'
,
_max = 50,
_not_empty = std::optional [no contained value],
_not_full = std::optional [no contained value],
_ex = {
_M_exception_object = 0x0
}
},
_stream_sem = {
<seastar::semaphore_default_exception_factory> = {<No data fields>},
members of seastar::basic_semaphore<seastar::semaphore_default_exception_factory, std::chrono::_V2::steady_clock>:
_count = 102400,
_ex = {
_M_exception_object = 0x0
},
_wait_list = {
_front = std::unique_ptr<seastar::internal::abortable_fifo<seastar::basic_semaphore<seastar::semaphore_default_exception_factory, std::chrono::_V2::steady_clock>::entry, seastar::basic_semaphore<seastar::semaphore_default_exception_factory, std::chrono::_V2::steady_clock>::expiry_handler>::entry> = {
get() = 0x0
},
_list = {
_front_chunk = 0x0,
_back_chunk = 0x0,
_nchunks = 0,
_free_chunks = 0x0,
_nfree_chunks = 0
},
_on_abort = {
sem = @0xe0a00a458788
},
_size = 0
},
_used = {<No data fields>}
},
_sink_closed = true,
_source_closed = true,
_sink_closed_future = {
<seastar::internal::future_base> = {
_promise = 0x0
},
members of seastar::future<bool>:
_state = {
<seastar::future_state_base> = {
_u = {
st = seastar::future_state_base::state::invalid,
ex = {
_M_exception_object = 0x0
}
}
},
<seastar::internal::uninitialized_wrapper<bool>> = {
_v = {
value = false
}
},
}
}
}(gdb) down
#7 std::_Optional_payload_base<seastar::rpc::connection::socket_and_buffers>::_M_destroy (
this=0xe0a00a458520)
at /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/optional:307
307 in /usr/lib/gcc/aarch64-redhat-linux/15/../../../../include/c++/15/optional
(gdb) p this->_M_payload._M_value
$4 = {
fd = {
_csi = std::unique_ptr<seastar::net::connected_socket_impl> = {
get() = 0xe0a00bbbde40
}
},
read_buf = {
_fd = {
_dsi = std::unique_ptr<seastar::data_source_impl> = {
get() = 0xe0a00f396d50
}
},
_buf = {
_buffer = 0x0,
_size = 0,
_deleter = {
_impl = 0x0
}
},
_eof = true
},
write_buf = {
_fd = {
_dsi = std::unique_ptr<seastar::data_sink_impl> = {
get() = 0xe0a00c58fe40
}
},
_buf = {
_buffer = 0x0,
_size = 0,
_deleter = {
_impl = 0x0
}
},
_zc_bufs = std::vector of length 1, capacity 1 = {{
_buffer = 0xe0a00a0a0820 "\211",
_size = 141,
_deleter = {
_impl = 0xe0a00a0a0821
}
}},
_size = 8192,
_end = 0,
_zc_len = 141,
_trim_to_size = false,
_batch_flushes = true,
_in_batch = std::optional = {
[contained value] = {
<seastar::internal::promise_base_with_type<void>> = {
<seastar::internal::promise_base> = {
_future = 0x0,
_state = 0xe0a00a4585c0,
_task = 0x0
},
},
members of seastar::promise<void>:
_local_state = {
<seastar::future_state_base> = {
_u = {
st = seastar::future_state_base::state::future,
ex = {
_M_exception_object = 0x1
}
}
},
<seastar::internal::uninitialized_wrapper<seastar::internal::monostate>> = {
_v = {<No data fields>}
},
}
}
},
_flush = true,
_flushing = false,
_ex = {
_M_exception_object = 0x0
},
_in_poller = {
<boost::intrusive::generic_hook<(boost::intrusive::algo_types)1, boost::intrusive::slist_node_traits<void*>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)1, (boost::intrusive::base_hook_type)0>> = {
<boost::intrusive::slist_node<void*>> = {
next_ = 0xe0a00a0ae9e0
},
<boost::intrusive::hook_tags_definer<boost::intrusive::generic_hook<(boost::intrusive::algo_types)1, boost::intrusive::slist_node_traits<void*>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)1, (boost::intrusive::base_hook_type)0>, 0u>> = {<No data fields>}, <No data fields>}, <No data fields>}
}
}Source: scylladb/seastar