#128·3FS

Release build encounters "Broken promise" error during meta configuration update on aarch64 (Ubuntu 22.04)

Author: Autumn-BaiCreated Mar 7, 2025Updated Mar 31, 2026

When deploying 3FS on Ubuntu 22.04 (aarch64) using the officially recommended compiler, we encounter a critical issue specifically in release builds.

After successfully compiling and starting mgmtd_main.server, executing the configuration update command:

bash

/opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml \
         --config.mgmtd_client.mgmtd_server_addresses '["RDMA://192.168.1.1:8000"]' \
         "set-config --type META --file /opt/3fs/etc/meta_main.toml"

results in: Broken promise for type name 'folly::Expected<folly::Unit, hf3fs::Status>'

Key observations:

The set-config process hangs indefinitely despite the meta configuration being actually updated
Subsequent meta node startup fails with heartbeat exceptions
Debug builds do not exhibit this behavior (deploy succeeds though FIO tests remain problematic)

Stack trace available via GDB：

Thread 11 "CliBG0" hit Catchpoint 1 (exception thrown), 0x0000fffff7e22dbc in __cxa_throw () from /lib/aarch64-linux-gnu/libstdc++.so.6
(gdb) bt
#0  0x0000fffff7e22dbc in __cxa_throw () from /lib/aarch64-linux-gnu/libstdc++.so.6
#1  0x0000fffff7e243f8 in operator new(unsigned long) () from /lib/aarch64-linux-gnu/libstdc++.so.6
#2  0x0000aaaaac2b5130 in std::__new_allocator<char>::allocate (this=0xffffea23761c, __n=187650013613149) at /usr/lib/gcc/aarch64-linux-gnu/12/../../../../include/c++/12/bits/new_allocator.h:137
#3  std::allocator<char>::allocate (this=0xffffea23761c, __n=187650013613149) at /usr/lib/gcc/aarch64-linux-gnu/12/../../../../include/c++/12/bits/allocator.h:188
#4  std::allocator_traits<std::allocator<char> >::allocate (__a=..., __n=187650013613149) at /usr/lib/gcc/aarch64-linux-gnu/12/../../../../include/c++/12/bits/alloc_traits.h:464
#5  fmt::v10::basic_memory_buffer<char, 500ul, std::allocator<char> >::grow (this=0xffffea237408, size=281474842078216) at /home/build/3fs/third_party/fmt/include/fmt/format.h:943
#6  0x0000aaaaaca70264 in fmt::v10::detail::buffer<char>::try_reserve (this=0xffffea237408, new_capacity=281474842078216) at /home/build/3fs/third_party/fmt/include/fmt/core.h:858
#7  fmt::v10::detail::buffer<char>::append<char> (this=0xffffea237408, begin=0x11f407841a8c0 <error: Cannot access memory at address 0x11f407841a8c0>,
    begin@entry=0xaaaaac674c50 <hf3fs::client::MgmtdClient::Impl::invoke<hf3fs::client::(anonymous namespace)::SetConfigOp>(hf3fs::client::(anonymous namespace)::SetConfigOp&)::{lambda(hf3fs::client::MgmtdClient::Impl&)#1}::operator()(hf3fs::client::MgmtdClient::Impl&) const+88> "\b", end=0x1c9eb24a8f510 <error: Cannot access memory at address 0x1c9eb24a8f510>) at /home/build/3fs/third_party/fmt/include/fmt/format.h:874
#8  fmt::v10::detail::copy_str<char, char const*> (begin=<optimized out>,
    begin@entry=0xaaaaac674c50 <hf3fs::client::MgmtdClient::Impl::invoke<hf3fs::client::(anonymous namespace)::SetConfigOp>(hf3fs::client::(anonymous namespace)::SetConfigOp&)::{lambda(hf3fs::client::MgmtdClient::Impl&)#1}::operator()(hf3fs::client::MgmtdClient::Impl&) const+88> "\b", end=0x1c9eb24a8f510 <error: Cannot access memory at address 0x1c9eb24a8f510>, out=...) at /home/build/3fs/third_party/fmt/include/fmt/core.h:1486
#9  fmt::v10::detail::copy_str_noinline<char, char const*, fmt::v10::appender> (begin=<optimized out>, begin@entry=0x11f407841a8c0 <error: Cannot access memory at address 0x11f407841a8c0>,
    end=0x1c9eb24a8f510 <error: Cannot access memory at address 0x1c9eb24a8f510>, out=out@entry=...) at /home/build/3fs/third_party/fmt/include/fmt/format.h:645
#10 0x0000aaaaaca70e58 in fmt::v10::detail::parse_replacement_field<char, fmt::v10::detail::vformat_to<char>(fmt::v10::detail::buffer<char>&, fmt::v10::basic_string_view<char>, fmt::v10::detail::vformat_args<char>::type, fmt::v10::detail::locale_ref)::format_handler&>(char const*, char const*, fmt::v10::detail::vformat_to<char>(fmt::v10::detail::buffer<char>&, fmt::v10::basic_string_view<char>, fmt::v10::detail::vformat_args<char>::type, fmt::v10::detail::locale_ref)::format_handler&) (begin=0xaaaaaabe2d62 "}", begin@entry=0xaaaaaabe2d61 "{}", end=<optimized out>, end@entry=0xaaaaaabe2d63 "", handler=...) at /home/build/3fs/third_party/fmt/include/fmt/core.h:1914
#11 0x0000aaaaaca679d8 in fmt::v10::detail::parse_format_string<false, char, fmt::v10::detail::vformat_to<char>(fmt::v10::detail::buffer<char>&, fmt::v10::basic_string_view<char>, fmt::v10::detail::vformat_args<char>::type, fmt::v10::detail::locale_ref)::format_handler>(fmt::v10::basic_string_view<char>, fmt::v10::detail::vformat_to<char>(fmt::v10::detail::buffer<char>&, fmt::v10::basic_string_view<char>, fmt::v10::detail::vformat_args<char>::type, fmt::v10::detail::locale_ref)::format_handler&&) (format_str=..., handler=...) at /home/build/3fs/third_party/fmt/include/fmt/core.h:2469
#12 fmt::v10::detail::vformat_to<char> (buf=..., fmt=..., args=..., loc=...) at /home/build/3fs/third_party/fmt/include/fmt/format.h:4330
#13 0x0000aaaaaca669d4 in fmt::v10::vformat[abi:cxx11](fmt::v10::basic_string_view<char>, fmt::v10::basic_format_args<fmt::v10::basic_format_context<fmt::v10::appender, char> >) (fmt=..., args=...)
    at /home/build/3fs/third_party/fmt/include/fmt/format-inl.h:1424
#14 0x0000aaaaac674458 in fmt::v10::format<std::basic_string_view<char, std::char_traits<char> >&> (args=<error reading variable: Cannot access memory at address 0x9>, fmt=...) at /home/build/3fs/third_party/fmt/include/fmt/core.h:2756
#15 hf3fs::client::MgmtdClient::Impl::withRetry<hf3fs::client::(anonymous namespace)::SetConfigOp::handle(hf3fs::client::MgmtdClient::Impl&)::{lambda(hf3fs::mgmtd::IMgmtdServiceStub&)#1}>(std::basic_string_view<char, std::char_traits<char> >,hf3fs::client::(anonymous namespace)::SetConfigOp::handle(hf3fs::client::MgmtdClient::Impl&)::{lambda(hf3fs::mgmtd::IMgmtdServiceStub&)#1}&&) (this=<optimized out>, methodName="SetConfig", f=...)
    at /home/build/3fs/src/client/mgmtd/MgmtdClient.cc:183
#16 0x0000aaaaaca6626c in std::__n4861::coroutine_handle<void>::resume (this=<optimized out>) at /usr/lib/gcc/aarch64-linux-gnu/12/../../../../include/c++/12/coroutine:135
#17 folly::resumeCoroutineWithNewAsyncStackRoot (h=..., frame=...) at /home/build/3fs/third_party/folly/folly/tracing/AsyncStack.cpp:179
#18 0x0000aaaaac08132c in folly::coro::detail::ViaCoroutinePromiseBase::executeContinuation (this=0xaaaaae041210) at /home/build/3fs/third_party/folly/folly/experimental/coro/ViaIfAsync.h:90
#19 0x0000aaaaac9bbdcc in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0xffffea2379a0) at /home/build/3fs/third_party/folly/folly/Function.h:375
#20 folly::catch_exception<folly::Function<void ()>&, void (&)(char const*) noexcept, char const*&, void>(folly::Function<void ()>&, void (&)(char const*) noexcept, char const*&) (t=..., c=<optimized out>, a=<optimized out>)
    at /home/build/3fs/third_party/folly/folly/lang/Exception.h:285
#21 folly::Executor::invokeCatchingExns<folly::Function<void ()> >(char const*, folly::Function<void ()>) (f=..., p=<optimized out>) at /home/build/3fs/third_party/folly/folly/Executor.h:234
#22 folly::ThreadPoolExecutor::runTask (this=<optimized out>, thread=std::shared_ptr<folly::ThreadPoolExecutor::Thread> (use count 3, weak count 0) = {...}, task=...)
    at /home/build/3fs/third_party/folly/folly/executors/ThreadPoolExecutor.cpp:102
#23 0x0000aaaaac99425c in folly::CPUThreadPoolExecutor::threadRun (this=this@entry=0xaaaaae029700, thread=std::shared_ptr<folly::ThreadPoolExecutor::Thread> (use count 3, weak count 0) = {...})
    at /home/build/3fs/third_party/folly/folly/executors/CPUThreadPoolExecutor.cpp:333
#24 0x0000aaaaac9be6dc in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (
    __f=<optimized out>, __t=<optimized out>, __args=...) at /usr/lib/gcc/aarch64-linux-gnu/12/../../../../include/c++/12/bits/invoke.h:74
#25 0x0000fffff7e531fc in ?? () from /lib/aarch64-linux-gnu/libstdc++.so.6
#26 0x0000fffff587d5b8 in start_thread (arg=0x0) at ./nptl/pthread_create.c:442
#27 0x0000fffff58e5edc in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Source: deepseek-ai/3FS

View original on GitHub View discussion on GitHub