Improper thread detaching causes deadlock in LDR on Windows 10+
JNA: any JVM: any OpenJDK based OS: Windows 10+
Recently I analyzed a hang in a customer application raised against Zulu11.54+26-SA (before the same customer complained about a jdk-8 based JVM assemlage). The dump for the issue had the following stacktrace
.306 Id: 27f4.3f70 Suspend: 0 Teb: 00000000`54696000 Unfrozen "foo"
Start: .306 Id: 27f4.3f70 Suspend: 0 Teb: 00000000`54696000 Unfrozen "foo"
Start: ucrtbase!thread_start<unsigned int (__cdecl*)(void * __ptr64)> (00007ffb`6a092650)ucrtbase!thread_start<unsigned int (__cdecl*)(void * __ptr64)> (00007ffb`6a092650)
Priority: 0 Priority class: 32 Affinity: fff
# Child-SP RetAddr Call Site
Priority: 0 Priority class: 32 Affinity: fff
# Child-SP RetAddr Call Site
00 00 00000001`7f60e698 00007ffb`6a458933 ntdll!NtWaitForSingleObject+0x14
00000001`7f60e698 00007ffb`6a458933 ntdll!NtWaitForSingleObject+0x14
01 01 00000001`7f60e6a0 00007ffb`3bc1226f KERNELBASE!WaitForSingleObjectEx+0x93
00000001`7f60e6a0 00007ffb`3bc1226f KERNELBASE!WaitForSingleObjectEx+0x93
02 02 00000001`7f60e740 00007ffb`3bbe61dc jvm!os::PlatformEvent::park+0x8f00000001`7f60e740 00007ffb`3bbe61dc jvm!os::PlatformEvent::park+0x8f [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\os\windows\os_windows.cpp @ 5370] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\os\windows\os_windows.cpp @ 5370]
03 03 (Inline Function) --------`-------- jvm!ParkCommon+0x8(Inline Function) --------`-------- jvm!ParkCommon+0x8 [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\mutex.cpp @ 404] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\mutex.cpp @ 404]
04 04 00000001`7f60e770 00007ffb`3bbe6a81 jvm!Monitor::ILock+0x5c00000001`7f60e770 00007ffb`3bbe6a81 jvm!Monitor::ILock+0x5c [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\mutex.cpp @ 465] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\mutex.cpp @ 465]
05 05 (Inline Function) --------`-------- jvm!Monitor::lock_without_safepoint_check+0x8(Inline Function) --------`-------- jvm!Monitor::lock_without_safepoint_check+0x8 [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\mutex.cpp @ 938] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\mutex.cpp @ 938]
06 06 00000001`7f60e7a0 00007ffb`3bc7ad23 jvm!Monitor::lock_without_safepoint_check+0x3100000001`7f60e7a0 00007ffb`3bc7ad23 jvm!Monitor::lock_without_safepoint_check+0x31 [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\mutex.cpp @ 944] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\mutex.cpp @ 944]
07 07 00000001`7f60e7d0 00007ffb`3bc7d328 jvm!SafepointSynchronize::block+0x17300000001`7f60e7d0 00007ffb`3bc7d328 jvm!SafepointSynchronize::block+0x173 [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\safepoint.cpp @ 939] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\safepoint.cpp @ 939]
08 08 00000001`7f60e810 00007ffb`3bd5daa1 jvm!SafepointMechanism::block_if_requested_slow+0x1800000001`7f60e810 00007ffb`3bd5daa1 jvm!SafepointMechanism::block_if_requested_slow+0x18 [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\safepointmechanism.cpp @ 91] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\safepointmechanism.cpp @ 91]
09 09 (Inline Function) --------`-------- jvm!SafepointMechanism::block_if_requested+0x1f(Inline Function) --------`-------- jvm!SafepointMechanism::block_if_requested+0x1f [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\safepointmechanism.inline.hpp @ 62] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\safepointmechanism.inline.hpp @ 62]
0a 0a 00000001`7f60e840 00007ffb`3b794f08 jvm!JavaThread::check_safepoint_and_suspend_for_native_trans+0xe100000001`7f60e840 00007ffb`3b794f08 jvm!JavaThread::check_safepoint_and_suspend_for_native_trans+0xe1 [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\thread.cpp @ 2524] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\thread.cpp @ 2524]
0b 0b 00000001`7f60fb20 00007ffb`3ba266a5 jvm!ThreadStateTransition::transition_from_native+0x8800000001`7f60fb20 00007ffb`3ba266a5 jvm!ThreadStateTransition::transition_from_native+0x88 [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\interfacesupport.inline.hpp @ 175] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\runtime\interfacesupport.inline.hpp @ 175]
0c 0c 00000001`7f60fb50 00007ffb`446e1115 jvm!jni_DetachCurrentThread+0x6500000001`7f60fb50 00007ffb`446e1115 jvm!jni_DetachCurrentThread+0x65 [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\prims\jni.cpp @ 4359] [c:\jenkins\workspace\zulu11-build-win64\zulu-src.git\src\hotspot\share\prims\jni.cpp @ 4359]
0d 0d 00000001`7f60fb80 00007ffb`446e11ef jna6833526131579377318!Java_com_sun_jna_Native_getDirectByteBuffer__Lcom_sun_jna_Pointer_2JJJ+0x2665
00000001`7f60fb80 00007ffb`446e11ef jna6833526131579377318!Java_com_sun_jna_Native_getDirectByteBuffer__Lcom_sun_jna_Pointer_2JJJ+0x2665
0e 0e 00000001`7f60fbe0 00007ffb`446e956f jna6833526131579377318!Java_com_sun_jna_Native_getDirectByteBuffer__Lcom_sun_jna_Pointer_2JJJ+0x273f
00000001`7f60fbe0 00007ffb`446e956f jna6833526131579377318!Java_com_sun_jna_Native_getDirectByteBuffer__Lcom_sun_jna_Pointer_2JJJ+0x273f
0f 0f 00000001`7f60fc20 00007ffb`6de95d37 jna6833526131579377318!Java_com_sun_jna_Native_getDirectByteBuffer__Lcom_sun_jna_Pointer_2JJJ+0xaabf
00000001`7f60fc20 00007ffb`6de95d37 jna6833526131579377318!Java_com_sun_jna_Native_getDirectByteBuffer__Lcom_sun_jna_Pointer_2JJJ+0xaabf
10 10 00000001`7f60fc80 00007ffb`6de933f4 ntdll!LdrpCallInitRoutine+0x6f
00000001`7f60fc80 00007ffb`6de933f4 ntdll!LdrpCallInitRoutine+0x6f
11 11 00000001`7f60fcf0 00007ffb`6deca33e ntdll!LdrShutdownThread+0x154
00000001`7f60fcf0 00007ffb`6deca33e ntdll!LdrShutdownThread+0x154
12 12 00000001`7f60fdf0 00007ffb`6a49992a ntdll!RtlExitUserThread+0x3e
00000001`7f60fdf0 00007ffb`6a49992a ntdll!RtlExitUserThread+0x3e
13 13 00000001`7f60fe30 00007ffb`6a092794 KERNELBASE!FreeLibraryAndExitThread+0x4a
00000001`7f60fe30 00007ffb`6a092794 KERNELBASE!FreeLibraryAndExitThread+0x4a
14 14 00000001`7f60fe60 00007ffb`6a0e0809 ucrtbase!common_end_thread+0xa4
00000001`7f60fe60 00007ffb`6a0e0809 ucrtbase!common_end_thread+0xa4
15 15 00000001`7f60fe90 00007ffb`4593f72c ucrtbase!endthreadex+0x9
00000001`7f60fe90 00007ffb`4593f72c ucrtbase!endthreadex+0x9
16 16 00000001`7f60fec0 00007ffb`6a09268a glib_2_0_0!g_rw_lock_writer_unlock+0x33c
00000001`7f60fec0 00007ffb`6a09268a glib_2_0_0!g_rw_lock_writer_unlock+0x33c
17 17 00000001`7f60ff00 00007ffb`6b0b7974 ucrtbase!thread_start<unsigned int (__cdecl*)(void * __ptr64)>+0x3a
00000001`7f60ff00 00007ffb`6b0b7974 ucrtbase!thread_start<unsigned int (__cdecl*)(void * __ptr64)>+0x3a
18 18 00000001`7f60ff30 00007ffb`6deca2d1 kernel32!BaseThreadInitThunk+0x14
00000001`7f60ff30 00007ffb`6deca2d1 kernel32!BaseThreadInitThunk+0x14
19 19 00000001`7f60ff60 00000000`00000000 ntdll!RtlUserThreadStart+0x21
00000001`7f60ff60 00000000`00000000 ntdll!RtlUserThreadStart+0x21and I found the code causing such stack trace callback.c:587
static void dispose_thread_data(void* data) {
thread_storage* tls = (thread_storage*)data;
JavaVM* jvm = tls->jvm;
JNIEnv* env;
int is_attached = (*jvm)->GetEnv(jvm, (void*)&env, JNI_VERSION_1_4) == JNI_OK;
jboolean detached = JNI_TRUE;
if (is_attached) {
if ((*jvm)->DetachCurrentThread(jvm) != 0) {
fprintf(stderr, "JNA: could not detach native thread (automatic)\n");
detached = JNI_FALSE;
}
}
if (tls->termination_flag && detached) {
*(tls->termination_flag) = JNI_TRUE;
}
free(data);
}
BOOL WINAPI DllMain(HINSTANCE hDLL, DWORD fdwReason, LPVOID lpvReserved) {
switch (fdwReason) {
...
case DLL_THREAD_DETACH: {
thread_storage* tls = (thread_storage*)TlsGetValue(tls_thread_data_key);
if (tls) {
dispose_thread_data(tls);
TlsSetValue(tls_thread_data_key, 0);
}
break;
}
...
return TRUE;
}The Invokation API tells explicitly
A native thread attached to the VM must call DetachCurrentThread() to detach itself BEFORE exiting.
The problem here is that you cannot consider a thread to detach as completely done. If at the moment JVM runs a safepoint synchronization the thread gets lock and stays awaiting until all other threads reach the safepoint (see the callstack). That still might work before Windows 10 introduced parallel algorithm of DLL loading/releasing, but now... I do not have an access to Windows source code, but ReactOS project shows us that LDR calls DllMain(..., DLL_THREAD_DETACH, ...) under taken critical section
NTSTATUS
NTAPI
LdrShutdownThread(VOID)
{
...
/* Get the Ldr Lock */
RtlEnterCriticalSection(&LdrpLoaderLock);
...
/* Start at the end */
ListHead = &Peb->Ldr->InInitializationOrderModuleList;
NextEntry = ListHead->Blink;
while (NextEntry != ListHead)
{
...
LdrpCallInitRoutine(EntryPoint,
LdrEntry->DllBase,
DLL_THREAD_DETACH,
NULL);
...
}
...
/* Free TLS */
LdrpFreeTls();
RtlLeaveCriticalSection(&LdrpLoaderLock);
...
}that locks LDRP workers, so other threads loading/releasing DLL's at the moment get locked just like
. 76 Id: 27f4.2de0 Suspend: 0 Teb: 00000000`0023e000 Unfrozen
Start: . 76 Id: 27f4.2de0 Suspend: 0 Teb: 00000000`0023e000 Unfrozen
Start: mswsock!SockAsyncThread (00007ffb`696d0460)mswsock!SockAsyncThread (00007ffb`696d0460)
Priority: 1 Priority class: 32 Affinity: fff
# Child-SP RetAddr Call Site
Priority: 1 Priority class: 32 Affinity: fff
# Child-SP RetAddr Call Site
00 00 00000000`415efd18 00007ffb`6de983f5 ntdll!NtWaitForSingleObject+0x14
00000000`415efd18 00007ffb`6de983f5 ntdll!NtWaitForSingleObject+0x14
01 01 00000000`415efd20 00007ffb`6de93338 ntdll!LdrpDrainWorkQueue+0x15d
00000000`415efd20 00007ffb`6de93338 ntdll!LdrpDrainWorkQueue+0x15d
02 02 00000000`415efd60 00007ffb`6deca33e ntdll!LdrShutdownThread+0x98
00000000`415efd60 00007ffb`6deca33e ntdll!LdrShutdownThread+0x98
03 03 00000000`415efe60 00007ffb`6a49992a ntdll!RtlExitUserThread+0x3e
00000000`415efe60 00007ffb`6a49992a ntdll!RtlExitUserThread+0x3e
04 04 00000000`415efea0 00007ffb`696d0569 KERNELBASE!FreeLibraryAndExitThread+0x4a
00000000`415efea0 00007ffb`696d0569 KERNELBASE!FreeLibraryAndExitThread+0x4a
05 05 00000000`415efed0 00007ffb`6b0b7974 mswsock!SockAsyncThread+0x109
00000000`415efed0 00007ffb`6b0b7974 mswsock!SockAsyncThread+0x109
06 06 00000000`415eff30 00007ffb`6deca2d1 kernel32!BaseThreadInitThunk+0x14
00000000`415eff30 00007ffb`6deca2d1 kernel32!BaseThreadInitThunk+0x14
07 07 00000000`415eff60 00000000`00000000 ntdll!RtlUserThreadStart+0x21
00000000`415eff60 00000000`00000000 ntdll!RtlUserThreadStart+0x21
.319 Id: 27f4.3e58 Suspend: 0 Teb: 00000000`54700000 Unfrozen
Start: .319 Id: 27f4.3e58 Suspend: 0 Teb: 00000000`54700000 Unfrozen
Start: ucrtbase!thread_start<unsigned int (__cdecl*)(void * __ptr64)> (00007ffb`6a092650)ucrtbase!thread_start<unsigned int (__cdecl*)(void * __ptr64)> (00007ffb`6a092650)
Priority: 0 Priority class: 32 Affinity: fff
# Child-SP RetAddr Call Site
Priority: 0 Priority class: 32 Affinity: fff
# Child-SP RetAddr Call Site
00 00 00000001`5204f898 00007ffb`6de983f5 ntdll!NtWaitForSingleObject+0x14
00000001`5204f898 00007ffb`6de983f5 ntdll!NtWaitForSingleObject+0x14
01 01 00000001`5204f8a0 00007ffb`6de935f7 ntdll!LdrpDrainWorkQueue+0x15d
00000001`5204f8a0 00007ffb`6de935f7 ntdll!LdrpDrainWorkQueue+0x15d
02 02 00000001`5204f8e0 00007ffb`6dee8625 ntdll!LdrpInitializeThread+0x8b
00000001`5204f8e0 00007ffb`6dee8625 ntdll!LdrpInitializeThread+0x8b
03 03 00000001`5204f9c0 00007ffb`6dee8203 ntdll!LdrpInitialize+0x409
00000001`5204f9c0 00007ffb`6dee8203 ntdll!LdrpInitialize+0x409
04 04 00000001`5204fa60 00007ffb`6dee81ae ntdll!LdrpInitialize+0x3b
00000001`5204fa60 00007ffb`6dee81ae ntdll!LdrpInitialize+0x3b
05 05 00000001`5204fa90 00000000`00000000 ntdll!LdrInitializeThunk+0xe
00000001`5204fa90 00000000`00000000 ntdll!LdrInitializeThunk+0xe
and cannot get to the safepoint rendezvous. We get the deadlock.
We provided a workaround to the customer but the changes are very tricky and risky and they touch JDK logic that stays immutable for years, so there is not a chance to push the changes to OpenJDK. Please fix the issue on your side. Thank you
Source: java-native-access/jna