My program looped in the phn_merge for more than 20 minutes without releasing the mutex lock ecache, then blocked all other threads.

Env info Jemalloc Version: 5.3.0 Os : Centos 7 C++ tool chain: C++ 921

1. The first captured pstack info:

Thread 237 (Thread 0x7f833f9e7700 (LWP 2938)): #0 phn_merge (cmp=, offset=40, phn1=0x7f839882fc80, phn0=0x7f839882fc80) at include/jemalloc/internal/ph.h:140 #1 phn_merge_siblings (cmp=, offset=40, phn=) at include/jemalloc/internal/ph.h:209 #2 ph_merge_children (cmp=, offset=40, phn=0x7f80b3775000) at include/jemalloc/internal/ph.h:244 #3 ph_remove (cmp=, offset=40, phn=phn@entry=0x7f80b3775000, ph=0x7f8398803d58, ph@entry=0x0) at include/jemalloc/internal/ph.h:389 #4 je_edata_heap_remove (ph=ph@entry=0x7f8398803d58, phn=phn@entry=0x7f8398831a00) at src/edata.c:6 #5 0x0000000003119349 in je_eset_remove (eset=0x7f8398803a98, edata=edata@entry=0x7f8398831a00) at src/eset.c:126 #6 0x000000000311c462 in extent_activate_locked (pac=, ecache=, edata=0x7f8398831a00, eset=, tsdn=0x7f833f9e6900) at src/extent.c:280 #7 extent_recycle_extract (ehooks=0x7f83988000c0, guarded=, alignment=64, size=102400, expand_edata=, ecache=0x7f8398803a28, pac=0x7f83988039f0, tsdn=0x7f833f9e6900) at src/extent.c:444 #8 extent_recycle (tsdn=tsdn@entry=0x7f833f9e6900, pac=pac@entry=0x7f83988039f0, ehooks=ehooks@entry=0x7f83988000c0, ecache=ecache@entry=0x7f8398803a28, expand_edata=expand_edata@entry=0x0, size=size@entry=102400, alignment=alignment@entry=64, zero=zero@entry=false, commit=commit@entry=0x7f833f9e1800, growing_retained=growing_retained@entry=false, guarded=guarded@entry=false) at src/extent.c:606 #9 0x000000000311c8cd in je_ecache_alloc (tsdn=tsdn@entry=0x7f833f9e6900, pac=pac@entry=0x7f83988039f0, ehooks=ehooks@entry=0x7f83988000c0, ecache=ecache@entry=0x7f8398803a28, expand_edata=expand_edata@entry=0x0, size=size@entry=102400, alignment=alignment@entry=64, zero=zero@entry=false, guarded=guarded@entry=false) at src/extent.c:87 #10 0x000000000312989d in pac_alloc_real (guarded=false, zero=, alignment=64, size=102400, ehooks=0x7f83988000c0, pac=0x7f83988039f0, tsdn=0x7f833f9e6900) at src/pac.c:116 #11 pac_alloc_impl (tsdn=0x7f833f9e6900, self=0x7f83988039f0, size=102400, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7f833f9e1920) at src/pac.c:178 #12 0x0000000003127f0e in pai_alloc (deferred_work_generated=0x7f833f9e1920, frequent_reuse=false, guarded=, zero=false, alignment=64, size=102400, self=0x7f83988039f0, tsdn=0x7f833f9e6900) at include/jemalloc/internal/pai.h:43 #13 je_pa_alloc (tsdn=tsdn@entry=0x7f833f9e6900, shard=shard@entry=0x7f83988039d8, size=size@entry=102400, alignment=alignment@entry=64, slab=slab@entry=false, szind=46, zero=zero@entry=false, guarded=false, deferred_work_generated=deferred_work_generated@entry=0x7f833f9e1920) at src/pa.c:139 #14 0x00000000030f074f in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7f833f9e6900, arena=arena@entry=0x7f8398801040, usize=usize@entry=98304, alignment=alignment@entry=64, zero=zero@entry=false) at src/arena.c:338 #15 0x0000000003125948 in je_large_palloc (zero=false, alignment=64, usize=98304, arena=0x7f8398801040, tsdn=0x7f833f9e6900) at src/large.c:37 #16 je_large_malloc (tsdn=tsdn@entry=0x7f833f9e6900, arena=, arena@entry=0x7f8398801040, usize=98304, zero=zero@entry=false) at src/large.c:17 #17 0x00000000030f3867 in je_arena_malloc_hard (tsdn=tsdn@entry=0x7f833f9e6900, arena=0x7f8398801040, arena@entry=0x0, size=size@entry=94176, ind=ind@entry=46, zero=zero@entry=false) at src/arena.c:1205#18 0x00000000030e96df in arena_malloc (slow_path=, tcache=0x7f833f9e6c58, zero=false, ind=46, size=94176, arena=0x0, tsdn=0x7f833f9e6900) at include/jemalloc/internal/arena_inlines_b.h:162 #19 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7f833f9e6c58, zero=false, ind=46, size=94176, tsdn=0x7f833f9e6900) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #20 imalloc_no_sample (sopts=, ind=46, usize=98304, size=94176, tsd=0x7f833f9e6900, dopts=) at src/jemalloc.c:2398 #21 imalloc_body (tsd=0x7f833f9e6900, dopts=, sopts=) at src/jemalloc.c:2573 #22 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #23 je_malloc_default (size=94176) at src/jemalloc.c:2722 #24 0x0000000003be9da5 in operator new(unsigned long) () ...

1. After 1 minute, the second pstack info:

Thread 237 (Thread 0x7f833f9e7700 (LWP 2938)): #0 0x000000000311602d in edata_cmp_summary_comp (b=..., a=...) at include/jemalloc/internal/edata.h:666 #1 edata_snad_comp (b=0x7f839882fc80, a=0x7f839882fc80) at include/jemalloc/internal/edata.h:679 #2 edata_heap_ph_cmp (b=0x7f839882fc80, a=0x7f839882fc80) at src/edata.c:6 #3 phn_merge (cmp=, offset=40, phn1=0x7f839882fc80, phn0=0x7f839882fc80) at include/jemalloc/internal/ph.h:142 #4 phn_merge_siblings (cmp=, offset=40, phn=) at include/jemalloc/internal/ph.h:209 #5 ph_merge_children (cmp=, offset=40, phn=0x7f80b3775000) at include/jemalloc/internal/ph.h:244 #6 ph_remove (cmp=, offset=40, phn=phn@entry=0x7f80b3775000, ph=0x7f8398803d58, ph@entry=0x0) at include/jemalloc/internal/ph.h:389 #7 je_edata_heap_remove (ph=ph@entry=0x7f8398803d58, phn=phn@entry=0x7f8398831a00) at src/edata.c:6 #8 0x0000000003119349 in je_eset_remove (eset=0x7f8398803a98, edata=edata@entry=0x7f8398831a00) at src/eset.c:126 #9 0x000000000311c462 in extent_activate_locked (pac=, ecache=, edata=0x7f8398831a00, eset=, tsdn=0x7f833f9e6900) at src/extent.c:280 #10 extent_recycle_extract (ehooks=0x7f83988000c0, guarded=, alignment=64, size=102400, expand_edata=, ecache=0x7f8398803a28, pac=0x7f83988039f0, tsdn=0x7f833f9e6900) at src/extent.c:444 #11 extent_recycle (tsdn=tsdn@entry=0x7f833f9e6900, pac=pac@entry=0x7f83988039f0, ehooks=ehooks@entry=0x7f83988000c0, ecache=ecache@entry=0x7f8398803a28, expand_edata=expand_edata@entry=0x0, size=size@entry=102400, alignment=alignment@entry=64, zero=zero@entry=false, commit=commit@entry=0x7f833f9e1800, growing_retained=growing_retained@entry=false, guarded=guarded@entry=false) at src/extent.c:606 #12 0x000000000311c8cd in je_ecache_alloc (tsdn=tsdn@entry=0x7f833f9e6900, pac=pac@entry=0x7f83988039f0, ehooks=ehooks@entry=0x7f83988000c0, ecache=ecache@entry=0x7f8398803a28, expand_edata=expand_edata@entry=0x0, size=size@entry=102400, alignment=alignment@entry=64, zero=zero@entry=false, guarded=guarded@entry=false) at src/extent.c:87 #13 0x000000000312989d in pac_alloc_real (guarded=false, zero=, alignment=64, size=102400, ehooks=0x7f83988000c0, pac=0x7f83988039f0, tsdn=0x7f833f9e6900) at src/pac.c:116#14 pac_alloc_impl (tsdn=0x7f833f9e6900, self=0x7f83988039f0, size=102400, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7f833f9e1920) at src/pac.c:178 #15 0x0000000003127f0e in pai_alloc (deferred_work_generated=0x7f833f9e1920, frequent_reuse=false, guarded=, zero=false, alignment=64, size=102400, self=0x7f83988039f0, tsdn=0x7f833f9e6900) at include/jemalloc/internal/pai.h:43 #16 je_pa_alloc (tsdn=tsdn@entry=0x7f833f9e6900, shard=shard@entry=0x7f83988039d8, size=size@entry=102400, alignment=alignment@entry=64, slab=slab@entry=false, szind=46, zero=zero@entry=false, guarded=false, deferred_work_generated=deferred_work_generated@entry=0x7f833f9e1920) at src/pa.c:139 #17 0x00000000030f074f in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7f833f9e6900, arena=arena@entry=0x7f8398801040, usize=usize@entry=98304, alignment=alignment@entry=64, zero=zero@entry=false) at src/arena.c:338#18 0x0000000003125948 in je_large_palloc (zero=false, alignment=64, usize=98304, arena=0x7f8398801040, tsdn=0x7f833f9e6900) at src/large.c:37 #19 je_large_malloc (tsdn=tsdn@entry=0x7f833f9e6900, arena=, arena@entry=0x7f8398801040, usize=98304, zero=zero@entry=false) at src/large.c:17 #20 0x00000000030f3867 in je_arena_malloc_hard (tsdn=tsdn@entry=0x7f833f9e6900, arena=0x7f8398801040, arena@entry=0x0, size=size@entry=94176, ind=ind@entry=46, zero=zero@entry=false) at src/arena.c:1205 #21 0x00000000030e96df in arena_malloc (slow_path=, tcache=0x7f833f9e6c58, zero=false, ind=46, size=94176, arena=0x0, tsdn=0x7f833f9e6900) at include/jemalloc/internal/arena_inlines_b.h:162 #22 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7f833f9e6c58, zero=false, ind=46, size=94176, tsdn=0x7f833f9e6900) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #23 imalloc_no_sample (sopts=, ind=46, usize=98304, size=94176, tsd=0x7f833f9e6900, dopts=) at src/jemalloc.c:2398 #24 imalloc_body (tsd=0x7f833f9e6900, dopts=, sopts=) at src/jemalloc.c:2573 #25 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #26 je_malloc_default (size=94176) at src/jemalloc.c:2722 #27 0x0000000003be9da5 in operator new(unsigned long) () ...