[Question]: TimeoutError: Embedding func: Worker execution timeout after 60s

me=2026-02-05T17:20:37.468+02:00 level=INFO source=server.go:392 msg="starting runner" cmd="/home/shafay/libs/ollama/bin/ollama runner --ollama-engine --port 37955"

time=2026-02-05T17:20:40.468+02:00 level=INFO source=runner.go:449 msg="failure during GPU discovery" OLLAMA_LIBRARY_PATH="[/home/shafay/libs/ollama/lib/ollama /home/shafay/libs/ollama/lib/ollama/cuda_v12]" extra_envs=map[] error="failed to finish discovery before timeout"
time=2026-02-05T17:20:40.468+02:00 level=WARN source=runner.go:341 msg="unable to refresh free memory, using old values"
time=2026-02-05T17:20:41.114+02:00 level=WARN source=server.go:167 msg="requested context size too large for model" num_ctx=8192 n_ctx_train=2048
time=2026-02-05T17:20:41.114+02:00 level=WARN source=server.go:202 msg="flash attention enabled but not supported by model"
time=2026-02-05T17:20:41.114+02:00 level=INFO source=server.go:392 msg="starting runner" cmd="/home/shafay/libs/ollama/bin/ollama runner --ollama-engine --model /home/shafay/.ollama/models/blobs/sha256-0800cbac9c2064dde519420e75e512a83cb360de3ad5df176185dc69652fc515 --port 36327"
time=2026-02-05T17:20:41.115+02:00 level=INFO source=sched.go:443 msg="system memory" total="251.5 GiB" free="246.3 GiB" free_swap="976.0 MiB"
time=2026-02-05T17:20:41.115+02:00 level=INFO source=sched.go:450 msg="gpu memory" id=GPU-d8c577fe-c1e3-bed1-7fde-1997b2683130 library=CUDA available="42.9 GiB" free="43.3 GiB" minimum="457.0 MiB" overhead="0 B"
time=2026-02-05T17:20:41.121+02:00 level=INFO source=sched.go:450 msg="gpu memory" id=GPU-484b6a42-aba2-d87e-445b-f501f700bc1f library=CUDA available="43.5 GiB" free="44.0 GiB" minimum="457.0 MiB" overhead="0 B"
time=2026-02-05T17:20:41.121+02:00 level=INFO source=server.go:702 msg="loading model" "model layers"=25 requested=-1
time=2026-02-05T17:20:41.128+02:00 level=INFO source=runner.go:1398 msg="starting ollama engine"
time=2026-02-05T17:20:41.128+02:00 level=INFO source=runner.go:1433 msg="Server listening on 127.0.0.1:36327"
time=2026-02-05T17:20:41.148+02:00 level=INFO source=runner.go:1271 msg=load request="{Operation:fit LoraPath:[] Parallel:1 BatchSize:2048 FlashAttention:false KvSize:2048 KvCacheType: NumThreads:24 GPULayers:25[ID:GPU-484b6a42-aba2-d87e-445b-f501f700bc1f Layers:25(0..24)] MultiUserCache:false ProjectorPath: MainGPU:0 UseMmap:false}"
time=2026-02-05T17:20:41.187+02:00 level=INFO source=ggml.go:136 msg="" architecture=gemma3 file_type=BF16 name="Embeddinggemma 300M" description="" num_tensors=316 num_key_values=37
load_backend: loaded CPU backend from /home/shafay/libs/ollama/lib/ollama/libggml-cpu-icelake.so
ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
ggml_cuda_init: found 2 CUDA devices:
  Device 0: NVIDIA L40S, compute capability 8.9, VMM: yes, ID: GPU-d8c577fe-c1e3-bed1-7fde-1997b2683130
  Device 1: NVIDIA L40S, compute capability 8.9, VMM: yes, ID: GPU-484b6a42-aba2-d87e-445b-f501f700bc1f
load_backend: loaded CUDA backend from /home/shafay/libs/ollama/lib/ollama/cuda_v12/libggml-cuda.so
time=2026-02-05T17:20:44.175+02:00 level=INFO source=ggml.go:104 msg=system CPU.0.SSE3=1 CPU.0.SSSE3=1 CPU.0.AVX=1 CPU.0.AVX2=1 CPU.0.F16C=1 CPU.0.FMA=1 CPU.0.BMI2=1 CPU.0.AVX512=1 CPU.0.AVX512_VBMI=1 CPU.0.AVX512_VNNI=1 CPU.0.LLAMAFILE=1 CPU.1.LLAMAFILE=1 CUDA.0.ARCHS=500,520,600,610,700,750,800,860,890,900,1200 CUDA.0.USE_GRAPHS=1 CUDA.0.PEER_MAX_BATCH_SIZE=128 CUDA.1.ARCHS=500,520,600,610,700,750,800,860,890,900,1200 CUDA.1.USE_GRAPHS=1 CUDA.1.PEER_MAX_BATCH_SIZE=128 compiler=cgo(gcc)
time=2026-02-05T17:20:44.457+02:00 level=INFO source=runner.go:1271 msg=load request="{Operation:alloc LoraPath:[] Parallel:1 BatchSize:2048 FlashAttention:false KvSize:2048 KvCacheType: NumThreads:24 GPULayers:25[ID:GPU-484b6a42-aba2-d87e-445b-f501f700bc1f Layers:25(0..24)] MultiUserCache:false ProjectorPath: MainGPU:0 UseMmap:false}"
time=2026-02-05T17:20:44.511+02:00 level=INFO source=runner.go:1271 msg=load request="{Operation:commit LoraPath:[] Parallel:1 BatchSize:2048 FlashAttention:false KvSize:2048 KvCacheType: NumThreads:24 GPULayers:25[ID:GPU-484b6a42-aba2-d87e-445b-f501f700bc1f Layers:25(0..24)] MultiUserCache:false ProjectorPath: MainGPU:0 UseMmap:false}"
time=2026-02-05T17:20:44.511+02:00 level=INFO source=ggml.go:482 msg="offloading 24 repeating layers to GPU"
time=2026-02-05T17:20:44.511+02:00 level=INFO source=ggml.go:489 msg="offloading output layer to GPU"
time=2026-02-05T17:20:44.511+02:00 level=INFO source=ggml.go:494 msg="offloaded 25/25 layers to GPU"
time=2026-02-05T17:20:44.516+02:00 level=INFO source=device.go:240 msg="model weights" device=CUDA1 size="586.8 MiB"
time=2026-02-05T17:20:44.516+02:00 level=INFO source=device.go:245 msg="model weights" device=CPU size="384.0 MiB"
time=2026-02-05T17:20:44.516+02:00 level=INFO source=device.go:262 msg="compute graph" device=CUDA1 size="80.0 MiB"
time=2026-02-05T17:20:44.516+02:00 level=INFO source=device.go:267 msg="compute graph" device=CPU size="6.0 MiB"
time=2026-02-05T17:20:44.516+02:00 level=INFO source=device.go:272 msg="total memory" size="1.0 GiB"
time=2026-02-05T17:20:44.516+02:00 level=INFO source=sched.go:517 msg="loaded runners" count=1
time=2026-02-05T17:20:44.516+02:00 level=INFO source=server.go:1294 msg="waiting for llama runner to start responding"
time=2026-02-05T17:20:44.520+02:00 level=INFO source=server.go:1328 msg="waiting for server to become available" status="llm server loading model"
time=2026-02-05T17:20:44.777+02:00 level=INFO source=server.go:1332 msg="llama runner started in 3.66 seconds"
time=2026-02-05T17:22:21.007+02:00 level=INFO source=server.go:1640 msg="aborting embedding request due to client closing the connection"
time=2026-02-05T17:22:21.007+02:00 level=INFO source=server.go:1640 msg="aborting embedding request due to client closing the connection"
time=2026-02-05T17:22:21.007+02:00 level=INFO source=server.go:1640 msg="aborting embedding request due to client closing the connection"
time=2026-02-05T17:22:21.007+02:00 level=INFO source=server.go:1640 msg="aborting embedding request due to client closing the connection"
time=2026-02-05T17:22:21.007+02:00 level=INFO source=server.go:1640 msg="aborting embedding request due to client closing the connection"
time=2026-02-05T17:22:21.007+02:00 level=INFO source=server.go:1640 msg="aborting embedding request due to client closing the connection"
time=2026-02-05T17:22:21.007+02:00 level=INFO source=server.go:1640 msg="aborting embedding request due to client closing the connection"
[GIN] 2026/02/05 - 17:22:21 | 500 |         1m44s |       127.0.0.1 | POST     "/api/embed"

INFO: 10.10.9.34:34130 - "POST /documents/upload HTTP/1.1" 200

INFO: Successfully extracted and enqueued file: LLM for Data Management - 3685800.3685838.pdf
INFO: Processing 1 document(s)
INFO: Extracting stage 1/1: LLM for Data Management - 3685800.3685838.pdf
INFO: Processing d-id: doc-399ffc6654e4f90b75161028fc4f6e06
WARNING: Embedding func: Worker timeout for task 140199239737680_5717800.586878799 after 60s
ERROR: Traceback (most recent call last):
  File "/home/shafay/ragapp/.venv/lib/python3.13/site-packages/lightrag/utils.py", line 1016, in wait_func
    return await future
           ^^^^^^^^^^^^
lightrag.utils.WorkerTimeoutError: Worker execution timeout after 60s

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/shafay/ragapp/.venv/lib/python3.13/site-packages/lightrag/lightrag.py", line 1895, in process_document
    await asyncio.gather(*first_stage_tasks)
  File "/home/shafay/ragapp/.venv/lib/python3.13/site-packages/lightrag/kg/nano_vector_db_impl.py", line 124, in upsert
    embeddings_list = await asyncio.gather(*embedding_tasks)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/shafay/ragapp/.venv/lib/python3.13/site-packages/lightrag/utils.py", line 503, in __call__
    result = await self.func(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/shafay/ragapp/.venv/lib/python3.13/site-packages/lightrag/utils.py", line 1042, in wait_func
    raise TimeoutError(f"{queue_name}: {str(e)}")
TimeoutError: Embedding func: Worker execution timeout after 60s

ERROR: Failed to extract document 1/1: LLM for Data Management - 3685800.3685838.pdf
INFO: Enqueued document processing pipeline stopped