create_collection with local embedding model aborts server when CUDA execution provider fails to init; committed write becomes a crash loop

#!/bin/bash
# Issue: create_collection with a local embedding model aborts the whole server
#        when the ONNX CUDA execution provider fails to initialize, and the
#        committed write turns into a restart crash loop.
# Typesense Version: 30.2
# Description:
#   On a server where the onnxruntime CUDA provider libs are present (as
#   installed by the official typesense-gpu-deps package) but CUDA is not
#   usable (no GPU / broken driver), creating a collection with a local
#   embedding model field (e.g. ts/e5-small) aborts the typesense-server
#   process: the exception thrown inside libonnxruntime_providers_cuda.so
#   cannot unwind across the shared-library boundary into the statically
#   linked binary, so the process dies with SIGABRT
#   ("Typesense 30.2 is terminating abruptly.").
#   Because the create_collection write is already committed to the Raft log,
#   the crash recurs on replay until the skip_index mechanism cancels it, and
#   every NEW create with a local embed field crashes the server again.
#
#   Case 1 (control): stock typesense/typesense:30.2, no CUDA libs.
#                     Create succeeds (CPU fallback), server stays up.
#   Case 2 (bug):     same image + gpu-deps provider libs + CUDA 11 runtime
#                     libs, no GPU. Same create kills the server.
#   Case 3 (loop):    restart on the same data dir: server skips the poison
#                     write (collection is silently dropped) and a second
#                     create with the same embed field kills it again.
#
# NOTE: first run downloads ~50 MB (typesense-gpu-deps deb) and pulls the
#       nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04 image (~3 GB) to source
#       the CUDA 11 runtime libs. Both are cached under ./cache for reruns.

set -e

# ============================================================================
# CONFIGURATION
# ============================================================================

TYPESENSE_API_KEY=xyz
PORT=8231
TYPESENSE_HOST=http://localhost:${PORT}
VERSION=30.2
CONTAINER_NAME=typesense-issue-cuda-ep-abort
DATA_DIR_CTRL=$(pwd)/typesense-data-${CONTAINER_NAME}-ctrl
DATA_DIR_BUG=$(pwd)/typesense-data-${CONTAINER_NAME}-bug
CACHE_DIR=$(pwd)/cache
GPU_DEPS_DEB_URL="https://dl.typesense.org/releases/${VERSION}/typesense-gpu-deps-${VERSION}-amd64.deb"
CUDA_IMAGE="nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04"

# Windows (git-bash) path handling for docker -v
if command -v cygpath > /dev/null 2>&1; then
  hostpath() { cygpath -m "$1"; }
  export MSYS_NO_PATHCONV=1
else
  hostpath() { echo "$1"; }
fi

EMBED_SCHEMA() {
  # $1 = collection name
  cat <<EOF
{"name":"$1","fields":[
  {"name":"title","type":"string"},
  {"name":"embedding","type":"float[]","embed":{"from":["title"],"model_config":{"model_name":"ts/e5-small"}}}
]}
EOF
}

# ============================================================================
# CLEANUP (cache/ is kept intentionally: deb + extracted CUDA libs)
# ============================================================================

cleanup() {
  echo ""
  echo "=== Cleanup ==="
  docker rm -f ${CONTAINER_NAME} 2>/dev/null || true
  rm -rf "${DATA_DIR_CTRL}" "${DATA_DIR_BUG}"
  echo "Cleanup complete (cache/ kept for reruns)"
}
trap cleanup EXIT

wait_for_health() {
  local max_wait=${1:-90}
  local count=0
  while [ $count -lt $max_wait ]; do
    if curl -s -m 2 "${TYPESENSE_HOST}/health" 2>/dev/null | grep -q '"ok":true'; then
      return 0
    fi
    sleep 1
    count=$((count + 1))
  done
  echo "WARNING: server not healthy after ${max_wait}s"
  return 1
}

wait_for_collection() {
  local collection=$1
  local max_wait=${2:-30}
  local count=0
  while [ $count -lt $max_wait ]; do
    if curl -s -o /dev/null -w "%{http_code}" "${TYPESENSE_HOST}/collections/${collection}" \
      -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" 2>/dev/null | grep -q "200"; then
      return 0
    fi
    sleep 1
    count=$((count + 1))
  done
  echo "WARNING: Collection '${collection}' not ready after ${max_wait}s"
  return 1
}

wait_for_container_death() {
  local max_wait=${1:-300}
  local count=0
  while [ $count -lt $max_wait ]; do
    if [ "$(docker inspect -f '{{.State.Running}}' ${CONTAINER_NAME} 2>/dev/null)" = "false" ]; then
      return 0
    fi
    sleep 1
    count=$((count + 1))
  done
  return 1
}

# ============================================================================
# FETCH GPU DEPS (cached)
# ============================================================================

mkdir -p "${CACHE_DIR}"
HP_CACHE=$(hostpath "${CACHE_DIR}")

if [ ! -f "${CACHE_DIR}/extract/usr/lib/libonnxruntime_providers_cuda.so" ]; then
  echo "=== Downloading typesense-gpu-deps ${VERSION} (~50 MB) ==="
  curl -sL -o "${CACHE_DIR}/gpu-deps.deb" "${GPU_DEPS_DEB_URL}"
  docker run --rm -v "${HP_CACHE}:/work" debian:bookworm-slim \
    bash -c "dpkg-deb -x /work/gpu-deps.deb /work/extract" > /dev/null
fi

if [ ! -f "${CACHE_DIR}/cuda-libs/libcublasLt.so.11" ]; then
  echo "=== Extracting CUDA 11 runtime libs from ${CUDA_IMAGE} (~3 GB pull on first run) ==="
  mkdir -p "${CACHE_DIR}/cuda-libs"
  docker run --rm -v "${HP_CACHE}/cuda-libs:/out" ${CUDA_IMAGE} \
    bash -c "cp -L /usr/local/cuda/lib64/libcublasLt.so.11 /usr/local/cuda/lib64/libcublas.so.11 \
                   /usr/local/cuda/lib64/libcufft.so.10 /usr/local/cuda/lib64/libcudart.so.11.0 \
                   /usr/lib/x86_64-linux-gnu/libcudnn.so.8 /out/" > /dev/null
fi

# ============================================================================
# CASE 1 (control): stock image, no CUDA libs -> create succeeds, CPU fallback
# ============================================================================

echo ""
echo "=== Case 1 (control): stock ${VERSION}, no CUDA provider libs ==="
docker rm -f ${CONTAINER_NAME} 2>/dev/null || true
mkdir -p "${DATA_DIR_CTRL}"
docker run -d --name ${CONTAINER_NAME} -p ${PORT}:8108 \
  -v "$(hostpath "${DATA_DIR_CTRL}"):/data" \
  typesense/typesense:${VERSION} --data-dir /data --api-key=${TYPESENSE_API_KEY} > /dev/null
wait_for_health

CASE1_CODE=$(curl -s -m 300 -o /dev/null -w "%{http_code}" "${TYPESENSE_HOST}/collections" -X POST \
  -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" -H "Content-Type: application/json" \
  -d "$(EMBED_SCHEMA items)" || true)
wait_for_collection items || true
CASE1_RUNNING=$(docker inspect -f '{{.State.Running}}' ${CONTAINER_NAME})
echo "Case 1: create returned HTTP ${CASE1_CODE}, server running=${CASE1_RUNNING}"
docker logs ${CONTAINER_NAME} 2>&1 | grep -m1 "ONNX shared libs: off" || true

docker rm -f ${CONTAINER_NAME} > /dev/null

# ============================================================================
# CASE 2 (bug): CUDA provider libs mounted, no GPU -> same create aborts server
# ============================================================================

echo ""
echo "=== Case 2 (bug): + gpu-deps provider libs + CUDA 11 runtime libs, no GPU ==="
mkdir -p "${DATA_DIR_BUG}"
# reuse the already-downloaded model so case 2 does not re-download it
if [ -d "${DATA_DIR_CTRL}/models" ]; then
  cp -r "${DATA_DIR_CTRL}/models" "${DATA_DIR_BUG}/models" 2>/dev/null || true
fi
docker run -d --name ${CONTAINER_NAME} -p ${PORT}:8108 \
  -v "$(hostpath "${DATA_DIR_BUG}"):/data" \
  -v "${HP_CACHE}/extract/usr/lib/libonnxruntime_providers_shared.so:/usr/lib/libonnxruntime_providers_shared.so:ro" \
  -v "${HP_CACHE}/extract/usr/lib/libonnxruntime_providers_cuda.so:/usr/lib/libonnxruntime_providers_cuda.so:ro" \
  -v "${HP_CACHE}/cuda-libs:/usr/lib/cuda-rt:ro" \
  -e LD_LIBRARY_PATH=/usr/lib/cuda-rt \
  typesense/typesense:${VERSION} --data-dir /data --api-key=${TYPESENSE_API_KEY} > /dev/null
wait_for_health

curl -s -m 300 "${TYPESENSE_HOST}/collections" -X POST \
  -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" -H "Content-Type: application/json" \
  -d "$(EMBED_SCHEMA items)" > /dev/null || true
wait_for_container_death 300 && CASE2_DEAD=1 || CASE2_DEAD=0
CASE2_EXIT=$(docker inspect -f '{{.State.ExitCode}}' ${CONTAINER_NAME})
CASE2_TRACE=$(docker logs ${CONTAINER_NAME} 2>&1 | grep -c "libonnxruntime_providers_cuda.so" || true)
CASE2_ABRUPT=$(docker logs ${CONTAINER_NAME} 2>&1 | grep -c "terminating abruptly" || true)
echo "Case 2: server dead=${CASE2_DEAD} exit_code=${CASE2_EXIT}, stack frames in libonnxruntime_providers_cuda.so=${CASE2_TRACE}, 'terminating abruptly'=${CASE2_ABRUPT}"
docker logs ${CONTAINER_NAME} 2>&1 | grep -E "CreateProvider|terminating abruptly|Aborted" | tail -4

# ============================================================================
# CASE 3 (loop): restart same data dir -> poison write skipped (collection
#                silently dropped), next create with embed field crashes again
# ============================================================================

echo ""
echo "=== Case 3 (loop): restart on same data dir ==="
docker start ${CONTAINER_NAME} > /dev/null
wait_for_health
CASE3_SKIPS=$(docker logs ${CONTAINER_NAME} 2>&1 | grep -c "Skipping write log index" || true)
CASE3_COL=$(curl -s -o /dev/null -w "%{http_code}" "${TYPESENSE_HOST}/collections/items" \
  -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" || true)
echo "Case 3: after restart, 'Skipping write log index' lines=${CASE3_SKIPS}, GET /collections/items=${CASE3_COL} (write cancelled)"

curl -s -m 300 "${TYPESENSE_HOST}/collections" -X POST \
  -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" -H "Content-Type: application/json" \
  -d "$(EMBED_SCHEMA items2)" > /dev/null || true
wait_for_container_death 300 && CASE3_DEAD=1 || CASE3_DEAD=0
echo "Case 3: second create with embed field -> server dead again=${CASE3_DEAD}"

# ============================================================================
# VERDICT
# ============================================================================

echo ""
echo "=== SUMMARY ==="
echo "Case 1 (no CUDA libs):     create HTTP=${CASE1_CODE}, server running=${CASE1_RUNNING}   (expected: 201, true)"
echo "Case 2 (CUDA libs, no GPU): server dead=${CASE2_DEAD}, cuda frames=${CASE2_TRACE}, abrupt=${CASE2_ABRUPT}   (expected: 1, >0, >0)"
echo "Case 3 (restart):          skips=${CASE3_SKIPS}, items=${CASE3_COL}, dead again=${CASE3_DEAD}   (expected: >0, 404, 1)"

if [ "${CASE1_CODE}" = "201" ] && [ "${CASE1_RUNNING}" = "true" ] && \
   [ "${CASE2_DEAD}" = "1" ] && [ "${CASE2_TRACE}" -gt 0 ] && [ "${CASE2_ABRUPT}" -gt 0 ] && \
   [ "${CASE3_SKIPS}" -gt 0 ] && [ "${CASE3_COL}" = "404" ] && [ "${CASE3_DEAD}" = "1" ]; then
  echo ""
  echo "BUG REPRODUCED: local embed model create aborts the server when CUDA EP init fails, and the committed write crash-loops on restart."
  exit 0
else
  echo ""
  echo "Reproducer did not hit the expected state"
  exit 1
fi

E ... backward.hpp:4200] #7  Object "/usr/lib/libonnxruntime_providers_cuda.so", in onnxruntime::CUDAProviderFactory::CreateProvider() [clone .cold]
E ... backward.hpp:4200] #6  Object "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", in _Unwind_Resume
E ... backward.hpp:4200] #2  Object "/usr/lib/x86_64-linux-gnu/libc.so.6", in abort
Aborted (Signal sent by tkill() 1 0)
E ... typesense_server.cpp:171] Typesense 30.2 is terminating abruptly.

E ... Skipping write log index 7 which seems to have triggered a crash previously.

#3 Object "/usr/lib/libonnxruntime_providers_cuda.so", in onnxruntime::CUDAExecutionProvider::GetProviderOptions[abi:cxx11]() const
#2 Object "/usr/lib/libonnxruntime_providers_cuda.so", in onnxruntime::CUDAExecutionProviderInfo::ToProviderOptions[abi:cxx11](...)

{
  "name": "items",
  "fields": [
    {"name": "title", "type": "string"},
    {"name": "embedding", "type": "float[]", "embed": {"from": ["title"], "model_config": {"model_name": "ts/e5-small"}}}
  ]
}