assert "expandable_segments:True" not in conf报错

使用官方的docker镜像跑以下脚本出错了

镜像

https://github.com/OpenRLHF/OpenRLHF/blob/main/dockerfile/Dockerfile

脚本

set -x

ray job submit --address="http://127.0.0.1:8265"
--runtime-env-json='{"working_dir": "./"}'
-- python3 -m openrlhf.cli.train_ppo_ray
--colocate_all_models
--ref_num_nodes 2
--ref_num_gpus_per_node 8
--reward_num_nodes 1
--reward_num_gpus_per_node 8
--actor_num_nodes 2
--actor_num_gpus_per_node 8
--vllm_num_engines 16
--vllm_tensor_parallel_size 1
--vllm_gpu_memory_utilization 0.6
--init_kl_coef 1e-3
--gamma 1.0
--use_kl_loss
--kl_estimator k3
--advantage_estimator group_norm
--pretrain /mnt/geminigmceph/base_model/Qwen2.5-1.5B-Instruct
--remote_rm_url openrlhf/countdown_reward_fn.py
--save_path ckpt/1234
--ckpt_path ckpt/1234
--save_hf_ckpt
--micro_train_batch_size 1
--train_batch_size 16
--micro_rollout_batch_size 1
--rollout_batch_size 16
--n_samples_per_prompt 2
--max_epochs 1
--prompt_max_len 8
--max_samples 100000
--generate_max_len 8
--zero_stage 3
--bf16
--actor_learning_rate 5e-7
--critic_learning_rate 9e-6
--prompt_data data/openrlhf/test.json
--input_key prompt
--label_key groud_truth
--apply_chat_template
--normalize_reward
--gradient_checkpointing
--packing_samples
--vllm_sync_backend nccl
--enforce_eager
--vllm_enable_sleep
--deepspeed_enable_sleep

错误日志

Exception raised in creation task: The actor died because of an error raised in its creation task, ray::LLMRayActor.init() (pid=158927, ip=29.213.196.220, actor_id=b231c5480456e835076e38db06000000, repr=<openrlhf.trainer.ray.vllm_engine.LLMRayActor object at 0x7f8c9d7dc3d0>) (LLMRayActor pid=158927) File "/tmp/ray/session_2025-05-12_09-23-52_967155_81627/runtime_resources/working_dir_files/_ray_pkg_a60d1e73e6e66eb8/openrlhf/trainer/ray/vllm_engine.py", line 60, in init (LLMRayActor pid=158927) self.llm = vllm.LLM(*args, **kwargs) (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/utils.py", line 1161, in inner (LLMRayActor pid=158927) return fn(*args, **kwargs) (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/llm.py", line 247, in init (LLMRayActor pid=158927) self.llm_engine = LLMEngine.from_engine_args( (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/v1/engine/llm_engine.py", line 138, in from_engine_args (LLMRayActor pid=158927) return cls(vllm_config=vllm_config, (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/v1/engine/llm_engine.py", line 92, in init (LLMRayActor pid=158927) self.engine_core = EngineCoreClient.make_client( (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/v1/engine/core_client.py", line 75, in make_client (LLMRayActor pid=158927) return InprocClient(vllm_config, executor_class, log_stats) (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/v1/engine/core_client.py", line 198, in init (LLMRayActor pid=158927) self.engine_core = EngineCore(*args, **kwargs) (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/v1/engine/core.py", line 64, in init (LLMRayActor pid=158927) self.model_executor = executor_class(vllm_config) (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/executor/executor_base.py", line 52, in init (LLMRayActor pid=158927) self._init_executor() (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/executor/uniproc_executor.py", line 47, in _init_executor (LLMRayActor pid=158927) self.collective_rpc("load_model") (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/executor/uniproc_executor.py", line 56, in collective_rpc (LLMRayActor pid=158927) answer = run_method(self.driver_worker, method, args, kwargs) (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/utils.py", line 2456, in run_method (LLMRayActor pid=158927) return func(*args, **kwargs) (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/v1/worker/gpu_worker.py", line 153, in load_model (LLMRayActor pid=158927) allocator = CuMemAllocator.get_instance() (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/device_allocator/cumem.py", line 141, in get_instance (LLMRayActor pid=158927) CuMemAllocator.instance = CuMemAllocator() (LLMRayActor pid=158927) File "/usr/local/lib/python3.10/dist-packages/vllm/device_allocator/cumem.py", line 146, in init (LLMRayActor pid=158927) assert "expandable_segments:True" not in conf,
(LLMRayActor pid=158927) AssertionError: Expandable segments are not compatible with memory pool. Please track https://github.com/pytorch/pytorch/issues/147851 for the latest updates.

求问大佬怎么解决呢