2 nodes sft Qwen3.5-35B-A3B error FlashAttnVarlenFunc.apply ValueError: basic_string::_M_create

Checklist / 检查清单

• I have searched existing issues, and this is a new bug report. / 我已经搜索过现有的 issues，确认这是一个新的 bug report。

Bug Description / Bug 描述

I ran this experiment on 2 * 8H200 After weight is loaded, it failed at the first step of training log is below:

[rank7]: Traceback (most recent call last): [rank7]: File "/personal/ms-swift/swift/cli/_megatron/sft.py", line 7, in [rank7]: megatron_sft_main() [rank7]: File "/personal/ms-swift/swift/megatron/pipelines/train/sft.py", line 97, in megatron_sft_main [rank7]: return MegatronSft(args).main() [rank7]: File "/personal/ms-swift/swift/pipelines/base.py", line 52, in main [rank7]: result = self.run() [rank7]: File "/personal/ms-swift/swift/megatron/pipelines/train/sft.py", line 72, in run [rank7]: trainer.train(train_dataset, val_dataset) [rank7]: File "/personal/ms-swift/swift/megatron/trainers/base.py", line 636, in train [rank7]: metrics, grad_norm, update_successful = self.train_step(train_data_iterator) [rank7]: File "/personal/ms-swift/swift/megatron/trainers/base.py", line 857, in train_step [rank7]: metrics = forward_backward_func( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/pipeline_parallel/schedules.py", line 2159, in forward_backward_pipelining_without_interleaving [rank7]: output_tensor, num_tokens = forward_step( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/pipeline_parallel/schedules.py", line 423, in forward_step [rank7]: output_tensor, loss_func = forward_step_func(data_iterator, model) [rank7]: File "/personal/ms-swift/swift/megatron/trainers/trainer.py", line 124, in forward_step [rank7]: output_tensor = model(**data) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl [rank7]: return self._call_impl(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl [rank7]: return forward_call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/distributed/data_parallel_base.py", line 22, in forward [rank7]: return self.module(*inputs, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl [rank7]: return self._call_impl(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl [rank7]: return forward_call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/module.py", line 489, in forward [rank7]: outputs = self.module(*inputs, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl [rank7]: return self._call_impl(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl [rank7]: return forward_call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/mcore_bridge/model/mm_gpt_model.py", line 97, in forward [rank7]: return self.language_model( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl [rank7]: return self._call_impl(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl [rank7]: return forward_call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/mcore_bridge/model/gpt_model.py", line 331, in forward [rank7]: hidden_states = self.decoder( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/transformer_block.py", line 619, in call [rank7]: return super().call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/module.py", line 352, in call [rank7]: return super().call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl [rank7]: return self._call_impl(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl [rank7]: return forward_call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/transformer_block.py", line 736, in forward [rank7]: hidden_states = self._checkpointed_forward( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/transformer_block.py", line 538, in _checkpointed_forward [rank7]: hidden_states, context = checkpoint_handler( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/transformer_block.py", line 521, in checkpoint_handler [rank7]: return tensor_parallel.checkpoint( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/tensor_parallel/random.py", line 576, in checkpoint [rank7]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/autograd/function.py", line 576, in apply [rank7]: return super().apply(*args, **kwargs) # type: ignore[misc] [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/tensor_parallel/random.py", line 517, in forward [rank7]: outputs = run_function(*args) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/transformer_block.py", line 489, in custom_forward [rank7]: hidden_states, context = layer( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/transformer_layer.py", line 1217, in call [rank7]: return super().call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/module.py", line 352, in call [rank7]: return super().call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl [rank7]: return self._call_impl(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl [rank7]: return forward_call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/mcore_bridge/patcher.py", line 577, in forward [rank7]: hidden_states, context = self._forward_attention(*_args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/transformer_layer.py", line 597, in _forward_attention [rank7]: attention_output_with_bias = self.self_attention( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl [rank7]: return self._call_impl(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl [rank7]: return forward_call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/transformer/attention.py", line 1150, in forward [rank7]: core_attn_out = apply_module(self.core_attention)( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl [rank7]: return self._call_impl(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl [rank7]: return forward_call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/megatron/core/extensions/transformer_engine.py", line 1411, in forward [rank7]: core_attn_out = super().forward(query, key, value, attention_mask, **_fa_kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/transformer_engine/pytorch/jit.py", line 67, in wrapper [rank7]: return disabled_f(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/_dynamo/external_utils.py", line 198, in nonrecursive_disable_wrapper [rank7]: return fn(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/transformer_engine/pytorch/attention/dot_product_attention/dot_product_attention.py", line 1449, in forward [rank7]: return self.flash_attention( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl [rank7]: return self._call_impl(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl [rank7]: return forward_call(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/transformer_engine/pytorch/attention/dot_product_attention/backends.py", line 1068, in forward [rank7]: output = func( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/flash_attn_3/flash_attn_interface.py", line 913, in flash_attn_varlen_func [rank7]: return FlashAttnVarlenFunc.apply( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/autograd/function.py", line 576, in apply [rank7]: return super().apply(*args, **kwargs) # type: ignore[misc] [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/flash_attn_3/flash_attn_interface.py", line 672, in forward [rank7]: out, softmax_lse, *rest = _flash_attn_forward( [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/_library/custom_ops.py", line 681, in call [rank7]: return self._opoverload(*args, **kwargs) [rank7]: File "/root/miniconda3/lib/python3.10/site-packages/torch/_ops.py", line 829, in call [rank7]: return self._op(*args, **kwargs) [rank7]: ValueError: basic_string::_M_create

How to Reproduce / 如何复现

PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
NNODES=2
NODE_RANK=0
MASTER_ADDR=127.0.0.1
MASTER_PORT=29500
NPROC_PER_NODE=8
megatron sft
--model /personal/Qwen3.5-35B-A3B
--save_safetensors true
--dataset /personal/OpenHermes-2.5
--load_from_cache_file true
--split_dataset_ratio 0.01
--tensor_model_parallel_size 8
--pipeline_model_parallel_size 2
--expert_model_parallel_size 8
--micro_batch_size 1
--global_batch_size 16
--recompute_granularity full
--recompute_method uniform
--recompute_num_layers 1
--optimizer_cpu_offload true
--num_train_epochs 3
--finetune true
--cross_entropy_loss_fusion true
--lr 1e-5
--lr_warmup_fraction 0.05
--min_lr 1e-6
--output_dir /personal/megatron_output/Qwen3.5-35B-A3B
--eval_steps 200
--save_steps 200
--max_length 8192
--dataloader_num_workers 8
--dataset_num_proc 8
--no_save_optim true
--no_save_rng true
--sequence_parallel true
--freeze_llm false
--freeze_vit true
--freeze_aligner true
--attention_backend flash > /root/sft.log 2>&1 &

PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
NNODES=2
NODE_RANK=1
MASTER_ADDR=33.180.173.176
MASTER_PORT=29500
NPROC_PER_NODE=8
megatron sft
--model /personal/Qwen3.5-35B-A3B
--save_safetensors true
--dataset /personal/OpenHermes-2.5
--load_from_cache_file true
--split_dataset_ratio 0.01
--tensor_model_parallel_size 8
--pipeline_model_parallel_size 2
--expert_model_parallel_size 8
--micro_batch_size 1
--global_batch_size 16
--recompute_granularity full
--recompute_method uniform
--recompute_num_layers 1
--optimizer_cpu_offload true
--num_train_epochs 3
--finetune true
--cross_entropy_loss_fusion true
--lr 1e-5
--lr_warmup_fraction 0.05
--min_lr 1e-6
--output_dir /personal/megatron_output/Qwen3.5-35B-A3B
--eval_steps 200
--save_steps 200
--max_length 8192
--dataloader_num_workers 8
--dataset_num_proc 8
--no_save_optim true
--no_save_rng true
--sequence_parallel true
--freeze_llm false
--freeze_vit true
--freeze_aligner true
--attention_backend flash > /root/sft.log 2>&1 &

Additional Information / 补充信息

No response