[错误] seq-mask-tis 会跳过 token 层级的截断,可能产生无限大的 PPO 更新

import math import torch from openrlhf.models.loss import PolicyLoss loss_fn = PolicyLoss( policy_loss_type="ppo", enable_vllm_is_correction=True, vllm_is_truncated_threshold=[0.5, 5.0], vllm_is_correction_type="seq-mask-tis", ) def run(rollout_log_probs): old_log_probs = torch.tensor([[0.0, -1000.0]]) log_probs = old_log_probs.clone().requires_grad_(True) loss, *_ = loss_fn( log_probs, old_log_probs, torch.ones_like(log_probs), action_mask=torch.ones_like(log_probs), rollout_log_probs=torch.tensor([rollout_log_probs]) ) loss.backward() print(loss, log_probs.grad, torch.isfinite(loss), torch.isfinite(log_probs.grad).all() ) # Finite token ratios [10, 0.1] have geometric mean 1 and pass the sequence filter. d = math.log(10.0) old_log_probs = torch.tensor([[-0.1, -0.1 - d]]) log_probs = old_log_probs.clone().requires_grad_(True) moderate_loss, *_ = loss_fn( log_probs, old_log_probs, torch.ones_like(log_probs), action_mask=torch.ones_like(log_probs), rollout_log_probs=torch.tensor([[-0.1 - d, -0.1]]), ) print(moderate_loss) # -5.05 on main; TIS-clamped result should be -2.75 # Token ratios are exp([1000, -1000]); geometric mean is 1, so the sequence is accepted. run([-1000.0, 0.0]) # Token ratios are exp([1000, -996]); geometric mean is exp(2) > 5, so it is rejected. run([-1000.0, -4.0])