如果我们使用 GRPO,且 args.kl_coef 为非零,是否 KL 计算有误?

if args.kl_coef == 0: # when kl_coef is 0, we won't compute ref_log_prob xs = log_probs if log_probs is not None else values kl = [torch.zeros_like(x, dtype=torch.float32, device=x.device) for x in xs] else: kl = [ compute_approx_kl( log_probs[i], ref_log_probs[i], kl_loss_type=args.kl_loss_type, ) for i in range(len(log_probs)) ]

if args.advantage_estimator in ["grpo", "gspo"]: rewards = torch.tensor(rewards, dtype=torch.float32, device=kl[0].device) returns = get_grpo_returns(rewards, kl) # TODO: is the copy necessary? advantages = [r for r in returns]