BinaryTrajectoryComparisonConfig.comparison_temperature is silently discarded: the comparison query runs at the model's own temperature

comparison_temperature: float | None = None
"""Override the model's temperature. If None, take the temperature configured for the model."""

response = self._model.query(messages, temperature=self.config.comparison_temperature)["message"]  # type: ignore

def _query(
    self, messages: list[dict[str, str]], n: int | None = None, temperature: float | None = None
) -> list[dict]:
    if n is None:
        return self._single_query(messages, temperature=temperature)   # temperature honoured
    outputs = []
    # not needed for openai, but oh well.
    for _ in range(n):
        outputs.extend(self._single_query(messages))                   # temperature dropped
    return outputs

git clone https://github.com/SWE-agent/SWE-agent.git
cd SWE-agent
git checkout 3ea751c087f32b16e039a2233dd6eefecef325d5
python -m venv .venv-probe && .venv-probe/bin/pip install -e .
.venv-probe/bin/python repro_comparison_temperature_sampler.py

import sweagent.agent.models as M
from sweagent.agent.action_sampler import BinaryTrajectoryComparison, BinaryTrajectoryComparisonConfig
from sweagent.agent.models import GenericAPIModelConfig, LiteLLMModel
from sweagent.agent.problem_statement import EmptyProblemStatement
from sweagent.tools.parsing import Identity
from sweagent.tools.tools import ToolConfig, ToolHandler

TOOL_CONFIG = ToolConfig(parse_function=Identity())
TOOL_HANDLER = ToolHandler(TOOL_CONFIG)
MODEL_TEMPERATURE = 0.0
COMPARISON_TEMPERATURE = 0.9

calls: list[tuple[str, float | None]] = []


class _FakeMessage:
    def __init__(self, content: str):
        self.content = content
        self.tool_calls = None


class _FakeChoice:
    def __init__(self, content: str):
        self.message = _FakeMessage(content)


class _FakeResponse:
    def __init__(self, content: str):
        self.choices = [_FakeChoice(content)]
        self.usage = {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}


_CONTENTS = ["action-a", "action-b", "judgement"]


def fake_completion(**kwargs):
    # calls 0,1 = the two sampled candidate completions; call 2 = the pairwise comparison query
    content = _CONTENTS[len(calls)] if len(calls) < len(_CONTENTS) else "judgement"
    calls.append((content, kwargs.get("temperature")))
    return _FakeResponse(content)


M.litellm.completion = fake_completion
M.litellm.cost_calculator.completion_cost = lambda *a, **k: 0.0
M.litellm.utils.token_counter = lambda *a, **k: 1

config = GenericAPIModelConfig(
    name="gpt-4o",
    temperature=MODEL_TEMPERATURE,
    per_instance_cost_limit=0,
    total_cost_limit=0,
)
model = LiteLLMModel(config, TOOL_CONFIG)

sampler_config = BinaryTrajectoryComparisonConfig(
    min_n_samples=2,
    max_n_samples=2,
    comparison_temperature=COMPARISON_TEMPERATURE,
    instance_template="{{problem_statement}}",
    comparison_template="{{thought1}}{{action1}}{{thought2}}{{action2}}",
)
sampler = BinaryTrajectoryComparison(sampler_config, model, TOOL_HANDLER)

out = sampler.get_action(
    problem_statement=EmptyProblemStatement(),
    trajectory=[],
    history=[{"role": "user", "content": "solve it"}],
)

print(f"configured model temperature      : {MODEL_TEMPERATURE}")
print(f"config.comparison_temperature     : {COMPARISON_TEMPERATURE}")
print(f"temperatures reaching litellm     : {[t for _, t in calls]}")
print(f"n candidate samples, then comparison calls: {[c for c, _ in calls]}")
print(f"sampler returned completion       : {out.completion['message']!r}")
assert len(calls) == 3, f"expected 2 samples + 1 comparison call, got {calls}"
assert all(t != COMPARISON_TEMPERATURE for _, t in calls[2:]), "comparison temperature was applied"
print("=> the comparison query (call #3) ran at the model temperature, not the configured override")

model.query(hist, temperature=0.9)        # -> temperature reaching litellm: [0.0]
model.query(hist, n=1, temperature=0.9)   # -> temperature reaching litellm: [0.0]
model.query(hist, n=None, temperature=0.9)  # -> temperature reaching litellm: [0.9]  (documented branch)

=== A) direct LiteLLMModel.query calls, temperature=0.9 ===
query(hist, temperature=0.9)             -> temperature reaching litellm: [0.0]
query(hist, n=1, temperature=0.9)        -> temperature reaching litellm: [0.0]
query(hist, n=None, temperature=0.9)     -> temperature reaching litellm: [0.9]
=== B) real sampler: BinaryTrajectoryComparison.get_action ===
temperatures reaching litellm     : [0.0, 0.0, 0.0]
VERDICT: DROPPED

B) query(n=None, temp=0.9)     -> litellm (n, temp) calls: [(None, 0.9)]
A) query(temp=0.9) default n   -> litellm (n, temp) calls: [(None, 0.0)]
C) sampler comparison (temp=0.9) -> litellm (n, temp) calls: [(None, 0.0), (None, 0.0), (None, 0.0)]
D) _single_query(temp=0.9)     -> litellm (n, temp) calls: [(None, 0.9)]
# after 1-line fix (n=None at action_sampler.py:288):
C) sampler comparison (temp=0.9) -> litellm (n, temp) calls: [(None, 0.0), (None, 0.0), (None, 0.9)]