[CI] Detect and fail evals on when NaNs appear in logits (#50323)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Tyler Michael Smith
2026-08-04 23:02:58 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 3756bf1e2b
commit 12292d94b2
6 changed files with 60 additions and 3 deletions
@@ -14,6 +14,7 @@ import torch
from packaging.version import Version
from transformers import __version__ as TRANSFORMERS_VERSION
import vllm.envs as envs
from vllm import LLM
from vllm.platforms import current_platform
from vllm.v1.engine.llm_engine import LLMEngine
@@ -287,6 +288,36 @@ def test_failed_model_execution(vllm_runner, monkeypatch) -> None:
v1_test_failed_model_execution(vllm_model)
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
def test_raise_on_logit_nans(
vllm_runner, monkeypatch, use_v2_model_runner: bool
) -> None:
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_RAISE_ON_LOGIT_NANS", "1")
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
envs.disable_envs_cache()
try:
with vllm_runner("facebook/opt-125m", enforce_eager=True) as vllm_model:
engine_core = vllm_model.llm.llm_engine.engine_core.engine_core
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
assert model_runner.vllm_config.use_v2_model_runner is use_v2_model_runner
model_cls = type(model_runner.model)
original_compute_logits = model_cls.compute_logits
def compute_nan_logits(self, *args, **kwargs):
logits = original_compute_logits(self, *args, **kwargs)
logits[0, 0] = float("nan")
return logits
monkeypatch.setattr(model_cls, "compute_logits", compute_nan_logits)
with pytest.raises(RuntimeError, match="NaNs detected in logits"):
vllm_model.generate_greedy(["Hello, my name is"], 1, use_tqdm=False)
finally:
envs.disable_envs_cache()
def v1_test_failed_model_execution(vllm_model):
engine = vllm_model.llm.llm_engine
mocked_execute_model = Mock(side_effect=RuntimeError("Mocked Critical Error"))