[CI] Detect and fail evals on when NaNs appear in logits (#50323)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Tyler Michael Smith
2026-08-04 23:02:58 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 3756bf1e2b
commit 12292d94b2
6 changed files with 60 additions and 3 deletions
@@ -14,6 +14,7 @@ import torch
from packaging.version import Version
from transformers import __version__ as TRANSFORMERS_VERSION
import vllm.envs as envs
from vllm import LLM
from vllm.platforms import current_platform
from vllm.v1.engine.llm_engine import LLMEngine
@@ -287,6 +288,36 @@ def test_failed_model_execution(vllm_runner, monkeypatch) -> None:
v1_test_failed_model_execution(vllm_model)
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
def test_raise_on_logit_nans(
vllm_runner, monkeypatch, use_v2_model_runner: bool
) -> None:
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_RAISE_ON_LOGIT_NANS", "1")
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
envs.disable_envs_cache()
try:
with vllm_runner("facebook/opt-125m", enforce_eager=True) as vllm_model:
engine_core = vllm_model.llm.llm_engine.engine_core.engine_core
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
assert model_runner.vllm_config.use_v2_model_runner is use_v2_model_runner
model_cls = type(model_runner.model)
original_compute_logits = model_cls.compute_logits
def compute_nan_logits(self, *args, **kwargs):
logits = original_compute_logits(self, *args, **kwargs)
logits[0, 0] = float("nan")
return logits
monkeypatch.setattr(model_cls, "compute_logits", compute_nan_logits)
with pytest.raises(RuntimeError, match="NaNs detected in logits"):
vllm_model.generate_greedy(["Hello, my name is"], 1, use_tqdm=False)
finally:
envs.disable_envs_cache()
def v1_test_failed_model_execution(vllm_model):
engine = vllm_model.llm.llm_engine
mocked_execute_model = Mock(side_effect=RuntimeError("Mocked Critical Error"))
+2 -1
View File
@@ -524,7 +524,8 @@ def test_update_states_request_resumed(model_runner, dist_init):
assert _is_req_state_block_table_match(model_runner, req_id)
def test_get_nans_in_logits(model_runner, dist_init):
def test_get_nans_in_logits(model_runner, dist_init, monkeypatch):
monkeypatch.setattr(gpu_model_runner_module.envs, "VLLM_RAISE_ON_LOGIT_NANS", False)
req_ids = ("req_0", "req_1")
scheduler_output = _schedule_new_request(*req_ids)
+7
View File
@@ -228,6 +228,7 @@ if TYPE_CHECKING:
VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None
VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
VLLM_RAISE_ON_LOGIT_NANS: bool = False
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[
"FP", "INT8", "INT6", "INT4", "INT3", "NONE"
] = "NONE"
@@ -1720,6 +1721,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
# or bad hardware but it may add compute overhead.
"VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool(
int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0"))
or int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0"))
),
# Raise an exception when generated logits contain NaNs. Enabling this
# also enables the NaN computation required to detect them.
"VLLM_RAISE_ON_LOGIT_NANS": lambda: bool(
int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0"))
),
# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
+4
View File
@@ -5,6 +5,7 @@ import contextlib
import numpy as np
import torch
import vllm.envs as envs
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.v1.outputs import (
AsyncModelRunnerOutput,
@@ -14,6 +15,7 @@ from vllm.v1.outputs import (
RoutedExpertsTensors,
)
from vllm.v1.worker.gpu.sample.output import SamplerOutput
from vllm.v1.worker.utils import raise_if_nan_logits
class AsyncOutput(AsyncModelRunnerOutput):
@@ -80,6 +82,8 @@ class AsyncOutput(AsyncModelRunnerOutput):
self.model_runner_output.num_nans_in_logits = dict(
zip(self.model_runner_output.req_ids, self.num_nans.tolist())
)
if envs.VLLM_RAISE_ON_LOGIT_NANS:
raise_if_nan_logits(self.model_runner_output.num_nans_in_logits)
if self.logprobs_tensors is not None:
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
+3 -1
View File
@@ -231,7 +231,7 @@ from vllm.v1.worker.ubatch_utils import (
maybe_create_ubatch_slices,
split_attn_metadata,
)
from vllm.v1.worker.utils import is_residual_scattered_for_sp
from vllm.v1.worker.utils import is_residual_scattered_for_sp, raise_if_nan_logits
from vllm.v1.worker.workspace import lock_workspace
from .utils import (
@@ -5723,6 +5723,8 @@ class GPUModelRunner(
if num_nans_for_index is not None and req_index < logits.shape[0]
else 0
)
if envs.VLLM_RAISE_ON_LOGIT_NANS:
raise_if_nan_logits(num_nans_in_logits)
return num_nans_in_logits
except IndexError:
return {}
+13 -1
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections import defaultdict
from collections.abc import Iterable, Sequence
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from itertools import product as iprod
from typing import Any
@@ -41,6 +41,18 @@ from vllm.v1.worker.block_table import get_block_table_width
logger = init_logger(__name__)
def raise_if_nan_logits(num_nans_in_logits: Mapping[str, int]) -> None:
if not any(num_nans_in_logits.values()):
return
corrupted_requests = {
req_id: num_nans
for req_id, num_nans in num_nans_in_logits.items()
if num_nans > 0
}
raise RuntimeError(f"NaNs detected in logits: {corrupted_requests}")
@triton.jit(do_not_specialize=["n_blocks"])
def _zero_kv_blocks_kernel(
seg_addrs_ptr,