From 12292d94b25869be2af6b6d4f8eea6c2445e935f Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Tue, 4 Aug 2026 19:02:58 -0400 Subject: [PATCH] [CI] Detect and fail evals on when NaNs appear in logits (#50323) Signed-off-by: Tyler Michael Smith Co-authored-by: OpenAI Codex --- .../test_basic_correctness.py | 31 +++++++++++++++++++ tests/v1/worker/test_gpu_model_runner.py | 3 +- vllm/envs.py | 7 +++++ vllm/v1/worker/gpu/async_utils.py | 4 +++ vllm/v1/worker/gpu_model_runner.py | 4 ++- vllm/v1/worker/utils.py | 14 ++++++++- 6 files changed, 60 insertions(+), 3 deletions(-) diff --git a/tests/basic_correctness/test_basic_correctness.py b/tests/basic_correctness/test_basic_correctness.py index 4f99120c0c..583c87eb5b 100644 --- a/tests/basic_correctness/test_basic_correctness.py +++ b/tests/basic_correctness/test_basic_correctness.py @@ -14,6 +14,7 @@ import torch from packaging.version import Version from transformers import __version__ as TRANSFORMERS_VERSION +import vllm.envs as envs from vllm import LLM from vllm.platforms import current_platform from vllm.v1.engine.llm_engine import LLMEngine @@ -287,6 +288,36 @@ def test_failed_model_execution(vllm_runner, monkeypatch) -> None: v1_test_failed_model_execution(vllm_model) +@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"]) +def test_raise_on_logit_nans( + vllm_runner, monkeypatch, use_v2_model_runner: bool +) -> None: + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + monkeypatch.setenv("VLLM_RAISE_ON_LOGIT_NANS", "1") + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner))) + envs.disable_envs_cache() + + try: + with vllm_runner("facebook/opt-125m", enforce_eager=True) as vllm_model: + engine_core = vllm_model.llm.llm_engine.engine_core.engine_core + model_runner = engine_core.model_executor.driver_worker.worker.model_runner + assert model_runner.vllm_config.use_v2_model_runner is use_v2_model_runner + model_cls = type(model_runner.model) + original_compute_logits = model_cls.compute_logits + + def compute_nan_logits(self, *args, **kwargs): + logits = original_compute_logits(self, *args, **kwargs) + logits[0, 0] = float("nan") + return logits + + monkeypatch.setattr(model_cls, "compute_logits", compute_nan_logits) + + with pytest.raises(RuntimeError, match="NaNs detected in logits"): + vllm_model.generate_greedy(["Hello, my name is"], 1, use_tqdm=False) + finally: + envs.disable_envs_cache() + + def v1_test_failed_model_execution(vllm_model): engine = vllm_model.llm.llm_engine mocked_execute_model = Mock(side_effect=RuntimeError("Mocked Critical Error")) diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 282c022958..814d060767 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -524,7 +524,8 @@ def test_update_states_request_resumed(model_runner, dist_init): assert _is_req_state_block_table_match(model_runner, req_id) -def test_get_nans_in_logits(model_runner, dist_init): +def test_get_nans_in_logits(model_runner, dist_init, monkeypatch): + monkeypatch.setattr(gpu_model_runner_module.envs, "VLLM_RAISE_ON_LOGIT_NANS", False) req_ids = ("req_0", "req_1") scheduler_output = _schedule_new_request(*req_ids) diff --git a/vllm/envs.py b/vllm/envs.py index 0b16f78fcd..98014dc498 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -228,6 +228,7 @@ if TYPE_CHECKING: VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None VLLM_COMPUTE_NANS_IN_LOGITS: bool = False + VLLM_RAISE_ON_LOGIT_NANS: bool = False VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[ "FP", "INT8", "INT6", "INT4", "INT3", "NONE" ] = "NONE" @@ -1720,6 +1721,12 @@ environment_variables: dict[str, Callable[[], Any]] = { # or bad hardware but it may add compute overhead. "VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool( int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0")) + or int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0")) + ), + # Raise an exception when generated logits contain NaNs. Enabling this + # also enables the NaN computation required to detect them. + "VLLM_RAISE_ON_LOGIT_NANS": lambda: bool( + int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0")) ), # Timeout (in seconds) for MooncakeConnector in PD disaggregated setup. "VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int( diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index ab91d9dded..fa009230f8 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -5,6 +5,7 @@ import contextlib import numpy as np import torch +import vllm.envs as envs from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager from vllm.v1.outputs import ( AsyncModelRunnerOutput, @@ -14,6 +15,7 @@ from vllm.v1.outputs import ( RoutedExpertsTensors, ) from vllm.v1.worker.gpu.sample.output import SamplerOutput +from vllm.v1.worker.utils import raise_if_nan_logits class AsyncOutput(AsyncModelRunnerOutput): @@ -80,6 +82,8 @@ class AsyncOutput(AsyncModelRunnerOutput): self.model_runner_output.num_nans_in_logits = dict( zip(self.model_runner_output.req_ids, self.num_nans.tolist()) ) + if envs.VLLM_RAISE_ON_LOGIT_NANS: + raise_if_nan_logits(self.model_runner_output.num_nans_in_logits) if self.logprobs_tensors is not None: self.model_runner_output.logprobs = self.logprobs_tensors.tolists() diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 4291d72b56..d500b6720a 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -231,7 +231,7 @@ from vllm.v1.worker.ubatch_utils import ( maybe_create_ubatch_slices, split_attn_metadata, ) -from vllm.v1.worker.utils import is_residual_scattered_for_sp +from vllm.v1.worker.utils import is_residual_scattered_for_sp, raise_if_nan_logits from vllm.v1.worker.workspace import lock_workspace from .utils import ( @@ -5723,6 +5723,8 @@ class GPUModelRunner( if num_nans_for_index is not None and req_index < logits.shape[0] else 0 ) + if envs.VLLM_RAISE_ON_LOGIT_NANS: + raise_if_nan_logits(num_nans_in_logits) return num_nans_in_logits except IndexError: return {} diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index a8118ef9fa..ac9cce83f6 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import math from collections import defaultdict -from collections.abc import Iterable, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field from itertools import product as iprod from typing import Any @@ -41,6 +41,18 @@ from vllm.v1.worker.block_table import get_block_table_width logger = init_logger(__name__) +def raise_if_nan_logits(num_nans_in_logits: Mapping[str, int]) -> None: + if not any(num_nans_in_logits.values()): + return + + corrupted_requests = { + req_id: num_nans + for req_id, num_nans in num_nans_in_logits.items() + if num_nans > 0 + } + raise RuntimeError(f"NaNs detected in logits: {corrupted_requests}") + + @triton.jit(do_not_specialize=["n_blocks"]) def _zero_kv_blocks_kernel( seg_addrs_ptr,