[CI] Detect and fail evals on when NaNs appear in logits (#50323)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
3756bf1e2b
commit
12292d94b2
@@ -14,6 +14,7 @@ import torch
|
||||
from packaging.version import Version
|
||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import LLM
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
@@ -287,6 +288,36 @@ def test_failed_model_execution(vllm_runner, monkeypatch) -> None:
|
||||
v1_test_failed_model_execution(vllm_model)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
|
||||
def test_raise_on_logit_nans(
|
||||
vllm_runner, monkeypatch, use_v2_model_runner: bool
|
||||
) -> None:
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_RAISE_ON_LOGIT_NANS", "1")
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
|
||||
envs.disable_envs_cache()
|
||||
|
||||
try:
|
||||
with vllm_runner("facebook/opt-125m", enforce_eager=True) as vllm_model:
|
||||
engine_core = vllm_model.llm.llm_engine.engine_core.engine_core
|
||||
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
|
||||
assert model_runner.vllm_config.use_v2_model_runner is use_v2_model_runner
|
||||
model_cls = type(model_runner.model)
|
||||
original_compute_logits = model_cls.compute_logits
|
||||
|
||||
def compute_nan_logits(self, *args, **kwargs):
|
||||
logits = original_compute_logits(self, *args, **kwargs)
|
||||
logits[0, 0] = float("nan")
|
||||
return logits
|
||||
|
||||
monkeypatch.setattr(model_cls, "compute_logits", compute_nan_logits)
|
||||
|
||||
with pytest.raises(RuntimeError, match="NaNs detected in logits"):
|
||||
vllm_model.generate_greedy(["Hello, my name is"], 1, use_tqdm=False)
|
||||
finally:
|
||||
envs.disable_envs_cache()
|
||||
|
||||
|
||||
def v1_test_failed_model_execution(vllm_model):
|
||||
engine = vllm_model.llm.llm_engine
|
||||
mocked_execute_model = Mock(side_effect=RuntimeError("Mocked Critical Error"))
|
||||
|
||||
@@ -524,7 +524,8 @@ def test_update_states_request_resumed(model_runner, dist_init):
|
||||
assert _is_req_state_block_table_match(model_runner, req_id)
|
||||
|
||||
|
||||
def test_get_nans_in_logits(model_runner, dist_init):
|
||||
def test_get_nans_in_logits(model_runner, dist_init, monkeypatch):
|
||||
monkeypatch.setattr(gpu_model_runner_module.envs, "VLLM_RAISE_ON_LOGIT_NANS", False)
|
||||
req_ids = ("req_0", "req_1")
|
||||
|
||||
scheduler_output = _schedule_new_request(*req_ids)
|
||||
|
||||
@@ -228,6 +228,7 @@ if TYPE_CHECKING:
|
||||
VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
|
||||
VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None
|
||||
VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
|
||||
VLLM_RAISE_ON_LOGIT_NANS: bool = False
|
||||
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[
|
||||
"FP", "INT8", "INT6", "INT4", "INT3", "NONE"
|
||||
] = "NONE"
|
||||
@@ -1720,6 +1721,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# or bad hardware but it may add compute overhead.
|
||||
"VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool(
|
||||
int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0"))
|
||||
or int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0"))
|
||||
),
|
||||
# Raise an exception when generated logits contain NaNs. Enabling this
|
||||
# also enables the NaN computation required to detect them.
|
||||
"VLLM_RAISE_ON_LOGIT_NANS": lambda: bool(
|
||||
int(os.getenv("VLLM_RAISE_ON_LOGIT_NANS", "0"))
|
||||
),
|
||||
# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
|
||||
"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
|
||||
|
||||
@@ -5,6 +5,7 @@ import contextlib
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
|
||||
from vllm.v1.outputs import (
|
||||
AsyncModelRunnerOutput,
|
||||
@@ -14,6 +15,7 @@ from vllm.v1.outputs import (
|
||||
RoutedExpertsTensors,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.utils import raise_if_nan_logits
|
||||
|
||||
|
||||
class AsyncOutput(AsyncModelRunnerOutput):
|
||||
@@ -80,6 +82,8 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
self.model_runner_output.num_nans_in_logits = dict(
|
||||
zip(self.model_runner_output.req_ids, self.num_nans.tolist())
|
||||
)
|
||||
if envs.VLLM_RAISE_ON_LOGIT_NANS:
|
||||
raise_if_nan_logits(self.model_runner_output.num_nans_in_logits)
|
||||
|
||||
if self.logprobs_tensors is not None:
|
||||
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
|
||||
|
||||
@@ -231,7 +231,7 @@ from vllm.v1.worker.ubatch_utils import (
|
||||
maybe_create_ubatch_slices,
|
||||
split_attn_metadata,
|
||||
)
|
||||
from vllm.v1.worker.utils import is_residual_scattered_for_sp
|
||||
from vllm.v1.worker.utils import is_residual_scattered_for_sp, raise_if_nan_logits
|
||||
from vllm.v1.worker.workspace import lock_workspace
|
||||
|
||||
from .utils import (
|
||||
@@ -5723,6 +5723,8 @@ class GPUModelRunner(
|
||||
if num_nans_for_index is not None and req_index < logits.shape[0]
|
||||
else 0
|
||||
)
|
||||
if envs.VLLM_RAISE_ON_LOGIT_NANS:
|
||||
raise_if_nan_logits(num_nans_in_logits)
|
||||
return num_nans_in_logits
|
||||
except IndexError:
|
||||
return {}
|
||||
|
||||
+13
-1
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import product as iprod
|
||||
from typing import Any
|
||||
@@ -41,6 +41,18 @@ from vllm.v1.worker.block_table import get_block_table_width
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def raise_if_nan_logits(num_nans_in_logits: Mapping[str, int]) -> None:
|
||||
if not any(num_nans_in_logits.values()):
|
||||
return
|
||||
|
||||
corrupted_requests = {
|
||||
req_id: num_nans
|
||||
for req_id, num_nans in num_nans_in_logits.items()
|
||||
if num_nans > 0
|
||||
}
|
||||
raise RuntimeError(f"NaNs detected in logits: {corrupted_requests}")
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["n_blocks"])
|
||||
def _zero_kv_blocks_kernel(
|
||||
seg_addrs_ptr,
|
||||
|
||||
Reference in New Issue
Block a user