[Bugfix] Fix DSA crash under breakable piecewise cudagraphs (#49302)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
61e10f0116
commit
de6ec294ef
@@ -6,7 +6,6 @@ Unit tests for the breakable cudagraph primitives.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import patch
|
||||
@@ -14,7 +13,21 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1"
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _enable_breakable_cudagraph(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Enable breakable cudagraphs for this module's tests only.
|
||||
|
||||
eager_break_during_capture reads the env at decoration time, which
|
||||
happens inside the test bodies, so a per-test fixture suffices.
|
||||
monkeypatch restores the env so other test files running in the same
|
||||
pytest process are unaffected (a module-level os.environ assignment
|
||||
used to leak into test_cudagraph_dispatch.py and break it).
|
||||
"""
|
||||
import vllm.envs as envs
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "1")
|
||||
envs.disable_envs_cache()
|
||||
|
||||
|
||||
def test_piecewise_capture_builds_fresh_metadata_for_both_passes():
|
||||
@@ -86,10 +99,19 @@ def cuda_capture_stream():
|
||||
"""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required")
|
||||
from vllm.utils.torch_utils import _current_stream_tls
|
||||
|
||||
prev_stream = getattr(_current_stream_tls, "value", None)
|
||||
stream = torch.cuda.Stream()
|
||||
with torch.cuda.stream(stream):
|
||||
yield stream
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
# Exiting torch.cuda.stream() records the default stream in vllm's
|
||||
# patched set_stream cache. A later CUDAGraphWrapper capture in the same
|
||||
# process would then run on the default stream, which cannot capture
|
||||
# (this broke test_cudagraph_dispatch.py when run in-process after this
|
||||
# file). Restore the pre-fixture value so this module leaves no trace.
|
||||
_current_stream_tls.value = prev_stream
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -254,12 +254,12 @@ class CudaGraphManager:
|
||||
|
||||
if mixed_mode:
|
||||
# for PIECEWISE graphs there is no limit on requests when replaying
|
||||
# i.e. no request padding is needed
|
||||
# so we leave it as None
|
||||
# i.e. no request padding is needed, so we leave it as None.
|
||||
# For breakable PW graphs, break-point kernels read the real batch
|
||||
# from the forward context; in-graph kernels handle the token padding
|
||||
# themselves from the padded slot_mapping (rows with slot == -1).
|
||||
num_reqs = None
|
||||
if mixed_mode == CUDAGraphMode.FULL or (
|
||||
mixed_mode == CUDAGraphMode.PIECEWISE and self.use_breakable_cg
|
||||
):
|
||||
if mixed_mode == CUDAGraphMode.FULL:
|
||||
num_reqs = min(num_tokens, self.max_num_reqs)
|
||||
desc = BatchExecutionDescriptor(
|
||||
cg_mode=mixed_mode,
|
||||
|
||||
@@ -5,7 +5,6 @@ from typing import Any
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
@@ -140,10 +139,7 @@ class DefaultModelState(ModelState):
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
if cudagraph_mode == CUDAGraphMode.FULL or (
|
||||
cudagraph_mode == CUDAGraphMode.PIECEWISE
|
||||
and is_breakable_cudagraph_enabled()
|
||||
):
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
# Use padded sizes - padding is handled by model_runner.prepare_attn.
|
||||
num_reqs = input_batch.num_reqs_after_padding
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
|
||||
Reference in New Issue
Block a user