[Bugfix] Fall back to full decode graphs for noncompiled models (#55095)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Andreas Karatzas
2026-09-11 01:56:05 -07:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson
parent 9dcab80215
commit d0dfe587d5
7 changed files with 237 additions and 18 deletions
+122
View File
@@ -273,6 +273,44 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
default_breakable_cudagraph_architectures.cache_clear()
@pytest.mark.parametrize(
("architecture", "use_v2", "mode", "expected"),
[
("DeepseekV4ForCausalLM", True, None, CUDAGraphMode.NONE),
("DeepseekV4ForConditionalGeneration", True, None, CUDAGraphMode.NONE),
("DeepseekV4ForCausalLM", False, None, None),
("LlamaForCausalLM", True, None, None),
(
"DeepseekV4ForCausalLM",
True,
CUDAGraphMode.FULL_DECODE_ONLY,
CUDAGraphMode.FULL_DECODE_ONLY,
),
],
)
def test_rocm_gfx950_deepseek_v4_cudagraph_default(
monkeypatch, architecture, use_v2, mode, expected
):
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms import rocm
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
monkeypatch.setattr(rocm_aiter_ops, "is_fused_moe_enabled", lambda: False)
monkeypatch.setattr(rocm_aiter_ops, "is_linear_fp8_enabled", lambda: False)
monkeypatch.setattr(
rocm_aiter_ops, "is_fusion_moe_shared_experts_enabled", lambda: False
)
config = SimpleNamespace(
compilation_config=CompilationConfig(cudagraph_mode=mode),
model_config=SimpleNamespace(architecture=architecture),
use_v2_model_runner=use_v2,
)
rocm.RocmPlatform.apply_config_platform_defaults(config)
assert config.compilation_config.cudagraph_mode == expected
@pytest.mark.parametrize(
("model", "architecture"),
[
@@ -463,6 +501,66 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
@pytest.mark.parametrize(
("mode", "piecewise_capture_available", "attention_support", "expected"),
[
("PIECEWISE", False, "ALWAYS", "NONE"),
("FULL_AND_PIECEWISE", False, "ALWAYS", "FULL_DECODE_ONLY"),
("FULL_DECODE_ONLY", False, "ALWAYS", "FULL_DECODE_ONLY"),
("FULL_DECODE_ONLY", False, "NEVER", "NONE"),
],
)
def test_resolve_cudagraph_mode_uses_loaded_piecewise_provider(
mode, piecewise_capture_available, attention_support, expected
):
compilation_config = CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
cudagraph_mode=CUDAGraphMode[mode],
use_inductor_graph_partition=True,
)
resolved = compilation_config.resolve_cudagraph_mode_and_sizes(
AttentionCGSupport[attention_support],
"FakeAttentionBackend",
piecewise_capture_available=piecewise_capture_available,
)
assert resolved.name == expected
assert compilation_config.cudagraph_mode == resolved
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Requires CUDA graph support"
)
@pytest.mark.parametrize(
"engine_kwargs",
[
{"runner": "pooling", "convert": "embed"},
pytest.param(
{"prefill_context_parallel_size": 2, "tensor_parallel_size": 2},
marks=pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm PCP graph restriction"
),
),
],
)
def test_late_piecewise_restrictions_without_compilation(monkeypatch, engine_kwargs):
"""Late compatibility overrides must not restore unavailable piecewise graphs."""
from vllm.engine.arg_utils import EngineArgs
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "0")
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
config = EngineArgs(
model="facebook/opt-125m",
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
**engine_kwargs,
).create_engine_config()
assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
assert config.compilation_config.cudagraph_capture_sizes == []
assert config.compilation_config.max_cudagraph_capture_size == 0
def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
"""Cudagraph memory profiling uses a minimal KV cache, so the Mamba
block-count guard must only fire for the real cache sizing."""
@@ -498,6 +596,30 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
@pytest.mark.parametrize(
("graph_mode", "should_raise"),
[
(CUDAGraphMode.FULL_DECODE_ONLY, False),
(CUDAGraphMode.FULL_AND_PIECEWISE, False),
(CUDAGraphMode.NONE, True),
(CUDAGraphMode.PIECEWISE, True),
],
)
def test_adaptive_verification_requires_full_cudagraphs(graph_mode, should_raise):
config = SimpleNamespace(
speculative_config=SimpleNamespace(enable_adaptive_verification=True),
lora_config=None,
compilation_config=CompilationConfig(cudagraph_mode=graph_mode),
parallel_config=SimpleNamespace(pipeline_parallel_size=1),
)
if should_raise:
with pytest.raises(ValueError, match="requires full CUDA graphs"):
VllmConfig._validate_adaptive_verification(config)
else:
VllmConfig._validate_adaptive_verification(config)
@pytest.mark.parametrize(
("model_config", "expected"),
[
@@ -4,7 +4,9 @@
from types import SimpleNamespace
import numpy as np
import pytest
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.worker.gpu.async_utils import StepTimingSample
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
@@ -12,6 +14,7 @@ from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_mod
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
AdaptiveVerificationManager,
maybe_create_adaptive_verification_manager,
resolve_adaptive_cudagraph_mode,
)
from vllm.v1.worker.gpu.structured_outputs import _build_grammar_mapping
@@ -35,6 +38,25 @@ def make_manager(
return manager
@pytest.mark.parametrize(
("mode", "piecewise_capture_available", "expected"),
[
("FULL_DECODE_ONLY", True, "FULL_DECODE_ONLY"),
("FULL", True, "FULL_AND_PIECEWISE"),
("FULL", False, "FULL_DECODE_ONLY"),
("FULL_AND_PIECEWISE", True, "FULL_AND_PIECEWISE"),
("FULL_AND_PIECEWISE", False, "FULL_DECODE_ONLY"),
],
)
def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expected):
assert (
resolve_adaptive_cudagraph_mode(
CUDAGraphMode[mode], piecewise_capture_available=piecewise_capture_available
)
== CUDAGraphMode[expected]
)
def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch):
class Backend:
@classmethod