[Bugfix] Fall back to full decode graphs for noncompiled models (#55095)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
parent
9dcab80215
commit
d0dfe587d5
@@ -273,6 +273,44 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("architecture", "use_v2", "mode", "expected"),
|
||||
[
|
||||
("DeepseekV4ForCausalLM", True, None, CUDAGraphMode.NONE),
|
||||
("DeepseekV4ForConditionalGeneration", True, None, CUDAGraphMode.NONE),
|
||||
("DeepseekV4ForCausalLM", False, None, None),
|
||||
("LlamaForCausalLM", True, None, None),
|
||||
(
|
||||
"DeepseekV4ForCausalLM",
|
||||
True,
|
||||
CUDAGraphMode.FULL_DECODE_ONLY,
|
||||
CUDAGraphMode.FULL_DECODE_ONLY,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_rocm_gfx950_deepseek_v4_cudagraph_default(
|
||||
monkeypatch, architecture, use_v2, mode, expected
|
||||
):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms import rocm
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_fused_moe_enabled", lambda: False)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_linear_fp8_enabled", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_fusion_moe_shared_experts_enabled", lambda: False
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
compilation_config=CompilationConfig(cudagraph_mode=mode),
|
||||
model_config=SimpleNamespace(architecture=architecture),
|
||||
use_v2_model_runner=use_v2,
|
||||
)
|
||||
|
||||
rocm.RocmPlatform.apply_config_platform_defaults(config)
|
||||
|
||||
assert config.compilation_config.cudagraph_mode == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "architecture"),
|
||||
[
|
||||
@@ -463,6 +501,66 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
|
||||
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "piecewise_capture_available", "attention_support", "expected"),
|
||||
[
|
||||
("PIECEWISE", False, "ALWAYS", "NONE"),
|
||||
("FULL_AND_PIECEWISE", False, "ALWAYS", "FULL_DECODE_ONLY"),
|
||||
("FULL_DECODE_ONLY", False, "ALWAYS", "FULL_DECODE_ONLY"),
|
||||
("FULL_DECODE_ONLY", False, "NEVER", "NONE"),
|
||||
],
|
||||
)
|
||||
def test_resolve_cudagraph_mode_uses_loaded_piecewise_provider(
|
||||
mode, piecewise_capture_available, attention_support, expected
|
||||
):
|
||||
compilation_config = CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
cudagraph_mode=CUDAGraphMode[mode],
|
||||
use_inductor_graph_partition=True,
|
||||
)
|
||||
|
||||
resolved = compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||
AttentionCGSupport[attention_support],
|
||||
"FakeAttentionBackend",
|
||||
piecewise_capture_available=piecewise_capture_available,
|
||||
)
|
||||
|
||||
assert resolved.name == expected
|
||||
assert compilation_config.cudagraph_mode == resolved
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Requires CUDA graph support"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"engine_kwargs",
|
||||
[
|
||||
{"runner": "pooling", "convert": "embed"},
|
||||
pytest.param(
|
||||
{"prefill_context_parallel_size": 2, "tensor_parallel_size": 2},
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm PCP graph restriction"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_late_piecewise_restrictions_without_compilation(monkeypatch, engine_kwargs):
|
||||
"""Late compatibility overrides must not restore unavailable piecewise graphs."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "0")
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
config = EngineArgs(
|
||||
model="facebook/opt-125m",
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
**engine_kwargs,
|
||||
).create_engine_config()
|
||||
|
||||
assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
|
||||
assert config.compilation_config.cudagraph_capture_sizes == []
|
||||
assert config.compilation_config.max_cudagraph_capture_size == 0
|
||||
|
||||
|
||||
def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
"""Cudagraph memory profiling uses a minimal KV cache, so the Mamba
|
||||
block-count guard must only fire for the real cache sizing."""
|
||||
@@ -498,6 +596,30 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("graph_mode", "should_raise"),
|
||||
[
|
||||
(CUDAGraphMode.FULL_DECODE_ONLY, False),
|
||||
(CUDAGraphMode.FULL_AND_PIECEWISE, False),
|
||||
(CUDAGraphMode.NONE, True),
|
||||
(CUDAGraphMode.PIECEWISE, True),
|
||||
],
|
||||
)
|
||||
def test_adaptive_verification_requires_full_cudagraphs(graph_mode, should_raise):
|
||||
config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(enable_adaptive_verification=True),
|
||||
lora_config=None,
|
||||
compilation_config=CompilationConfig(cudagraph_mode=graph_mode),
|
||||
parallel_config=SimpleNamespace(pipeline_parallel_size=1),
|
||||
)
|
||||
|
||||
if should_raise:
|
||||
with pytest.raises(ValueError, match="requires full CUDA graphs"):
|
||||
VllmConfig._validate_adaptive_verification(config)
|
||||
else:
|
||||
VllmConfig._validate_adaptive_verification(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_config", "expected"),
|
||||
[
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.worker.gpu.async_utils import StepTimingSample
|
||||
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
|
||||
@@ -12,6 +14,7 @@ from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_mod
|
||||
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
|
||||
AdaptiveVerificationManager,
|
||||
maybe_create_adaptive_verification_manager,
|
||||
resolve_adaptive_cudagraph_mode,
|
||||
)
|
||||
from vllm.v1.worker.gpu.structured_outputs import _build_grammar_mapping
|
||||
|
||||
@@ -35,6 +38,25 @@ def make_manager(
|
||||
return manager
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "piecewise_capture_available", "expected"),
|
||||
[
|
||||
("FULL_DECODE_ONLY", True, "FULL_DECODE_ONLY"),
|
||||
("FULL", True, "FULL_AND_PIECEWISE"),
|
||||
("FULL", False, "FULL_DECODE_ONLY"),
|
||||
("FULL_AND_PIECEWISE", True, "FULL_AND_PIECEWISE"),
|
||||
("FULL_AND_PIECEWISE", False, "FULL_DECODE_ONLY"),
|
||||
],
|
||||
)
|
||||
def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expected):
|
||||
assert (
|
||||
resolve_adaptive_cudagraph_mode(
|
||||
CUDAGraphMode[mode], piecewise_capture_available=piecewise_capture_available
|
||||
)
|
||||
== CUDAGraphMode[expected]
|
||||
)
|
||||
|
||||
|
||||
def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch):
|
||||
class Backend:
|
||||
@classmethod
|
||||
|
||||
@@ -77,6 +77,13 @@ class CUDAGraphMode(enum.Enum):
|
||||
def requires_piecewise_compilation(self) -> bool:
|
||||
return self.has_mode(CUDAGraphMode.PIECEWISE)
|
||||
|
||||
def without_piecewise(self) -> "CUDAGraphMode":
|
||||
if self == CUDAGraphMode.PIECEWISE:
|
||||
return CUDAGraphMode.NONE
|
||||
if self == CUDAGraphMode.FULL_AND_PIECEWISE:
|
||||
return CUDAGraphMode.FULL_DECODE_ONLY
|
||||
return self
|
||||
|
||||
def max_cudagraph_mode(self) -> "CUDAGraphMode":
|
||||
return CUDAGraphMode(max(self.value)) if self.separate_routine() else self
|
||||
|
||||
@@ -1382,6 +1389,7 @@ class CompilationConfig:
|
||||
kv_cache_config: "KVCacheConfig | None" = None,
|
||||
max_num_reqs: int | None = None,
|
||||
is_profiling: bool = False,
|
||||
piecewise_capture_available: bool = True,
|
||||
) -> CUDAGraphMode:
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
|
||||
@@ -1464,6 +1472,20 @@ class CompilationConfig:
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
logger.warning(msg)
|
||||
|
||||
if (
|
||||
not piecewise_capture_available
|
||||
and cudagraph_mode.requires_piecewise_compilation()
|
||||
):
|
||||
fallback_mode = cudagraph_mode.without_piecewise()
|
||||
logger.warning_once(
|
||||
"Cudagraph mode %s requires piecewise capture, but the loaded "
|
||||
"model provides neither a compiled submodule nor breakable CUDA "
|
||||
"graphs. Overriding to %s.",
|
||||
cudagraph_mode,
|
||||
fallback_mode,
|
||||
)
|
||||
cudagraph_mode = fallback_mode
|
||||
|
||||
# double check that we can support full cudagraph if they are requested
|
||||
# even after automatic downgrades
|
||||
if (
|
||||
|
||||
+33
-17
@@ -991,6 +991,28 @@ class VllmConfig:
|
||||
)
|
||||
speculative_config.num_speculative_tokens_per_batch_size = None
|
||||
|
||||
def _normalize_piecewise_cudagraph_mode(
|
||||
self, *, breakable_cudagraph_enabled: bool
|
||||
) -> None:
|
||||
compilation_config = self.compilation_config
|
||||
if (
|
||||
compilation_config.cudagraph_mode.requires_piecewise_compilation()
|
||||
and compilation_config.mode != CompilationMode.VLLM_COMPILE
|
||||
and not breakable_cudagraph_enabled
|
||||
):
|
||||
fallback_mode = compilation_config.cudagraph_mode.without_piecewise()
|
||||
logger.info_once(
|
||||
"Cudagraph mode %s is not compatible with compilation mode %s. "
|
||||
"Overriding to %s.",
|
||||
compilation_config.cudagraph_mode,
|
||||
compilation_config.mode,
|
||||
fallback_mode,
|
||||
)
|
||||
compilation_config.cudagraph_mode = fallback_mode
|
||||
if fallback_mode == CUDAGraphMode.NONE:
|
||||
compilation_config.max_cudagraph_capture_size = 0
|
||||
compilation_config.cudagraph_capture_sizes = []
|
||||
|
||||
def _post_init_kv_transfer_config(self) -> None:
|
||||
"""Update KVTransferConfig based on top-level configs in VllmConfig.
|
||||
|
||||
@@ -1536,18 +1558,9 @@ class VllmConfig:
|
||||
self._maybe_disable_dynamic_sd_for_data_parallel()
|
||||
self._maybe_override_dynamic_sd_cudagraph_mode()
|
||||
|
||||
if (
|
||||
self.compilation_config.cudagraph_mode.requires_piecewise_compilation()
|
||||
and self.compilation_config.mode != CompilationMode.VLLM_COMPILE
|
||||
and not envs.VLLM_USE_BREAKABLE_CUDAGRAPH
|
||||
):
|
||||
logger.info_once(
|
||||
"Cudagraph mode %s is not compatible with compilation mode %s."
|
||||
"Overriding to NONE.",
|
||||
self.compilation_config.cudagraph_mode,
|
||||
self.compilation_config.mode,
|
||||
)
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
self._normalize_piecewise_cudagraph_mode(
|
||||
breakable_cudagraph_enabled=breakable_cudagraph_enabled
|
||||
)
|
||||
|
||||
# async tp is built on top of sequence parallelism and requires it.
|
||||
pass_config = self.compilation_config.pass_config
|
||||
@@ -1716,6 +1729,10 @@ class VllmConfig:
|
||||
)
|
||||
current_platform.check_and_update_config(self)
|
||||
|
||||
self._normalize_piecewise_cudagraph_mode(
|
||||
breakable_cudagraph_enabled=breakable_cudagraph_enabled
|
||||
)
|
||||
|
||||
self._resolve_mm_embedding_inputs()
|
||||
self._resolve_mm_processor_device()
|
||||
self._validate_mm_processor_device()
|
||||
@@ -2728,12 +2745,11 @@ class VllmConfig:
|
||||
"Adaptive verification is not currently compatible with LoRA"
|
||||
)
|
||||
|
||||
if self.compilation_config.cudagraph_mode == CUDAGraphMode.NONE:
|
||||
# The draft budget divides by step costs profiled from captured
|
||||
# cudagraphs; eager execution captures none.
|
||||
if not self.compilation_config.cudagraph_mode.has_full_cudagraphs():
|
||||
raise ValueError(
|
||||
"Adaptive verification is not currently compatible with "
|
||||
"enforce_eager/cudagraph_mode=none"
|
||||
"Adaptive verification requires full CUDA graphs. Use cudagraph "
|
||||
"mode FULL, FULL_DECODE_ONLY, or FULL_AND_PIECEWISE, or disable "
|
||||
"adaptive verification."
|
||||
)
|
||||
|
||||
if self.parallel_config.pipeline_parallel_size > 1:
|
||||
|
||||
@@ -880,8 +880,25 @@ class RocmPlatform(Platform):
|
||||
@classmethod
|
||||
def apply_config_platform_defaults(cls, vllm_config: "VllmConfig") -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
model_config = vllm_config.model_config
|
||||
if (
|
||||
compilation_config.cudagraph_mode is None
|
||||
and model_config is not None
|
||||
and on_gfx950()
|
||||
and vllm_config.use_v2_model_runner
|
||||
and model_config.architecture
|
||||
in {
|
||||
"DeepseekV4ForCausalLM",
|
||||
"DeepseekV4ForConditionalGeneration",
|
||||
}
|
||||
):
|
||||
# Default to eager after reported gfx950/MRV2 accuracy regressions:
|
||||
# https://github.com/vllm-project/vllm/issues/52644
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
use_aiter_fused_moe = rocm_aiter_ops.is_fused_moe_enabled()
|
||||
use_aiter_fp8_linear = rocm_aiter_ops.is_linear_fp8_enabled()
|
||||
use_aiter_fused_se = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
|
||||
@@ -104,6 +104,7 @@ from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
ModelCudaGraphManager,
|
||||
has_compiled_submodule,
|
||||
make_cudagraph_stats,
|
||||
)
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
@@ -152,6 +153,7 @@ from vllm.v1.worker.gpu.spec_decode import init_speculator
|
||||
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
|
||||
AdaptiveVerificationManager,
|
||||
maybe_create_adaptive_verification_manager,
|
||||
resolve_adaptive_cudagraph_mode,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
|
||||
set_eagle3_aux_hidden_state_layers,
|
||||
@@ -674,8 +676,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.kv_cache_config,
|
||||
use_replayssm=self.cache_config.use_replayssm,
|
||||
)
|
||||
piecewise_capture_available = bool(
|
||||
envs.VLLM_USE_BREAKABLE_CUDAGRAPH or has_compiled_submodule(self.model)
|
||||
)
|
||||
if self.adaptive_verification is not None:
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
self.compilation_config.cudagraph_mode = resolve_adaptive_cudagraph_mode(
|
||||
self.compilation_config.cudagraph_mode,
|
||||
piecewise_capture_available=piecewise_capture_available,
|
||||
)
|
||||
cudagraph_mode = self.compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||
attn_cg_support.min_cg_support,
|
||||
attn_cg_support.min_cg_attn_backend,
|
||||
@@ -685,6 +693,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
is_profiling=is_profiling,
|
||||
piecewise_capture_available=piecewise_capture_available,
|
||||
)
|
||||
self.cudagraph_manager = ModelCudaGraphManager(
|
||||
self.vllm_config,
|
||||
|
||||
@@ -10,6 +10,7 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -33,6 +34,16 @@ if TYPE_CHECKING:
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
def resolve_adaptive_cudagraph_mode(
|
||||
mode: CUDAGraphMode, *, piecewise_capture_available: bool
|
||||
) -> CUDAGraphMode:
|
||||
"""Select a separate decode route for variable-length verification."""
|
||||
assert mode.has_full_cudagraphs()
|
||||
if mode == CUDAGraphMode.FULL_DECODE_ONLY or not piecewise_capture_available:
|
||||
return CUDAGraphMode.FULL_DECODE_ONLY
|
||||
return CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
|
||||
|
||||
def _assign_draft_token_budget(
|
||||
confidence_probs: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user