[Bugfix] Capture prefill kernels for mixed FULL graphs (#58275)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-24 02:05:07 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent bbc4ddeca9
commit 23110a0c9e
4 changed files with 117 additions and 16 deletions
+25 -15
View File
@@ -36,7 +36,7 @@ def temporary_environ(env_vars):
model_backends_full_cudagraph = []
# deepseek-ai/DeepSeek-V2-Lite with MLA
MLA_backends = ["FlashMLA", "FlashAttentionMLA", "CutlassMLA"]
MLA_backends = ["FlashMLA", "FlashAttentionMLA", "CutlassMLA", "FlashInferMLA"]
for mla_backend in MLA_backends:
model_backends_full_cudagraph.append(
("deepseek-ai/DeepSeek-V2-Lite", backend_configs[mla_backend])
@@ -53,27 +53,28 @@ for backend_config in other_backend_configs:
@pytest.fixture(scope="class")
def llm_pair(request):
model, backend_config, use_inductor_graph_partition = request.param
backend_config.comp_config["use_inductor_graph_partition"] = (
use_inductor_graph_partition
)
if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"):
pytest.skip("Inductor graph partition only supported in torch>=2.9")
backend = AttentionBackendEnum[backend_config.attention_config["backend"]]
if current_platform.is_rocm() and backend not in (
AttentionBackendEnum.TRITON_ATTN,
AttentionBackendEnum.ROCM_ATTN,
):
pytest.skip(f"{backend_config.name} is a CUDA-only attention backend")
if backend == AttentionBackendEnum.ROCM_ATTN and not current_platform.is_rocm():
pytest.skip("ROCM_ATTN requires ROCm")
# Dynamically skip test if GPU capability is not met
if (
backend_config.specific_gpu_arch
and backend_config.specific_gpu_arch != current_platform.get_device_capability()
if backend_config.specific_gpu_arch and (
not current_platform.is_cuda()
or backend_config.specific_gpu_arch != current_platform.get_device_capability()
):
if backend_config.specific_gpu_arch == (9, 0):
pytest.skip("Only Hopper GPUs support FA3 and FlashMLA")
elif backend_config.specific_gpu_arch == (10, 0):
pytest.skip("Only Blackwell GPUs support Cutlass MLA")
# FlashInfer is not supported on ROCm
if backend_config == AttentionBackendEnum.FLASHINFER and current_platform.is_rocm():
pytest.skip("FlashInfer is not supported on ROCm")
env_vars = {
# Force native sampler to avoid potential nondeterminism in FlashInfer
# when per-request generators are not used in V1.
@@ -86,7 +87,11 @@ def llm_pair(request):
trust_remote_code=True,
max_model_len=1024,
max_num_seqs=128,
compilation_config=CompilationConfig(**backend_config.comp_config),
attention_config=backend_config.attention_config,
compilation_config=CompilationConfig(
**backend_config.comp_config,
use_inductor_graph_partition=use_inductor_graph_partition,
),
generation_config="vllm",
seed=42,
)
@@ -96,8 +101,10 @@ def llm_pair(request):
trust_remote_code=True,
max_model_len=1024,
max_num_seqs=128,
attention_config=backend_config.attention_config,
compilation_config=CompilationConfig(
cudagraph_mode=CUDAGraphMode.PIECEWISE
cudagraph_mode=CUDAGraphMode.PIECEWISE,
use_inductor_graph_partition=use_inductor_graph_partition,
),
generation_config="vllm",
seed=42,
@@ -117,7 +124,10 @@ def llm_pair(request):
@pytest.mark.parametrize(
"llm_pair",
[
pytest.param((model, backend_config, use_inductor_graph_partition))
pytest.param(
(model, backend_config, use_inductor_graph_partition),
id=f"{backend_config.name}-partition={use_inductor_graph_partition}",
)
for model, backend_config in model_backends_full_cudagraph
for use_inductor_graph_partition in [True, False]
],
@@ -22,6 +22,9 @@ from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.input_batch import InputBuffers
from vllm.v1.worker.gpu.pcp_manager import PCPManager
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
pytestmark = pytest.mark.cpu_test
@@ -165,6 +168,89 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
block_tables.get_dummy_slot_mappings.assert_not_called()
@pytest.mark.parametrize(
"full_cudagraph,max_query_len,expected_max_query_len",
[(False, None, None), (True, None, 8), (True, 1, 1), (True, 3, 3)],
)
def test_capture_query_length_covers_replay_batches(
full_cudagraph, max_query_len, expected_max_query_len
):
"""Mixed FULL graphs must capture prefill kernels as well as decode kernels."""
num_tokens = num_reqs = 8
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
block_tables = MagicMock()
block_tables.cp_size = 1
block_tables.get_dummy_block_tables.return_value = ()
block_tables.get_dummy_slot_mappings.return_value = torch.empty(
0, num_tokens, dtype=torch.int64
)
model_state = MagicMock()
kv_cache_config = KVCacheConfig(
num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[]
)
gpu_cudagraph_utils.prepare_inputs_to_capture(
num_reqs,
num_tokens,
model_state,
buffers,
block_tables,
[],
kv_cache_config,
full_cudagraph=full_cudagraph,
max_query_len=max_query_len,
)
batch = model_state.prepare_attn.call_args.args[0]
assert batch.num_scheduled_tokens.max() == 1
assert batch.max_query_len == expected_max_query_len
@pytest.mark.parametrize(
"uniform_token_count,max_query_len,expected_max_query_len",
[(1, None, 1), (None, 3, 3), (None, None, 8)],
)
def test_speculator_capture_preserves_decode_query_bounds(
monkeypatch, uniform_token_count, max_query_len, expected_max_query_len
):
"""Draft decode graphs keep narrow queries while mixed graphs cover prefill."""
num_tokens = num_reqs = 8
desc = BatchExecutionDescriptor(
CUDAGraphMode.FULL,
num_tokens,
num_reqs,
uniform_token_count=uniform_token_count,
max_query_len=max_query_len,
)
manager = SpeculatorCudaGraphManager.__new__(SpeculatorCudaGraphManager)
manager.max_num_reqs = num_reqs
manager.dp_size = 1
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
block_tables = MagicMock()
block_tables.cp_size = 1
block_tables.get_dummy_block_tables.return_value = ()
block_tables.get_dummy_slot_mappings.return_value = torch.empty(
0, num_tokens, dtype=torch.int64
)
model_state = MagicMock()
kv_cache_config = KVCacheConfig(
num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[]
)
def capture(self, create_forward_fn, progress_bar_desc):
for warmup in (True, False):
create_forward_fn(desc, warmup)(desc.cg_mode)
monkeypatch.setattr(gpu_cudagraph_utils.CudaGraphManager, "capture", capture)
manager.capture(
MagicMock(), model_state, buffers, block_tables, [], kv_cache_config
)
assert model_state.prepare_attn.call_count == 2
for call in model_state.prepare_attn.call_args_list:
assert call.args[0].max_query_len == expected_max_query_len
_DECODE_QUERY_LEN = 3
+5 -1
View File
@@ -704,7 +704,7 @@ class ModelCudaGraphManager(CudaGraphManager):
attn_groups,
kv_cache_config,
full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL,
max_query_len=desc.max_query_len,
max_query_len=desc.max_query_len or desc.uniform_token_count,
pcp_manager=pcp_manager,
)
@@ -775,6 +775,10 @@ def prepare_inputs_to_capture(
max_query_len: int | None = None,
pcp_manager: "PCPManager | None" = None,
) -> AttentionState:
if full_cudagraph and max_query_len is None:
# Mixed graphs can replay a single prefill spanning the entire batch,
# even when the dummy batch distributes one token to each request.
max_query_len = num_tokens
input_batch = InputBatch.make_dummy(
num_reqs, num_tokens, input_buffers, max_query_len=max_query_len
)
@@ -57,6 +57,7 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
attn_groups,
kv_cache_config,
full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL,
max_query_len=desc.max_query_len or desc.uniform_token_count,
)
return lambda cg_mode: forward_fn(