[Bugfix] Capture prefill kernels for mixed FULL graphs (#58275)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-24 02:05:07 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent bbc4ddeca9
commit 23110a0c9e
4 changed files with 117 additions and 16 deletions
@@ -22,6 +22,9 @@ from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.input_batch import InputBuffers
from vllm.v1.worker.gpu.pcp_manager import PCPManager
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
pytestmark = pytest.mark.cpu_test
@@ -165,6 +168,89 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
block_tables.get_dummy_slot_mappings.assert_not_called()
@pytest.mark.parametrize(
"full_cudagraph,max_query_len,expected_max_query_len",
[(False, None, None), (True, None, 8), (True, 1, 1), (True, 3, 3)],
)
def test_capture_query_length_covers_replay_batches(
full_cudagraph, max_query_len, expected_max_query_len
):
"""Mixed FULL graphs must capture prefill kernels as well as decode kernels."""
num_tokens = num_reqs = 8
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
block_tables = MagicMock()
block_tables.cp_size = 1
block_tables.get_dummy_block_tables.return_value = ()
block_tables.get_dummy_slot_mappings.return_value = torch.empty(
0, num_tokens, dtype=torch.int64
)
model_state = MagicMock()
kv_cache_config = KVCacheConfig(
num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[]
)
gpu_cudagraph_utils.prepare_inputs_to_capture(
num_reqs,
num_tokens,
model_state,
buffers,
block_tables,
[],
kv_cache_config,
full_cudagraph=full_cudagraph,
max_query_len=max_query_len,
)
batch = model_state.prepare_attn.call_args.args[0]
assert batch.num_scheduled_tokens.max() == 1
assert batch.max_query_len == expected_max_query_len
@pytest.mark.parametrize(
"uniform_token_count,max_query_len,expected_max_query_len",
[(1, None, 1), (None, 3, 3), (None, None, 8)],
)
def test_speculator_capture_preserves_decode_query_bounds(
monkeypatch, uniform_token_count, max_query_len, expected_max_query_len
):
"""Draft decode graphs keep narrow queries while mixed graphs cover prefill."""
num_tokens = num_reqs = 8
desc = BatchExecutionDescriptor(
CUDAGraphMode.FULL,
num_tokens,
num_reqs,
uniform_token_count=uniform_token_count,
max_query_len=max_query_len,
)
manager = SpeculatorCudaGraphManager.__new__(SpeculatorCudaGraphManager)
manager.max_num_reqs = num_reqs
manager.dp_size = 1
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
block_tables = MagicMock()
block_tables.cp_size = 1
block_tables.get_dummy_block_tables.return_value = ()
block_tables.get_dummy_slot_mappings.return_value = torch.empty(
0, num_tokens, dtype=torch.int64
)
model_state = MagicMock()
kv_cache_config = KVCacheConfig(
num_blocks=0, kv_cache_tensors=[], kv_cache_groups=[]
)
def capture(self, create_forward_fn, progress_bar_desc):
for warmup in (True, False):
create_forward_fn(desc, warmup)(desc.cg_mode)
monkeypatch.setattr(gpu_cudagraph_utils.CudaGraphManager, "capture", capture)
manager.capture(
MagicMock(), model_state, buffers, block_tables, [], kv_cache_config
)
assert model_state.prepare_attn.call_count == 2
for call in model_state.prepare_attn.call_args_list:
assert call.args[0].max_query_len == expected_max_query_len
_DECODE_QUERY_LEN = 3