From 0a94d85a66499cad8297ead86a470967de5c0212 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 11 Aug 2026 17:15:06 -0700 Subject: [PATCH] [Bugfix][MRV2] Require all requests to be decoding for uniform-decode dispatch (#51865) Signed-off-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Signed-off-by: Nick Hill Co-authored-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Co-authored-by: Janelle Cai --- tests/v1/spec_decode/test_dynamic_sd_cug.py | 72 +++++++++ tests/v1/worker/test_gpu_batch_ordering.py | 151 +++++++++++++++++- vllm/v1/worker/gpu/cudagraph_utils.py | 14 +- vllm/v1/worker/gpu/input_batch.py | 8 +- vllm/v1/worker/gpu/model_runner.py | 111 +++++++++---- vllm/v1/worker/gpu/pcp_manager.py | 1 + .../spec_decode/autoregressive/speculator.py | 10 +- .../multi_module_mtp/speculator.py | 10 +- vllm/v1/worker/utils.py | 18 +++ 9 files changed, 341 insertions(+), 54 deletions(-) diff --git a/tests/v1/spec_decode/test_dynamic_sd_cug.py b/tests/v1/spec_decode/test_dynamic_sd_cug.py index d75495ea60..f47c7d3796 100644 --- a/tests/v1/spec_decode/test_dynamic_sd_cug.py +++ b/tests/v1/spec_decode/test_dynamic_sd_cug.py @@ -16,6 +16,7 @@ from vllm.config import ( VllmConfig, ) from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils +from vllm.v1.worker.utils import get_uniform_decode_token_count pytestmark = pytest.mark.cpu_test @@ -196,6 +197,77 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch): assert desc.num_active_loras == 0 +def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch): + """A batch holding K+1-token prompt chunks must not replay a decode graph. + + Prompt chunks of exactly K + 1 tokens give the batch a spec-decode shape + by coincidence; replaying the FULL graph over them corrupts every request + in the batch. See https://github.com/vllm-project/vllm/issues/49918. + """ + + max_spec_tokens = 7 + decode_query_len = max_spec_tokens + 1 + + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + + vllm_config = _create_vllm_config_for_dsd( + max_num_seqs=64, + max_spec_tokens=max_spec_tokens, + use_dynamic_sd=False, + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=vllm_config, + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=decode_query_len, + ) + manager._graphs_captured = True + + num_decodes, num_chunks = 6, 2 + num_reqs = num_decodes + num_chunks + num_tokens = num_reqs * decode_query_len + + # The batch shape is indistinguishable from a full batch of spec decodes. + assert ( + gpu_cudagraph_utils.get_uniform_token_count( + num_reqs, num_tokens, decode_query_len + ) + == decode_query_len + ) + + # Two of the requests are decode_query_len tokens into a longer prompt. + uniform_tok_count = get_uniform_decode_token_count( + num_reqs, num_tokens, decode_query_len, has_prefill=True + ) + assert uniform_tok_count is None + desc = manager.dispatch( + num_reqs=num_reqs, + num_tokens=num_tokens, + uniform_token_count=uniform_tok_count, + num_active_loras=0, + ) + assert desc.cg_mode == CUDAGraphMode.PIECEWISE + assert desc.uniform_token_count is None + + # The same shape with every request decoding still gets its FULL graph. + uniform_tok_count = get_uniform_decode_token_count( + num_reqs, num_tokens, decode_query_len, has_prefill=False + ) + assert uniform_tok_count == decode_query_len + desc = manager.dispatch( + num_reqs=num_reqs, + num_tokens=num_tokens, + uniform_token_count=uniform_tok_count, + num_active_loras=0, + ) + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.uniform_token_count == decode_query_len + + def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch): """Without DSD, only the max decode query length should get FULL graphs. diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index 2c2115dc81..d0cd402ce7 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -1,18 +1,28 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Batch ordering in Model Runner V2 (vllm.v1.worker.gpu). +"""Batch ordering and classification in Model Runner V2 (vllm.v1.worker.gpu). split_decodes_and_prefills assumes decode -> short_extend -> prefill request ordering. With spec decode (decode_query_len > 1), a shorter chunked-prefill tail sorted in front of the uniform decodes misclassifies every decode as a prefill. + +Conversely, a prompt chunk of exactly decode_query_len tokens has a decode +batch's shape, and must not be classified as a uniform decode batch. """ +import ast +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import numpy as np import torch from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.utils import split_decodes_and_prefills -from vllm.v1.worker.gpu.model_runner import sort_batch_req_ids +from vllm.v1.worker.gpu.model_runner import GPUModelRunner, sort_batch_req_ids +from vllm.v1.worker.utils import get_uniform_decode_token_count def _make_common_attn_metadata(query_lens: list[int]) -> CommonAttentionMetadata: @@ -37,6 +47,72 @@ def _make_common_attn_metadata(query_lens: list[int]) -> CommonAttentionMetadata ) +def _make_runner( + req_states: dict[str, tuple[int, int]], decode_query_len: int = 8 +) -> Any: + """Build a runner stub from {req_id: (num_computed_tokens, prefill_len)}.""" + prefill_lens = np.array([s[1] for s in req_states.values()], dtype=np.int32) + num_computed = np.array([s[0] for s in req_states.values()], dtype=np.int32) + runner: Any = GPUModelRunner.__new__(GPUModelRunner) + runner.decode_query_len = decode_query_len + runner.req_states = SimpleNamespace( + req_id_to_index={req_id: i for i, req_id in enumerate(req_states)}, + # The runner keeps this as min(num_computed_tokens, prefill_len). + num_computed_prefill_tokens=np.minimum(num_computed, prefill_lens), + prefill_len=SimpleNamespace(np=prefill_lens), + ) + return runner + + +def _uniform_token_count( + req_states: dict[str, tuple[int, int]], + query_len: int, + dummy_run: bool = False, +) -> int | None: + """Classify via the runner's pre-dispatch gather, as execute_model does.""" + scheduler_output = SimpleNamespace( + num_scheduled_tokens={req_id: query_len for req_id in req_states}, + total_num_scheduled_tokens=query_len * len(req_states), + ) + runner = _make_runner(req_states, decode_query_len=query_len) + _, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, dummy_run) + return uniform_tok_count + + +def test_spec_decode_batch_is_uniform_decode(): + # decode_query_len == 8 (K=7): every request past its prefill. + decodes = {f"d{i}": (16, 16) for i in range(6)} + assert _uniform_token_count(decodes, 8) == 8 + + +def test_prompt_chunk_of_decode_query_len_is_not_uniform_decode(): + # Two prompt chunks whose query length coincides with the K+1 spec-decode + # query length must reject the batch (issue #49918). + batch = {f"d{i}": (16, 16) for i in range(6)} + batch.update({f"p{i}": (8, 40) for i in range(2)}) + assert _uniform_token_count(batch, 8) is None + + # Same collision without spec decoding: 1-token prompt vs 1-token decode. + assert _uniform_token_count({"p0": (0, 1)}, 1) is None + + # A fresh prompt of exactly decode_query_len tokens. + assert _uniform_token_count({"p0": (0, 8)}, 8) is None + + +def test_dummy_batches_stay_uniform_decode(): + # Dummy batches have no request state to consult and must stay classified + # by shape alone; rejecting them would make capture and replay disagree. + scheduler_output = SimpleNamespace( + num_scheduled_tokens={f"_dummy_req_{i}": 8 for i in range(4)}, + total_num_scheduled_tokens=32, + ) + state, uniform_tok_count = _make_runner({}).gather_batch_req_state( + scheduler_output, True + ) + assert state is None + assert uniform_tok_count == 8 + + def test_sort_batch_req_ids_no_spec(): # decode_query_len == 1: plain ascending order (decodes first). num_tokens_per_req = {"p1": 100, "d1": 1, "p2": 7, "d2": 1} @@ -68,3 +144,74 @@ def test_spec_decodes_lead_short_prefill_tail(): ) assert (num_decodes, num_prefills) == (8, 1) assert (num_decode_tokens, num_prefill_tokens) == (16, 1) + + +def test_uniform_decode_uses_state_index_not_batch_position(): + """The gather must read each request's own state, not its batch slot. + + Batch position and state index diverge as requests come and go; here the + only prefilling request sits at state index 0, so a gather keyed on batch + position would read its row and wrongly reject the two decodes. + """ + runner: Any = GPUModelRunner.__new__(GPUModelRunner) + runner.decode_query_len = 8 + # State arrays in state-index order: a prefilling request, then two decodes. + runner.req_states = SimpleNamespace( + req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2}, + num_computed_prefill_tokens=np.array([8, 16, 16], dtype=np.int32), + prefill_len=SimpleNamespace(np=np.array([40, 16, 16], dtype=np.int32)), + ) + scheduler_output = SimpleNamespace( + num_scheduled_tokens={"decode_a": 8, "decode_b": 8}, + total_num_scheduled_tokens=16, + ) + + state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False) + assert not state.has_prefill + assert uniform_tok_count == 8 + + # Scheduling the prefilling request alongside them must reject the batch. + scheduler_output.num_scheduled_tokens = { + "decode_a": 8, + "prefilling": 8, + "decode_b": 8, + } + scheduler_output.total_num_scheduled_tokens = 24 + state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False) + assert state.has_prefill + assert uniform_tok_count is None + + +def test_uniform_decode_predicate(): + # Shape and prefill state must both pass. + assert get_uniform_decode_token_count(2, 16, 8, False) == 8 + assert get_uniform_decode_token_count(2, 16, 8, True) is None + # 12 tokens over 2 requests is no shared query length. + assert get_uniform_decode_token_count(2, 12, 8, False) is None + + +def test_no_speculator_dispatches_on_query_length_alone(): + """No speculator may pick its cudagraph from a shape-only test. + + Written over the whole package because speculators are added by copying an + existing one: the shape-only call already reappeared verbatim in a + speculator added after the original call sites were fixed. + """ + import vllm.v1.worker.gpu.spec_decode as spec_decode + + shape_only = {"get_uniform_token_count", "is_uniform_query_len"} + root = Path(spec_decode.__file__).parent + offenders = [] + for path in sorted(root.rglob("speculator.py")): + for node in ast.walk(ast.parse(path.read_text())): + if not isinstance(node, ast.Call): + continue + func = node.func + name = func.id if isinstance(func, ast.Name) else getattr(func, "attr", "") + if name in shape_only: + offenders.append(f"{path.relative_to(root)}:{node.lineno} {name}") + + assert not offenders, ( + "these speculators classify a decode batch by query length alone; use " + f"get_uniform_decode_token_count instead: {offenders}" + ) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 3a174cba80..b7731b6ada 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -35,7 +35,7 @@ from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.interface import ModelState -from vllm.v1.worker.utils import AttentionGroup +from vllm.v1.worker.utils import AttentionGroup, is_uniform_query_len logger = init_logger(__name__) @@ -90,17 +90,17 @@ def _is_compatible( def get_uniform_token_count( - num_reqs: int, - num_tokens: int, - max_query_len: int, + num_reqs: int, num_tokens: int, max_query_len: int ) -> int | None: """ Return the uniform token count if batch is uniform, else None. A batch is uniform if all requests have the same number of tokens. + + Shape test only, valid for batches known to be decoding by construction + (e.g. dummy runs). Scheduled batches must use + `get_uniform_decode_token_count`. """ - if (max_query_len == num_tokens // num_reqs) and ( - num_tokens == max_query_len * num_reqs - ): + if is_uniform_query_len(num_reqs, num_tokens, max_query_len): return max_query_len return None diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 8a952aa043..6362e86c1d 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -76,6 +76,8 @@ class InputBatch: num_computed_prefill_tokens_np: np.ndarray # [num_reqs] CPU bool array == (num_computed_prefill_tokens_np < prefill_len_np). is_prefilling_np: np.ndarray + # == np.any(is_prefilling_np) + has_prefill: bool # [num_reqs] only populated when pipeline parallelism is enabled. max_seq_len_np: np.ndarray | None @@ -101,10 +103,7 @@ class InputBatch: @classmethod def make_dummy( - cls, - num_reqs: int, - num_tokens: int, - input_buffers: InputBuffers, + cls, num_reqs: int, num_tokens: int, input_buffers: InputBuffers ) -> "InputBatch": assert 0 < num_reqs <= num_tokens device = input_buffers.device @@ -175,6 +174,7 @@ class InputBatch: prefill_len_np=np.zeros(num_reqs, dtype=np.int32), num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), + has_prefill=False, max_seq_len_np=None, input_ids=input_ids, positions=positions, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 3bf89ee156..43477e1ec1 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -133,7 +133,11 @@ from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin -from vllm.v1.worker.utils import KVBlockZeroer, copy_kv_cache_blocks_inplace +from vllm.v1.worker.utils import ( + KVBlockZeroer, + copy_kv_cache_blocks_inplace, + get_uniform_decode_token_count, +) logger = init_logger(__name__) @@ -974,21 +978,20 @@ class GPUModelRunner(LoRAModelRunnerMixin): scheduler_output.kv_cache_block_copies, ) - def prepare_inputs( - self, scheduler_output: SchedulerOutput, batch_desc: BatchExecutionDescriptor - ) -> InputBatch: - num_tokens = scheduler_output.total_num_scheduled_tokens - num_tokens_after_padding = batch_desc.num_tokens - assert num_tokens > 0 - if envs.VLLM_MOE_SKIP_PADDING: - # Mark trailing cudagraph-padding rows so kernels can skip work for - # them when supported. - self.input_buffers.is_padding[:num_tokens].fill_(False) - self.input_buffers.is_padding[num_tokens:num_tokens_after_padding].fill_( - True - ) + def gather_batch_req_state( + self, scheduler_output: SchedulerOutput, dummy_run: bool + ) -> tuple["BatchReqState | None", int | None]: + """Gather CPU request state for the scheduled batch, in batch order. + Returns (batch_state, uniform_decode_token_count) + """ num_tokens_per_req = scheduler_output.num_scheduled_tokens num_reqs = len(num_tokens_per_req) + num_toks = scheduler_output.total_num_scheduled_tokens + max_query_len = max(scheduler_output.num_scheduled_tokens.values()) + + if dummy_run: + # Dummy batches are uniform by construction. + return None, get_uniform_token_count(num_reqs, num_toks, max_query_len) # batch_idx -> req_id req_ids = sort_batch_req_ids(num_tokens_per_req, self.decode_query_len) @@ -997,7 +1000,46 @@ class GPUModelRunner(LoRAModelRunnerMixin): idx_mapping_iter = map(self.req_states.req_id_to_index.__getitem__, req_ids) idx_mapping_np = np.fromiter(idx_mapping_iter, dtype=np.intp, count=num_reqs) + prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np] + num_computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens[ + idx_mapping_np + ] + is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np + + batch_state = BatchReqState( + req_ids=req_ids, + num_scheduled_tokens=num_scheduled_tokens, + idx_mapping_np=idx_mapping_np, + prefill_len_np=prefill_len_np, + num_computed_prefill_tokens_np=num_computed_prefill_tokens_np, + is_prefilling_np=is_prefilling_np, + has_prefill=bool(is_prefilling_np.any()), + ) + return batch_state, get_uniform_decode_token_count( + num_reqs, num_toks, max_query_len, batch_state.has_prefill + ) + + def prepare_inputs( + self, + scheduler_output: SchedulerOutput, + batch_req_state: "BatchReqState", + batch_desc: BatchExecutionDescriptor, + ) -> InputBatch: + num_tokens = scheduler_output.total_num_scheduled_tokens + num_tokens_after_padding = batch_desc.num_tokens + assert num_tokens > 0 + if envs.VLLM_MOE_SKIP_PADDING: + # Mark trailing cudagraph-padding rows so kernels can skip work for + # them when supported. + is_padding = self.input_buffers.is_padding + is_padding[:num_tokens].fill_(False) + is_padding[num_tokens:num_tokens_after_padding].fill_(True) + + req_ids = batch_req_state.req_ids + num_scheduled_tokens_np = batch_req_state.num_scheduled_tokens + idx_mapping_np = batch_req_state.idx_mapping_np idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device) + num_reqs = len(req_ids) # Get the number of draft tokens for each request. draft_tokens = scheduler_output.scheduled_spec_decode_tokens @@ -1039,20 +1081,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_reqs_padded = batch_desc.num_reqs or num_reqs query_start_loc_np = np.empty(self.max_num_reqs + 1, dtype=np.int32) query_start_loc_np[0] = 0 - np.cumsum(num_scheduled_tokens, out=query_start_loc_np[1 : num_reqs + 1]) + np.cumsum(num_scheduled_tokens_np, out=query_start_loc_np[1 : num_reqs + 1]) # Pad for full CUDA graph mode. # Some attention backends like FA3 require query_start_loc to be non-decreasing. query_start_loc_np[num_reqs + 1 :] = num_tokens async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc) query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1] query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1] - prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np] - computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens - num_computed_prefill_tokens_np = computed_prefill_tokens_np[idx_mapping_np] - is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np # Get prefill tokens if any. - if np.any(is_prefilling_np): + if batch_req_state.has_prefill: prepare_prefill_inputs( self.input_buffers.input_ids, self.req_states.next_prefill_tokens, @@ -1106,7 +1144,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32) np.add( num_computed_tokens_np, - num_scheduled_tokens, + num_scheduled_tokens_np, out=seq_lens_cpu_upper_bound_np[:num_reqs], ) seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np) @@ -1129,7 +1167,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): idx_mapping_np=idx_mapping_np, expanded_idx_mapping=expanded_idx_mapping, expanded_local_pos=expanded_local_pos, - num_scheduled_tokens=num_scheduled_tokens, + num_scheduled_tokens=num_scheduled_tokens_np, num_tokens=num_tokens, num_tokens_after_padding=num_tokens_after_padding, num_draft_tokens=total_num_draft_tokens, @@ -1140,9 +1178,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=dcp_local_seq_lens, num_computed_tokens_np=num_computed_tokens_np, - prefill_len_np=prefill_len_np, - num_computed_prefill_tokens_np=num_computed_prefill_tokens_np, - is_prefilling_np=is_prefilling_np, + prefill_len_np=batch_req_state.prefill_len_np, + num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np, + is_prefilling_np=batch_req_state.is_prefilling_np, + has_prefill=batch_req_state.has_prefill, max_seq_len_np=max_seq_len_np, input_ids=self.input_buffers.input_ids[:num_tokens_after_padding], positions=self.input_buffers.positions[:num_tokens_after_padding], @@ -1275,8 +1314,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Get batch descriptor and sync across DP ranks. num_reqs = len(scheduler_output.num_scheduled_tokens) num_toks = scheduler_output.total_num_scheduled_tokens - max_query_len = max(scheduler_output.num_scheduled_tokens.values()) - uniform_tok_count = get_uniform_token_count(num_reqs, num_toks, max_query_len) + batch_req_state, uniform_tok_count = self.gather_batch_req_state( + scheduler_output, dummy_run + ) num_active_loras = 0 if self.lora_config: @@ -1311,7 +1351,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): if not dummy_run: # Common case. # Prepare all the inputs and copy to the input buffers. - input_batch = self.prepare_inputs(scheduler_output, batch_desc) + assert batch_req_state is not None + input_batch = self.prepare_inputs( + scheduler_output, batch_req_state, batch_desc + ) block_tables, slot_mappings = self.prepare_attn(input_batch) # Mamba "align" pre-copy: migrate recurrent state across block # boundaries before the forward. Runs only on real batches, and @@ -1790,6 +1833,18 @@ class ExecuteModelState(NamedTuple): routed_experts: RoutedExpertsTensors | None +class BatchReqState(NamedTuple): + """CPU request state for a scheduled batch, in batch (sorted) order.""" + + req_ids: list[str] + num_scheduled_tokens: np.ndarray # [num_reqs] + idx_mapping_np: np.ndarray # [num_reqs] + prefill_len_np: np.ndarray # [num_reqs] + num_computed_prefill_tokens_np: np.ndarray # [num_reqs] + is_prefilling_np: np.ndarray # [num_reqs] + has_prefill: bool + + def sort_batch_req_ids( num_tokens_per_req: dict[str, int], decode_query_len: int ) -> list[str]: diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index 587293471c..d80e246f17 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -529,6 +529,7 @@ class PCPManager: prefill_len_np=local_prefill_len_np, num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np, is_prefilling_np=local_is_prefilling_np, + has_prefill=bool(local_is_prefilling_np.any()), max_seq_len_np=global_batch.max_seq_len_np[local_to_global_batch_req_idx_np] if global_batch.max_seq_len_np is not None else None, diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index 7d357533e2..57430529f6 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -13,10 +13,7 @@ from vllm.triton_utils import tl, triton from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.block_table import BlockTables -from vllm.v1.worker.gpu.cudagraph_utils import ( - BatchExecutionDescriptor, - get_uniform_token_count, -) +from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.interface import ModelState @@ -24,7 +21,7 @@ from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import ( SpeculatorCudaGraphManager, ) from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator -from vllm.v1.worker.utils import AttentionGroup +from vllm.v1.worker.utils import AttentionGroup, get_uniform_decode_token_count logger = init_logger(__name__) @@ -274,12 +271,13 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): # When all requests are decoding (no true prefills), each has # num_speculative_steps + 1 tokens, enabling FULL graph replay. - uniform_token_count = get_uniform_token_count( + uniform_token_count = get_uniform_decode_token_count( num_reqs, # Use the actual number of tokens without padding added by # the target model during FULL cudagraph. num_tokens, max_query_len, + input_batch.has_prefill, ) prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( self.prefill_cudagraph_manager, diff --git a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py index 4be22b4474..fc147b3f49 100644 --- a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py @@ -12,9 +12,6 @@ from vllm.logger import init_logger from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer -from vllm.v1.worker.gpu.cudagraph_utils import ( - get_uniform_token_count, -) from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import ( @@ -22,6 +19,7 @@ from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import ( ) from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator +from vllm.v1.worker.utils import get_uniform_decode_token_count logger = init_logger(__name__) @@ -190,10 +188,8 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator): # When all requests are decoding (no true prefills), each has # num_speculative_steps + 1 tokens, enabling FULL graph replay. - uniform_token_count = get_uniform_token_count( - num_reqs, - num_tokens, - max_query_len, + uniform_token_count = get_uniform_decode_token_count( + num_reqs, num_tokens, max_query_len, input_batch.has_prefill ) batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( self.cudagraph_manager, diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index 3c8630a5e3..f495579ddb 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -611,6 +611,24 @@ def copy_kv_cache_blocks_inplace( blocks[dst_indices] = blocks[src_indices] +def is_uniform_query_len(num_reqs: int, num_tokens: int, max_query_len: int) -> bool: + """Whether every request in the batch has the same query length. + + Shape test only; use ``get_uniform_decode_token_count`` to classify a + scheduled batch, since a prompt chunk can have a decode batch's shape. + """ + return num_reqs > 0 and num_tokens == max_query_len * num_reqs + + +def get_uniform_decode_token_count( + num_reqs: int, num_tokens: int, max_query_len: int, has_prefill: bool +) -> int | None: + """Per-request token count of a uniform decode batch, or None.""" + if not has_prefill and is_uniform_query_len(num_reqs, num_tokens, max_query_len): + return max_query_len + return None + + def is_residual_scattered_for_sp( vllm_config: VllmConfig, num_input_tokens: int ) -> bool: