[Bugfix][MRV2] Require all requests to be decoding for uniform-decode dispatch (#51865)

Signed-off-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com>
Co-authored-by: Janelle Cai <janelle.cai@modal.com>
This commit is contained in:
Nick Hill
2026-08-11 17:15:06 -07:00
committed by GitHub
co-authored by Rahul Chalamala Janelle Cai
parent 1d2d83a07f
commit 0a94d85a66
9 changed files with 341 additions and 54 deletions
@@ -16,6 +16,7 @@ from vllm.config import (
VllmConfig,
)
from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils
from vllm.v1.worker.utils import get_uniform_decode_token_count
pytestmark = pytest.mark.cpu_test
@@ -196,6 +197,77 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
assert desc.num_active_loras == 0
def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
"""A batch holding K+1-token prompt chunks must not replay a decode graph.
Prompt chunks of exactly K + 1 tokens give the batch a spec-decode shape
by coincidence; replaying the FULL graph over them corrupts every request
in the batch. See https://github.com/vllm-project/vllm/issues/49918.
"""
max_spec_tokens = 7
decode_query_len = max_spec_tokens + 1
monkeypatch.setattr(
gpu_cudagraph_utils,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
vllm_config = _create_vllm_config_for_dsd(
max_num_seqs=64,
max_spec_tokens=max_spec_tokens,
use_dynamic_sd=False,
)
manager = gpu_cudagraph_utils.CudaGraphManager(
vllm_config=vllm_config,
device=torch.device("cpu"),
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
decode_query_len=decode_query_len,
)
manager._graphs_captured = True
num_decodes, num_chunks = 6, 2
num_reqs = num_decodes + num_chunks
num_tokens = num_reqs * decode_query_len
# The batch shape is indistinguishable from a full batch of spec decodes.
assert (
gpu_cudagraph_utils.get_uniform_token_count(
num_reqs, num_tokens, decode_query_len
)
== decode_query_len
)
# Two of the requests are decode_query_len tokens into a longer prompt.
uniform_tok_count = get_uniform_decode_token_count(
num_reqs, num_tokens, decode_query_len, has_prefill=True
)
assert uniform_tok_count is None
desc = manager.dispatch(
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
# The same shape with every request decoding still gets its FULL graph.
uniform_tok_count = get_uniform_decode_token_count(
num_reqs, num_tokens, decode_query_len, has_prefill=False
)
assert uniform_tok_count == decode_query_len
desc = manager.dispatch(
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.uniform_token_count == decode_query_len
def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
"""Without DSD, only the max decode query length should get FULL graphs.
+149 -2
View File
@@ -1,18 +1,28 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Batch ordering in Model Runner V2 (vllm.v1.worker.gpu).
"""Batch ordering and classification in Model Runner V2 (vllm.v1.worker.gpu).
split_decodes_and_prefills assumes decode -> short_extend -> prefill request
ordering. With spec decode (decode_query_len > 1), a shorter chunked-prefill
tail sorted in front of the uniform decodes misclassifies every decode as a
prefill.
Conversely, a prompt chunk of exactly decode_query_len tokens has a decode
batch's shape, and must not be classified as a uniform decode batch.
"""
import ast
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import numpy as np
import torch
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.worker.gpu.model_runner import sort_batch_req_ids
from vllm.v1.worker.gpu.model_runner import GPUModelRunner, sort_batch_req_ids
from vllm.v1.worker.utils import get_uniform_decode_token_count
def _make_common_attn_metadata(query_lens: list[int]) -> CommonAttentionMetadata:
@@ -37,6 +47,72 @@ def _make_common_attn_metadata(query_lens: list[int]) -> CommonAttentionMetadata
)
def _make_runner(
req_states: dict[str, tuple[int, int]], decode_query_len: int = 8
) -> Any:
"""Build a runner stub from {req_id: (num_computed_tokens, prefill_len)}."""
prefill_lens = np.array([s[1] for s in req_states.values()], dtype=np.int32)
num_computed = np.array([s[0] for s in req_states.values()], dtype=np.int32)
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
runner.decode_query_len = decode_query_len
runner.req_states = SimpleNamespace(
req_id_to_index={req_id: i for i, req_id in enumerate(req_states)},
# The runner keeps this as min(num_computed_tokens, prefill_len).
num_computed_prefill_tokens=np.minimum(num_computed, prefill_lens),
prefill_len=SimpleNamespace(np=prefill_lens),
)
return runner
def _uniform_token_count(
req_states: dict[str, tuple[int, int]],
query_len: int,
dummy_run: bool = False,
) -> int | None:
"""Classify via the runner's pre-dispatch gather, as execute_model does."""
scheduler_output = SimpleNamespace(
num_scheduled_tokens={req_id: query_len for req_id in req_states},
total_num_scheduled_tokens=query_len * len(req_states),
)
runner = _make_runner(req_states, decode_query_len=query_len)
_, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, dummy_run)
return uniform_tok_count
def test_spec_decode_batch_is_uniform_decode():
# decode_query_len == 8 (K=7): every request past its prefill.
decodes = {f"d{i}": (16, 16) for i in range(6)}
assert _uniform_token_count(decodes, 8) == 8
def test_prompt_chunk_of_decode_query_len_is_not_uniform_decode():
# Two prompt chunks whose query length coincides with the K+1 spec-decode
# query length must reject the batch (issue #49918).
batch = {f"d{i}": (16, 16) for i in range(6)}
batch.update({f"p{i}": (8, 40) for i in range(2)})
assert _uniform_token_count(batch, 8) is None
# Same collision without spec decoding: 1-token prompt vs 1-token decode.
assert _uniform_token_count({"p0": (0, 1)}, 1) is None
# A fresh prompt of exactly decode_query_len tokens.
assert _uniform_token_count({"p0": (0, 8)}, 8) is None
def test_dummy_batches_stay_uniform_decode():
# Dummy batches have no request state to consult and must stay classified
# by shape alone; rejecting them would make capture and replay disagree.
scheduler_output = SimpleNamespace(
num_scheduled_tokens={f"_dummy_req_{i}": 8 for i in range(4)},
total_num_scheduled_tokens=32,
)
state, uniform_tok_count = _make_runner({}).gather_batch_req_state(
scheduler_output, True
)
assert state is None
assert uniform_tok_count == 8
def test_sort_batch_req_ids_no_spec():
# decode_query_len == 1: plain ascending order (decodes first).
num_tokens_per_req = {"p1": 100, "d1": 1, "p2": 7, "d2": 1}
@@ -68,3 +144,74 @@ def test_spec_decodes_lead_short_prefill_tail():
)
assert (num_decodes, num_prefills) == (8, 1)
assert (num_decode_tokens, num_prefill_tokens) == (16, 1)
def test_uniform_decode_uses_state_index_not_batch_position():
"""The gather must read each request's own state, not its batch slot.
Batch position and state index diverge as requests come and go; here the
only prefilling request sits at state index 0, so a gather keyed on batch
position would read its row and wrongly reject the two decodes.
"""
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
runner.decode_query_len = 8
# State arrays in state-index order: a prefilling request, then two decodes.
runner.req_states = SimpleNamespace(
req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2},
num_computed_prefill_tokens=np.array([8, 16, 16], dtype=np.int32),
prefill_len=SimpleNamespace(np=np.array([40, 16, 16], dtype=np.int32)),
)
scheduler_output = SimpleNamespace(
num_scheduled_tokens={"decode_a": 8, "decode_b": 8},
total_num_scheduled_tokens=16,
)
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
assert not state.has_prefill
assert uniform_tok_count == 8
# Scheduling the prefilling request alongside them must reject the batch.
scheduler_output.num_scheduled_tokens = {
"decode_a": 8,
"prefilling": 8,
"decode_b": 8,
}
scheduler_output.total_num_scheduled_tokens = 24
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
assert state.has_prefill
assert uniform_tok_count is None
def test_uniform_decode_predicate():
# Shape and prefill state must both pass.
assert get_uniform_decode_token_count(2, 16, 8, False) == 8
assert get_uniform_decode_token_count(2, 16, 8, True) is None
# 12 tokens over 2 requests is no shared query length.
assert get_uniform_decode_token_count(2, 12, 8, False) is None
def test_no_speculator_dispatches_on_query_length_alone():
"""No speculator may pick its cudagraph from a shape-only test.
Written over the whole package because speculators are added by copying an
existing one: the shape-only call already reappeared verbatim in a
speculator added after the original call sites were fixed.
"""
import vllm.v1.worker.gpu.spec_decode as spec_decode
shape_only = {"get_uniform_token_count", "is_uniform_query_len"}
root = Path(spec_decode.__file__).parent
offenders = []
for path in sorted(root.rglob("speculator.py")):
for node in ast.walk(ast.parse(path.read_text())):
if not isinstance(node, ast.Call):
continue
func = node.func
name = func.id if isinstance(func, ast.Name) else getattr(func, "attr", "")
if name in shape_only:
offenders.append(f"{path.relative_to(root)}:{node.lineno} {name}")
assert not offenders, (
"these speculators classify a decode batch by query length alone; use "
f"get_uniform_decode_token_count instead: {offenders}"
)
+7 -7
View File
@@ -35,7 +35,7 @@ from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.utils import AttentionGroup
from vllm.v1.worker.utils import AttentionGroup, is_uniform_query_len
logger = init_logger(__name__)
@@ -90,17 +90,17 @@ def _is_compatible(
def get_uniform_token_count(
num_reqs: int,
num_tokens: int,
max_query_len: int,
num_reqs: int, num_tokens: int, max_query_len: int
) -> int | None:
"""
Return the uniform token count if batch is uniform, else None.
A batch is uniform if all requests have the same number of tokens.
Shape test only, valid for batches known to be decoding by construction
(e.g. dummy runs). Scheduled batches must use
`get_uniform_decode_token_count`.
"""
if (max_query_len == num_tokens // num_reqs) and (
num_tokens == max_query_len * num_reqs
):
if is_uniform_query_len(num_reqs, num_tokens, max_query_len):
return max_query_len
return None
+4 -4
View File
@@ -76,6 +76,8 @@ class InputBatch:
num_computed_prefill_tokens_np: np.ndarray
# [num_reqs] CPU bool array == (num_computed_prefill_tokens_np < prefill_len_np).
is_prefilling_np: np.ndarray
# == np.any(is_prefilling_np)
has_prefill: bool
# [num_reqs] only populated when pipeline parallelism is enabled.
max_seq_len_np: np.ndarray | None
@@ -101,10 +103,7 @@ class InputBatch:
@classmethod
def make_dummy(
cls,
num_reqs: int,
num_tokens: int,
input_buffers: InputBuffers,
cls, num_reqs: int, num_tokens: int, input_buffers: InputBuffers
) -> "InputBatch":
assert 0 < num_reqs <= num_tokens
device = input_buffers.device
@@ -175,6 +174,7 @@ class InputBatch:
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
has_prefill=False,
max_seq_len_np=None,
input_ids=input_ids,
positions=positions,
+83 -28
View File
@@ -133,7 +133,11 @@ from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
from vllm.v1.worker.utils import KVBlockZeroer, copy_kv_cache_blocks_inplace
from vllm.v1.worker.utils import (
KVBlockZeroer,
copy_kv_cache_blocks_inplace,
get_uniform_decode_token_count,
)
logger = init_logger(__name__)
@@ -974,21 +978,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
scheduler_output.kv_cache_block_copies,
)
def prepare_inputs(
self, scheduler_output: SchedulerOutput, batch_desc: BatchExecutionDescriptor
) -> InputBatch:
num_tokens = scheduler_output.total_num_scheduled_tokens
num_tokens_after_padding = batch_desc.num_tokens
assert num_tokens > 0
if envs.VLLM_MOE_SKIP_PADDING:
# Mark trailing cudagraph-padding rows so kernels can skip work for
# them when supported.
self.input_buffers.is_padding[:num_tokens].fill_(False)
self.input_buffers.is_padding[num_tokens:num_tokens_after_padding].fill_(
True
)
def gather_batch_req_state(
self, scheduler_output: SchedulerOutput, dummy_run: bool
) -> tuple["BatchReqState | None", int | None]:
"""Gather CPU request state for the scheduled batch, in batch order.
Returns (batch_state, uniform_decode_token_count)
"""
num_tokens_per_req = scheduler_output.num_scheduled_tokens
num_reqs = len(num_tokens_per_req)
num_toks = scheduler_output.total_num_scheduled_tokens
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
if dummy_run:
# Dummy batches are uniform by construction.
return None, get_uniform_token_count(num_reqs, num_toks, max_query_len)
# batch_idx -> req_id
req_ids = sort_batch_req_ids(num_tokens_per_req, self.decode_query_len)
@@ -997,7 +1000,46 @@ class GPUModelRunner(LoRAModelRunnerMixin):
idx_mapping_iter = map(self.req_states.req_id_to_index.__getitem__, req_ids)
idx_mapping_np = np.fromiter(idx_mapping_iter, dtype=np.intp, count=num_reqs)
prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np]
num_computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens[
idx_mapping_np
]
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
batch_state = BatchReqState(
req_ids=req_ids,
num_scheduled_tokens=num_scheduled_tokens,
idx_mapping_np=idx_mapping_np,
prefill_len_np=prefill_len_np,
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
is_prefilling_np=is_prefilling_np,
has_prefill=bool(is_prefilling_np.any()),
)
return batch_state, get_uniform_decode_token_count(
num_reqs, num_toks, max_query_len, batch_state.has_prefill
)
def prepare_inputs(
self,
scheduler_output: SchedulerOutput,
batch_req_state: "BatchReqState",
batch_desc: BatchExecutionDescriptor,
) -> InputBatch:
num_tokens = scheduler_output.total_num_scheduled_tokens
num_tokens_after_padding = batch_desc.num_tokens
assert num_tokens > 0
if envs.VLLM_MOE_SKIP_PADDING:
# Mark trailing cudagraph-padding rows so kernels can skip work for
# them when supported.
is_padding = self.input_buffers.is_padding
is_padding[:num_tokens].fill_(False)
is_padding[num_tokens:num_tokens_after_padding].fill_(True)
req_ids = batch_req_state.req_ids
num_scheduled_tokens_np = batch_req_state.num_scheduled_tokens
idx_mapping_np = batch_req_state.idx_mapping_np
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
num_reqs = len(req_ids)
# Get the number of draft tokens for each request.
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
@@ -1039,20 +1081,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_reqs_padded = batch_desc.num_reqs or num_reqs
query_start_loc_np = np.empty(self.max_num_reqs + 1, dtype=np.int32)
query_start_loc_np[0] = 0
np.cumsum(num_scheduled_tokens, out=query_start_loc_np[1 : num_reqs + 1])
np.cumsum(num_scheduled_tokens_np, out=query_start_loc_np[1 : num_reqs + 1])
# Pad for full CUDA graph mode.
# Some attention backends like FA3 require query_start_loc to be non-decreasing.
query_start_loc_np[num_reqs + 1 :] = num_tokens
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np]
computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens
num_computed_prefill_tokens_np = computed_prefill_tokens_np[idx_mapping_np]
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
# Get prefill tokens if any.
if np.any(is_prefilling_np):
if batch_req_state.has_prefill:
prepare_prefill_inputs(
self.input_buffers.input_ids,
self.req_states.next_prefill_tokens,
@@ -1106,7 +1144,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
np.add(
num_computed_tokens_np,
num_scheduled_tokens,
num_scheduled_tokens_np,
out=seq_lens_cpu_upper_bound_np[:num_reqs],
)
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
@@ -1129,7 +1167,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
idx_mapping_np=idx_mapping_np,
expanded_idx_mapping=expanded_idx_mapping,
expanded_local_pos=expanded_local_pos,
num_scheduled_tokens=num_scheduled_tokens,
num_scheduled_tokens=num_scheduled_tokens_np,
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens_after_padding,
num_draft_tokens=total_num_draft_tokens,
@@ -1140,9 +1178,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=dcp_local_seq_lens,
num_computed_tokens_np=num_computed_tokens_np,
prefill_len_np=prefill_len_np,
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
is_prefilling_np=is_prefilling_np,
prefill_len_np=batch_req_state.prefill_len_np,
num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np,
is_prefilling_np=batch_req_state.is_prefilling_np,
has_prefill=batch_req_state.has_prefill,
max_seq_len_np=max_seq_len_np,
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
positions=self.input_buffers.positions[:num_tokens_after_padding],
@@ -1275,8 +1314,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Get batch descriptor and sync across DP ranks.
num_reqs = len(scheduler_output.num_scheduled_tokens)
num_toks = scheduler_output.total_num_scheduled_tokens
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
uniform_tok_count = get_uniform_token_count(num_reqs, num_toks, max_query_len)
batch_req_state, uniform_tok_count = self.gather_batch_req_state(
scheduler_output, dummy_run
)
num_active_loras = 0
if self.lora_config:
@@ -1311,7 +1351,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if not dummy_run:
# Common case.
# Prepare all the inputs and copy to the input buffers.
input_batch = self.prepare_inputs(scheduler_output, batch_desc)
assert batch_req_state is not None
input_batch = self.prepare_inputs(
scheduler_output, batch_req_state, batch_desc
)
block_tables, slot_mappings = self.prepare_attn(input_batch)
# Mamba "align" pre-copy: migrate recurrent state across block
# boundaries before the forward. Runs only on real batches, and
@@ -1790,6 +1833,18 @@ class ExecuteModelState(NamedTuple):
routed_experts: RoutedExpertsTensors | None
class BatchReqState(NamedTuple):
"""CPU request state for a scheduled batch, in batch (sorted) order."""
req_ids: list[str]
num_scheduled_tokens: np.ndarray # [num_reqs]
idx_mapping_np: np.ndarray # [num_reqs]
prefill_len_np: np.ndarray # [num_reqs]
num_computed_prefill_tokens_np: np.ndarray # [num_reqs]
is_prefilling_np: np.ndarray # [num_reqs]
has_prefill: bool
def sort_batch_req_ids(
num_tokens_per_req: dict[str, int], decode_query_len: int
) -> list[str]:
+1
View File
@@ -529,6 +529,7 @@ class PCPManager:
prefill_len_np=local_prefill_len_np,
num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np,
is_prefilling_np=local_is_prefilling_np,
has_prefill=bool(local_is_prefilling_np.any()),
max_seq_len_np=global_batch.max_seq_len_np[local_to_global_batch_req_idx_np]
if global_batch.max_seq_len_np is not None
else None,
@@ -13,10 +13,7 @@ from vllm.triton_utils import tl, triton
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
get_uniform_token_count,
)
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
@@ -24,7 +21,7 @@ from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.utils import AttentionGroup
from vllm.v1.worker.utils import AttentionGroup, get_uniform_decode_token_count
logger = init_logger(__name__)
@@ -274,12 +271,13 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
# When all requests are decoding (no true prefills), each has
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
uniform_token_count = get_uniform_token_count(
uniform_token_count = get_uniform_decode_token_count(
num_reqs,
# Use the actual number of tokens without padding added by
# the target model during FULL cudagraph.
num_tokens,
max_query_len,
input_batch.has_prefill,
)
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
self.prefill_cudagraph_manager,
@@ -12,9 +12,6 @@ from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.cudagraph_utils import (
get_uniform_token_count,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
@@ -22,6 +19,7 @@ from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
)
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.utils import get_uniform_decode_token_count
logger = init_logger(__name__)
@@ -190,10 +188,8 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
# When all requests are decoding (no true prefills), each has
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
uniform_token_count = get_uniform_token_count(
num_reqs,
num_tokens,
max_query_len,
uniform_token_count = get_uniform_decode_token_count(
num_reqs, num_tokens, max_query_len, input_batch.has_prefill
)
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
self.cudagraph_manager,
+18
View File
@@ -611,6 +611,24 @@ def copy_kv_cache_blocks_inplace(
blocks[dst_indices] = blocks[src_indices]
def is_uniform_query_len(num_reqs: int, num_tokens: int, max_query_len: int) -> bool:
"""Whether every request in the batch has the same query length.
Shape test only; use ``get_uniform_decode_token_count`` to classify a
scheduled batch, since a prompt chunk can have a decode batch's shape.
"""
return num_reqs > 0 and num_tokens == max_query_len * num_reqs
def get_uniform_decode_token_count(
num_reqs: int, num_tokens: int, max_query_len: int, has_prefill: bool
) -> int | None:
"""Per-request token count of a uniform decode batch, or None."""
if not has_prefill and is_uniform_query_len(num_reqs, num_tokens, max_query_len):
return max_query_len
return None
def is_residual_scattered_for_sp(
vllm_config: VllmConfig, num_input_tokens: int
) -> bool: