[Bugfix][MRV2] Require all requests to be decoding for uniform-decode dispatch (#51865)
Signed-off-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Co-authored-by: Janelle Cai <janelle.cai@modal.com>
This commit is contained in:
co-authored by
Rahul Chalamala
Janelle Cai
parent
1d2d83a07f
commit
0a94d85a66
@@ -16,6 +16,7 @@ from vllm.config import (
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils
|
||||
from vllm.v1.worker.utils import get_uniform_decode_token_count
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -196,6 +197,77 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
|
||||
assert desc.num_active_loras == 0
|
||||
|
||||
|
||||
def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
"""A batch holding K+1-token prompt chunks must not replay a decode graph.
|
||||
|
||||
Prompt chunks of exactly K + 1 tokens give the batch a spec-decode shape
|
||||
by coincidence; replaying the FULL graph over them corrupts every request
|
||||
in the batch. See https://github.com/vllm-project/vllm/issues/49918.
|
||||
"""
|
||||
|
||||
max_spec_tokens = 7
|
||||
decode_query_len = max_spec_tokens + 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
|
||||
vllm_config = _create_vllm_config_for_dsd(
|
||||
max_num_seqs=64,
|
||||
max_spec_tokens=max_spec_tokens,
|
||||
use_dynamic_sd=False,
|
||||
)
|
||||
manager = gpu_cudagraph_utils.CudaGraphManager(
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cpu"),
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
decode_query_len=decode_query_len,
|
||||
)
|
||||
manager._graphs_captured = True
|
||||
|
||||
num_decodes, num_chunks = 6, 2
|
||||
num_reqs = num_decodes + num_chunks
|
||||
num_tokens = num_reqs * decode_query_len
|
||||
|
||||
# The batch shape is indistinguishable from a full batch of spec decodes.
|
||||
assert (
|
||||
gpu_cudagraph_utils.get_uniform_token_count(
|
||||
num_reqs, num_tokens, decode_query_len
|
||||
)
|
||||
== decode_query_len
|
||||
)
|
||||
|
||||
# Two of the requests are decode_query_len tokens into a longer prompt.
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=True
|
||||
)
|
||||
assert uniform_tok_count is None
|
||||
desc = manager.dispatch(
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=uniform_tok_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
assert desc.uniform_token_count is None
|
||||
|
||||
# The same shape with every request decoding still gets its FULL graph.
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=False
|
||||
)
|
||||
assert uniform_tok_count == decode_query_len
|
||||
desc = manager.dispatch(
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=uniform_tok_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
assert desc.cg_mode == CUDAGraphMode.FULL
|
||||
assert desc.uniform_token_count == decode_query_len
|
||||
|
||||
|
||||
def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
"""Without DSD, only the max decode query length should get FULL graphs.
|
||||
|
||||
|
||||
@@ -1,18 +1,28 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Batch ordering in Model Runner V2 (vllm.v1.worker.gpu).
|
||||
"""Batch ordering and classification in Model Runner V2 (vllm.v1.worker.gpu).
|
||||
|
||||
split_decodes_and_prefills assumes decode -> short_extend -> prefill request
|
||||
ordering. With spec decode (decode_query_len > 1), a shorter chunked-prefill
|
||||
tail sorted in front of the uniform decodes misclassifies every decode as a
|
||||
prefill.
|
||||
|
||||
Conversely, a prompt chunk of exactly decode_query_len tokens has a decode
|
||||
batch's shape, and must not be classified as a uniform decode batch.
|
||||
"""
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.worker.gpu.model_runner import sort_batch_req_ids
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner, sort_batch_req_ids
|
||||
from vllm.v1.worker.utils import get_uniform_decode_token_count
|
||||
|
||||
|
||||
def _make_common_attn_metadata(query_lens: list[int]) -> CommonAttentionMetadata:
|
||||
@@ -37,6 +47,72 @@ def _make_common_attn_metadata(query_lens: list[int]) -> CommonAttentionMetadata
|
||||
)
|
||||
|
||||
|
||||
def _make_runner(
|
||||
req_states: dict[str, tuple[int, int]], decode_query_len: int = 8
|
||||
) -> Any:
|
||||
"""Build a runner stub from {req_id: (num_computed_tokens, prefill_len)}."""
|
||||
prefill_lens = np.array([s[1] for s in req_states.values()], dtype=np.int32)
|
||||
num_computed = np.array([s[0] for s in req_states.values()], dtype=np.int32)
|
||||
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.decode_query_len = decode_query_len
|
||||
runner.req_states = SimpleNamespace(
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(req_states)},
|
||||
# The runner keeps this as min(num_computed_tokens, prefill_len).
|
||||
num_computed_prefill_tokens=np.minimum(num_computed, prefill_lens),
|
||||
prefill_len=SimpleNamespace(np=prefill_lens),
|
||||
)
|
||||
return runner
|
||||
|
||||
|
||||
def _uniform_token_count(
|
||||
req_states: dict[str, tuple[int, int]],
|
||||
query_len: int,
|
||||
dummy_run: bool = False,
|
||||
) -> int | None:
|
||||
"""Classify via the runner's pre-dispatch gather, as execute_model does."""
|
||||
scheduler_output = SimpleNamespace(
|
||||
num_scheduled_tokens={req_id: query_len for req_id in req_states},
|
||||
total_num_scheduled_tokens=query_len * len(req_states),
|
||||
)
|
||||
runner = _make_runner(req_states, decode_query_len=query_len)
|
||||
_, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, dummy_run)
|
||||
return uniform_tok_count
|
||||
|
||||
|
||||
def test_spec_decode_batch_is_uniform_decode():
|
||||
# decode_query_len == 8 (K=7): every request past its prefill.
|
||||
decodes = {f"d{i}": (16, 16) for i in range(6)}
|
||||
assert _uniform_token_count(decodes, 8) == 8
|
||||
|
||||
|
||||
def test_prompt_chunk_of_decode_query_len_is_not_uniform_decode():
|
||||
# Two prompt chunks whose query length coincides with the K+1 spec-decode
|
||||
# query length must reject the batch (issue #49918).
|
||||
batch = {f"d{i}": (16, 16) for i in range(6)}
|
||||
batch.update({f"p{i}": (8, 40) for i in range(2)})
|
||||
assert _uniform_token_count(batch, 8) is None
|
||||
|
||||
# Same collision without spec decoding: 1-token prompt vs 1-token decode.
|
||||
assert _uniform_token_count({"p0": (0, 1)}, 1) is None
|
||||
|
||||
# A fresh prompt of exactly decode_query_len tokens.
|
||||
assert _uniform_token_count({"p0": (0, 8)}, 8) is None
|
||||
|
||||
|
||||
def test_dummy_batches_stay_uniform_decode():
|
||||
# Dummy batches have no request state to consult and must stay classified
|
||||
# by shape alone; rejecting them would make capture and replay disagree.
|
||||
scheduler_output = SimpleNamespace(
|
||||
num_scheduled_tokens={f"_dummy_req_{i}": 8 for i in range(4)},
|
||||
total_num_scheduled_tokens=32,
|
||||
)
|
||||
state, uniform_tok_count = _make_runner({}).gather_batch_req_state(
|
||||
scheduler_output, True
|
||||
)
|
||||
assert state is None
|
||||
assert uniform_tok_count == 8
|
||||
|
||||
|
||||
def test_sort_batch_req_ids_no_spec():
|
||||
# decode_query_len == 1: plain ascending order (decodes first).
|
||||
num_tokens_per_req = {"p1": 100, "d1": 1, "p2": 7, "d2": 1}
|
||||
@@ -68,3 +144,74 @@ def test_spec_decodes_lead_short_prefill_tail():
|
||||
)
|
||||
assert (num_decodes, num_prefills) == (8, 1)
|
||||
assert (num_decode_tokens, num_prefill_tokens) == (16, 1)
|
||||
|
||||
|
||||
def test_uniform_decode_uses_state_index_not_batch_position():
|
||||
"""The gather must read each request's own state, not its batch slot.
|
||||
|
||||
Batch position and state index diverge as requests come and go; here the
|
||||
only prefilling request sits at state index 0, so a gather keyed on batch
|
||||
position would read its row and wrongly reject the two decodes.
|
||||
"""
|
||||
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.decode_query_len = 8
|
||||
# State arrays in state-index order: a prefilling request, then two decodes.
|
||||
runner.req_states = SimpleNamespace(
|
||||
req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2},
|
||||
num_computed_prefill_tokens=np.array([8, 16, 16], dtype=np.int32),
|
||||
prefill_len=SimpleNamespace(np=np.array([40, 16, 16], dtype=np.int32)),
|
||||
)
|
||||
scheduler_output = SimpleNamespace(
|
||||
num_scheduled_tokens={"decode_a": 8, "decode_b": 8},
|
||||
total_num_scheduled_tokens=16,
|
||||
)
|
||||
|
||||
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
|
||||
assert not state.has_prefill
|
||||
assert uniform_tok_count == 8
|
||||
|
||||
# Scheduling the prefilling request alongside them must reject the batch.
|
||||
scheduler_output.num_scheduled_tokens = {
|
||||
"decode_a": 8,
|
||||
"prefilling": 8,
|
||||
"decode_b": 8,
|
||||
}
|
||||
scheduler_output.total_num_scheduled_tokens = 24
|
||||
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
|
||||
assert state.has_prefill
|
||||
assert uniform_tok_count is None
|
||||
|
||||
|
||||
def test_uniform_decode_predicate():
|
||||
# Shape and prefill state must both pass.
|
||||
assert get_uniform_decode_token_count(2, 16, 8, False) == 8
|
||||
assert get_uniform_decode_token_count(2, 16, 8, True) is None
|
||||
# 12 tokens over 2 requests is no shared query length.
|
||||
assert get_uniform_decode_token_count(2, 12, 8, False) is None
|
||||
|
||||
|
||||
def test_no_speculator_dispatches_on_query_length_alone():
|
||||
"""No speculator may pick its cudagraph from a shape-only test.
|
||||
|
||||
Written over the whole package because speculators are added by copying an
|
||||
existing one: the shape-only call already reappeared verbatim in a
|
||||
speculator added after the original call sites were fixed.
|
||||
"""
|
||||
import vllm.v1.worker.gpu.spec_decode as spec_decode
|
||||
|
||||
shape_only = {"get_uniform_token_count", "is_uniform_query_len"}
|
||||
root = Path(spec_decode.__file__).parent
|
||||
offenders = []
|
||||
for path in sorted(root.rglob("speculator.py")):
|
||||
for node in ast.walk(ast.parse(path.read_text())):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
func = node.func
|
||||
name = func.id if isinstance(func, ast.Name) else getattr(func, "attr", "")
|
||||
if name in shape_only:
|
||||
offenders.append(f"{path.relative_to(root)}:{node.lineno} {name}")
|
||||
|
||||
assert not offenders, (
|
||||
"these speculators classify a decode batch by query length alone; use "
|
||||
f"get_uniform_decode_token_count instead: {offenders}"
|
||||
)
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
from vllm.v1.worker.utils import AttentionGroup, is_uniform_query_len
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -90,17 +90,17 @@ def _is_compatible(
|
||||
|
||||
|
||||
def get_uniform_token_count(
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
max_query_len: int,
|
||||
num_reqs: int, num_tokens: int, max_query_len: int
|
||||
) -> int | None:
|
||||
"""
|
||||
Return the uniform token count if batch is uniform, else None.
|
||||
A batch is uniform if all requests have the same number of tokens.
|
||||
|
||||
Shape test only, valid for batches known to be decoding by construction
|
||||
(e.g. dummy runs). Scheduled batches must use
|
||||
`get_uniform_decode_token_count`.
|
||||
"""
|
||||
if (max_query_len == num_tokens // num_reqs) and (
|
||||
num_tokens == max_query_len * num_reqs
|
||||
):
|
||||
if is_uniform_query_len(num_reqs, num_tokens, max_query_len):
|
||||
return max_query_len
|
||||
return None
|
||||
|
||||
|
||||
@@ -76,6 +76,8 @@ class InputBatch:
|
||||
num_computed_prefill_tokens_np: np.ndarray
|
||||
# [num_reqs] CPU bool array == (num_computed_prefill_tokens_np < prefill_len_np).
|
||||
is_prefilling_np: np.ndarray
|
||||
# == np.any(is_prefilling_np)
|
||||
has_prefill: bool
|
||||
|
||||
# [num_reqs] only populated when pipeline parallelism is enabled.
|
||||
max_seq_len_np: np.ndarray | None
|
||||
@@ -101,10 +103,7 @@ class InputBatch:
|
||||
|
||||
@classmethod
|
||||
def make_dummy(
|
||||
cls,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
input_buffers: InputBuffers,
|
||||
cls, num_reqs: int, num_tokens: int, input_buffers: InputBuffers
|
||||
) -> "InputBatch":
|
||||
assert 0 < num_reqs <= num_tokens
|
||||
device = input_buffers.device
|
||||
@@ -175,6 +174,7 @@ class InputBatch:
|
||||
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
has_prefill=False,
|
||||
max_seq_len_np=None,
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
|
||||
@@ -133,7 +133,11 @@ from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
|
||||
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
|
||||
from vllm.v1.worker.utils import KVBlockZeroer, copy_kv_cache_blocks_inplace
|
||||
from vllm.v1.worker.utils import (
|
||||
KVBlockZeroer,
|
||||
copy_kv_cache_blocks_inplace,
|
||||
get_uniform_decode_token_count,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -974,21 +978,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
scheduler_output.kv_cache_block_copies,
|
||||
)
|
||||
|
||||
def prepare_inputs(
|
||||
self, scheduler_output: SchedulerOutput, batch_desc: BatchExecutionDescriptor
|
||||
) -> InputBatch:
|
||||
num_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
num_tokens_after_padding = batch_desc.num_tokens
|
||||
assert num_tokens > 0
|
||||
if envs.VLLM_MOE_SKIP_PADDING:
|
||||
# Mark trailing cudagraph-padding rows so kernels can skip work for
|
||||
# them when supported.
|
||||
self.input_buffers.is_padding[:num_tokens].fill_(False)
|
||||
self.input_buffers.is_padding[num_tokens:num_tokens_after_padding].fill_(
|
||||
True
|
||||
)
|
||||
def gather_batch_req_state(
|
||||
self, scheduler_output: SchedulerOutput, dummy_run: bool
|
||||
) -> tuple["BatchReqState | None", int | None]:
|
||||
"""Gather CPU request state for the scheduled batch, in batch order.
|
||||
Returns (batch_state, uniform_decode_token_count)
|
||||
"""
|
||||
num_tokens_per_req = scheduler_output.num_scheduled_tokens
|
||||
num_reqs = len(num_tokens_per_req)
|
||||
num_toks = scheduler_output.total_num_scheduled_tokens
|
||||
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
|
||||
|
||||
if dummy_run:
|
||||
# Dummy batches are uniform by construction.
|
||||
return None, get_uniform_token_count(num_reqs, num_toks, max_query_len)
|
||||
|
||||
# batch_idx -> req_id
|
||||
req_ids = sort_batch_req_ids(num_tokens_per_req, self.decode_query_len)
|
||||
@@ -997,7 +1000,46 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
idx_mapping_iter = map(self.req_states.req_id_to_index.__getitem__, req_ids)
|
||||
idx_mapping_np = np.fromiter(idx_mapping_iter, dtype=np.intp, count=num_reqs)
|
||||
prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np]
|
||||
num_computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens[
|
||||
idx_mapping_np
|
||||
]
|
||||
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
|
||||
|
||||
batch_state = BatchReqState(
|
||||
req_ids=req_ids,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
prefill_len_np=prefill_len_np,
|
||||
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=is_prefilling_np,
|
||||
has_prefill=bool(is_prefilling_np.any()),
|
||||
)
|
||||
return batch_state, get_uniform_decode_token_count(
|
||||
num_reqs, num_toks, max_query_len, batch_state.has_prefill
|
||||
)
|
||||
|
||||
def prepare_inputs(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
batch_req_state: "BatchReqState",
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
) -> InputBatch:
|
||||
num_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
num_tokens_after_padding = batch_desc.num_tokens
|
||||
assert num_tokens > 0
|
||||
if envs.VLLM_MOE_SKIP_PADDING:
|
||||
# Mark trailing cudagraph-padding rows so kernels can skip work for
|
||||
# them when supported.
|
||||
is_padding = self.input_buffers.is_padding
|
||||
is_padding[:num_tokens].fill_(False)
|
||||
is_padding[num_tokens:num_tokens_after_padding].fill_(True)
|
||||
|
||||
req_ids = batch_req_state.req_ids
|
||||
num_scheduled_tokens_np = batch_req_state.num_scheduled_tokens
|
||||
idx_mapping_np = batch_req_state.idx_mapping_np
|
||||
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
|
||||
num_reqs = len(req_ids)
|
||||
|
||||
# Get the number of draft tokens for each request.
|
||||
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
@@ -1039,20 +1081,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
||||
query_start_loc_np = np.empty(self.max_num_reqs + 1, dtype=np.int32)
|
||||
query_start_loc_np[0] = 0
|
||||
np.cumsum(num_scheduled_tokens, out=query_start_loc_np[1 : num_reqs + 1])
|
||||
np.cumsum(num_scheduled_tokens_np, out=query_start_loc_np[1 : num_reqs + 1])
|
||||
# Pad for full CUDA graph mode.
|
||||
# Some attention backends like FA3 require query_start_loc to be non-decreasing.
|
||||
query_start_loc_np[num_reqs + 1 :] = num_tokens
|
||||
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
|
||||
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
|
||||
prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np]
|
||||
computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens
|
||||
num_computed_prefill_tokens_np = computed_prefill_tokens_np[idx_mapping_np]
|
||||
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
|
||||
|
||||
# Get prefill tokens if any.
|
||||
if np.any(is_prefilling_np):
|
||||
if batch_req_state.has_prefill:
|
||||
prepare_prefill_inputs(
|
||||
self.input_buffers.input_ids,
|
||||
self.req_states.next_prefill_tokens,
|
||||
@@ -1106,7 +1144,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
|
||||
np.add(
|
||||
num_computed_tokens_np,
|
||||
num_scheduled_tokens,
|
||||
num_scheduled_tokens_np,
|
||||
out=seq_lens_cpu_upper_bound_np[:num_reqs],
|
||||
)
|
||||
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
|
||||
@@ -1129,7 +1167,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
expanded_idx_mapping=expanded_idx_mapping,
|
||||
expanded_local_pos=expanded_local_pos,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
num_scheduled_tokens=num_scheduled_tokens_np,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_tokens_after_padding,
|
||||
num_draft_tokens=total_num_draft_tokens,
|
||||
@@ -1140,9 +1178,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
num_computed_tokens_np=num_computed_tokens_np,
|
||||
prefill_len_np=prefill_len_np,
|
||||
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=is_prefilling_np,
|
||||
prefill_len_np=batch_req_state.prefill_len_np,
|
||||
num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=batch_req_state.is_prefilling_np,
|
||||
has_prefill=batch_req_state.has_prefill,
|
||||
max_seq_len_np=max_seq_len_np,
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
||||
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
||||
@@ -1275,8 +1314,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Get batch descriptor and sync across DP ranks.
|
||||
num_reqs = len(scheduler_output.num_scheduled_tokens)
|
||||
num_toks = scheduler_output.total_num_scheduled_tokens
|
||||
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
|
||||
uniform_tok_count = get_uniform_token_count(num_reqs, num_toks, max_query_len)
|
||||
batch_req_state, uniform_tok_count = self.gather_batch_req_state(
|
||||
scheduler_output, dummy_run
|
||||
)
|
||||
|
||||
num_active_loras = 0
|
||||
if self.lora_config:
|
||||
@@ -1311,7 +1351,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if not dummy_run:
|
||||
# Common case.
|
||||
# Prepare all the inputs and copy to the input buffers.
|
||||
input_batch = self.prepare_inputs(scheduler_output, batch_desc)
|
||||
assert batch_req_state is not None
|
||||
input_batch = self.prepare_inputs(
|
||||
scheduler_output, batch_req_state, batch_desc
|
||||
)
|
||||
block_tables, slot_mappings = self.prepare_attn(input_batch)
|
||||
# Mamba "align" pre-copy: migrate recurrent state across block
|
||||
# boundaries before the forward. Runs only on real batches, and
|
||||
@@ -1790,6 +1833,18 @@ class ExecuteModelState(NamedTuple):
|
||||
routed_experts: RoutedExpertsTensors | None
|
||||
|
||||
|
||||
class BatchReqState(NamedTuple):
|
||||
"""CPU request state for a scheduled batch, in batch (sorted) order."""
|
||||
|
||||
req_ids: list[str]
|
||||
num_scheduled_tokens: np.ndarray # [num_reqs]
|
||||
idx_mapping_np: np.ndarray # [num_reqs]
|
||||
prefill_len_np: np.ndarray # [num_reqs]
|
||||
num_computed_prefill_tokens_np: np.ndarray # [num_reqs]
|
||||
is_prefilling_np: np.ndarray # [num_reqs]
|
||||
has_prefill: bool
|
||||
|
||||
|
||||
def sort_batch_req_ids(
|
||||
num_tokens_per_req: dict[str, int], decode_query_len: int
|
||||
) -> list[str]:
|
||||
|
||||
@@ -529,6 +529,7 @@ class PCPManager:
|
||||
prefill_len_np=local_prefill_len_np,
|
||||
num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=local_is_prefilling_np,
|
||||
has_prefill=bool(local_is_prefilling_np.any()),
|
||||
max_seq_len_np=global_batch.max_seq_len_np[local_to_global_batch_req_idx_np]
|
||||
if global_batch.max_seq_len_np is not None
|
||||
else None,
|
||||
|
||||
@@ -13,10 +13,7 @@ from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
get_uniform_token_count,
|
||||
)
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
@@ -24,7 +21,7 @@ from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
SpeculatorCudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
from vllm.v1.worker.utils import AttentionGroup, get_uniform_decode_token_count
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -274,12 +271,13 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
|
||||
# When all requests are decoding (no true prefills), each has
|
||||
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
|
||||
uniform_token_count = get_uniform_token_count(
|
||||
uniform_token_count = get_uniform_decode_token_count(
|
||||
num_reqs,
|
||||
# Use the actual number of tokens without padding added by
|
||||
# the target model during FULL cudagraph.
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
input_batch.has_prefill,
|
||||
)
|
||||
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
self.prefill_cudagraph_manager,
|
||||
|
||||
@@ -12,9 +12,6 @@ from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
get_uniform_token_count,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
@@ -22,6 +19,7 @@ from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.utils import get_uniform_decode_token_count
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -190,10 +188,8 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
|
||||
# When all requests are decoding (no true prefills), each has
|
||||
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
|
||||
uniform_token_count = get_uniform_token_count(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
uniform_token_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, max_query_len, input_batch.has_prefill
|
||||
)
|
||||
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
self.cudagraph_manager,
|
||||
|
||||
@@ -611,6 +611,24 @@ def copy_kv_cache_blocks_inplace(
|
||||
blocks[dst_indices] = blocks[src_indices]
|
||||
|
||||
|
||||
def is_uniform_query_len(num_reqs: int, num_tokens: int, max_query_len: int) -> bool:
|
||||
"""Whether every request in the batch has the same query length.
|
||||
|
||||
Shape test only; use ``get_uniform_decode_token_count`` to classify a
|
||||
scheduled batch, since a prompt chunk can have a decode batch's shape.
|
||||
"""
|
||||
return num_reqs > 0 and num_tokens == max_query_len * num_reqs
|
||||
|
||||
|
||||
def get_uniform_decode_token_count(
|
||||
num_reqs: int, num_tokens: int, max_query_len: int, has_prefill: bool
|
||||
) -> int | None:
|
||||
"""Per-request token count of a uniform decode batch, or None."""
|
||||
if not has_prefill and is_uniform_query_len(num_reqs, num_tokens, max_query_len):
|
||||
return max_query_len
|
||||
return None
|
||||
|
||||
|
||||
def is_residual_scattered_for_sp(
|
||||
vllm_config: VllmConfig, num_input_tokens: int
|
||||
) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user