[Refactor][MRV2] Unify uniform decode token count helper (#51917)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
f2906b492e
commit
02ac17851d
@@ -124,10 +124,11 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
|
||||
# Uniform decode means every request contributes the same number of
|
||||
# tokens, so the total token count is exactly num_reqs * query_len.
|
||||
num_tokens = num_reqs * max_query_len
|
||||
uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count(
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
)
|
||||
|
||||
# The scheduler should mark every one of these shapes as a uniform
|
||||
@@ -233,8 +234,8 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
|
||||
# The batch shape is indistinguishable from a full batch of spec decodes.
|
||||
assert (
|
||||
gpu_cudagraph_utils.get_uniform_token_count(
|
||||
num_reqs, num_tokens, decode_query_len
|
||||
get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=False
|
||||
)
|
||||
== decode_query_len
|
||||
)
|
||||
@@ -305,10 +306,11 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
# These are still uniform decode batches, but basic SD should only
|
||||
# have FULL graphs for query_len == max_decode_query_len.
|
||||
num_tokens = num_reqs * max_query_len
|
||||
uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count(
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
)
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
@@ -370,10 +372,11 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
for num_reqs in range(1, max_num_seqs + 1):
|
||||
for max_query_len in range(1, max_decode_query_len + 1):
|
||||
num_tokens = num_reqs * max_query_len
|
||||
uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count(
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
)
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.utils import AttentionGroup, is_uniform_query_len
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -89,22 +89,6 @@ def _is_compatible(
|
||||
)
|
||||
|
||||
|
||||
def get_uniform_token_count(
|
||||
num_reqs: int, num_tokens: int, max_query_len: int
|
||||
) -> int | None:
|
||||
"""
|
||||
Return the uniform token count if batch is uniform, else None.
|
||||
A batch is uniform if all requests have the same number of tokens.
|
||||
|
||||
Shape test only, valid for batches known to be decoding by construction
|
||||
(e.g. dummy runs). Scheduled batches must use
|
||||
`get_uniform_decode_token_count`.
|
||||
"""
|
||||
if is_uniform_query_len(num_reqs, num_tokens, max_query_len):
|
||||
return max_query_len
|
||||
return None
|
||||
|
||||
|
||||
class CudaGraphManager:
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -88,7 +88,6 @@ from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
ModelCudaGraphManager,
|
||||
get_uniform_token_count,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.ec_connector import get_ec_connector
|
||||
@@ -991,7 +990,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
if dummy_run:
|
||||
# Dummy batches are uniform by construction.
|
||||
return None, get_uniform_token_count(num_reqs, num_toks, max_query_len)
|
||||
return None, get_uniform_decode_token_count(
|
||||
num_reqs, num_toks, max_query_len, has_prefill=False
|
||||
)
|
||||
|
||||
# batch_idx -> req_id
|
||||
req_ids = sort_batch_req_ids(num_tokens_per_req, self.decode_query_len)
|
||||
|
||||
Reference in New Issue
Block a user