[Bugfix][Model Runner V2] Order uniform decodes first so spec decodes aren't misclassified as prefills (#47381)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude Fable 5
Nick Hill
parent
b83be00cdd
commit
85b3a7264b
@@ -1156,7 +1156,12 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
cache_config,
|
||||
quant_config,
|
||||
prefix,
|
||||
skip_topk=_skip_topk,
|
||||
# MTP layers must never start with skip_topk=True: their indexer
|
||||
# computes indices at draft step 0, and the runtime toggle
|
||||
# (set_skip_topk, index_share_for_mtp_iteration) only exists in
|
||||
# the V1 proposer. A frozen True would leave the draft reading a
|
||||
# never-written topk buffer.
|
||||
skip_topk=_skip_topk and not is_mtp_layer,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -854,9 +854,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_tokens_per_req = scheduler_output.num_scheduled_tokens
|
||||
num_reqs = len(num_tokens_per_req)
|
||||
|
||||
# Decode first, then prefill.
|
||||
# batch_idx -> req_id
|
||||
req_ids = sorted(num_tokens_per_req, key=num_tokens_per_req.get) # type: ignore[arg-type]
|
||||
req_ids = sort_batch_req_ids(num_tokens_per_req, self.decode_query_len)
|
||||
numtoks_iter = map(num_tokens_per_req.get, req_ids)
|
||||
num_scheduled_tokens = np.fromiter(numtoks_iter, dtype=np.int32, count=num_reqs)
|
||||
|
||||
@@ -1607,3 +1606,12 @@ class ExecuteModelState(NamedTuple):
|
||||
hidden_states: torch.Tensor | None
|
||||
aux_hidden_states: list[torch.Tensor] | None
|
||||
finished_req_ids: set[str]
|
||||
|
||||
|
||||
def sort_batch_req_ids(
|
||||
num_tokens_per_req: dict[str, int], decode_query_len: int
|
||||
) -> list[str]:
|
||||
# Order decode -> short_extend -> prefill; split_decodes_and_prefills
|
||||
# relies on uniform decodes (query_len == decode_query_len) leading.
|
||||
key = lambda r: ((num := num_tokens_per_req[r]) != decode_query_len, num)
|
||||
return sorted(num_tokens_per_req, key=key)
|
||||
|
||||
Reference in New Issue
Block a user