[BugFix] Fix DP token padding in dflash attention metadata (#56181)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -1,10 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
@@ -299,33 +297,6 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_reqs, self.num_speculative_steps
|
||||
)
|
||||
|
||||
def _build_draft_attn_metadata(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_reqs_padded: int,
|
||||
num_tokens_padded: int,
|
||||
seq_lens_cpu_upper_bound: torch.Tensor,
|
||||
step: int,
|
||||
num_query_per_req: int | None = None,
|
||||
causal: bool | Mapping[int, bool] = False,
|
||||
query_start_loc_np: np.ndarray | None = None,
|
||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self.draft_attn_layer_names:
|
||||
return None
|
||||
assert num_query_per_req is None # Omitted for DFlash, read from self instead
|
||||
return super()._build_draft_attn_metadata(
|
||||
num_reqs,
|
||||
num_reqs_padded,
|
||||
num_tokens_padded,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
step=step,
|
||||
num_query_per_req=self.num_query_per_req,
|
||||
causal=causal,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def propose(
|
||||
self,
|
||||
@@ -471,7 +442,6 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
need_eager=is_profile,
|
||||
dp_sync=batch_sync,
|
||||
)
|
||||
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
||||
num_tokens_padded = batch_desc.num_tokens
|
||||
num_tokens_across_dp = (
|
||||
batch_sync.num_tokens_across_dp if batch_sync is not None else None
|
||||
@@ -479,10 +449,10 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
|
||||
# Rebuild the draft attention metadata even when replaying the FULL
|
||||
# graph so that any attention metadata builder state is updated.
|
||||
draft_attn_metadata = self._build_draft_attn_metadata(
|
||||
draft_attn_metadata = self._build_uniform_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
num_tokens_padded=num_tokens_padded,
|
||||
batch_desc=batch_desc,
|
||||
num_query_per_req=self.num_query_per_req,
|
||||
seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound,
|
||||
step=self.num_query_per_req,
|
||||
causal=self._group_causal,
|
||||
|
||||
Reference in New Issue
Block a user