[BugFix] Fix DP token padding in dflash attention metadata (#56181)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-09-11 14:33:53 -07:00
committed by GitHub
parent c1b69aa0d4
commit 9dcf6bf344
5 changed files with 97 additions and 97 deletions
@@ -532,11 +532,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, self.kv_cache_config
)
attn_metadata = self._build_draft_attn_metadata(
attn_metadata = self._build_uniform_attn_metadata(
num_reqs=num_reqs,
num_reqs_padded=batch_desc.num_reqs or num_reqs,
# One query per request; exclude DP-only model padding.
num_tokens_padded=batch_desc.num_reqs or num_reqs,
batch_desc=batch_desc,
num_query_per_req=1,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
step=step,
)
@@ -581,11 +580,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, self.kv_cache_config
)
attn_metadata = self._build_draft_attn_metadata(
attn_metadata = self._build_uniform_attn_metadata(
num_reqs=num_reqs,
num_reqs_padded=batch_desc.num_reqs or num_reqs,
# One query per request; exclude DP-only model padding.
num_tokens_padded=batch_desc.num_reqs or num_reqs,
batch_desc=batch_desc,
num_query_per_req=1,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
step=1,
)
@@ -1,10 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from collections.abc import Mapping
from typing import Any
import numpy as np
import torch
import torch.nn as nn
@@ -299,33 +297,6 @@ class DFlashSpeculator(DraftModelSpeculator):
num_reqs, self.num_speculative_steps
)
def _build_draft_attn_metadata(
self,
num_reqs: int,
num_reqs_padded: int,
num_tokens_padded: int,
seq_lens_cpu_upper_bound: torch.Tensor,
step: int,
num_query_per_req: int | None = None,
causal: bool | Mapping[int, bool] = False,
query_start_loc_np: np.ndarray | None = None,
dcp_local_seq_lens: torch.Tensor | None = None,
) -> dict[str, Any] | None:
if not self.draft_attn_layer_names:
return None
assert num_query_per_req is None # Omitted for DFlash, read from self instead
return super()._build_draft_attn_metadata(
num_reqs,
num_reqs_padded,
num_tokens_padded,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
step=step,
num_query_per_req=self.num_query_per_req,
causal=causal,
query_start_loc_np=query_start_loc_np,
dcp_local_seq_lens=dcp_local_seq_lens,
)
@torch.inference_mode()
def propose(
self,
@@ -471,7 +442,6 @@ class DFlashSpeculator(DraftModelSpeculator):
need_eager=is_profile,
dp_sync=batch_sync,
)
num_reqs_padded = batch_desc.num_reqs or num_reqs
num_tokens_padded = batch_desc.num_tokens
num_tokens_across_dp = (
batch_sync.num_tokens_across_dp if batch_sync is not None else None
@@ -479,10 +449,10 @@ class DFlashSpeculator(DraftModelSpeculator):
# Rebuild the draft attention metadata even when replaying the FULL
# graph so that any attention metadata builder state is updated.
draft_attn_metadata = self._build_draft_attn_metadata(
draft_attn_metadata = self._build_uniform_attn_metadata(
num_reqs=num_reqs,
num_reqs_padded=num_reqs_padded,
num_tokens_padded=num_tokens_padded,
batch_desc=batch_desc,
num_query_per_req=self.num_query_per_req,
seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound,
step=self.num_query_per_req,
causal=self._group_causal,
@@ -226,10 +226,9 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
slot_mappings = build_slot_mappings_by_layer(
slot_mappings_tensor, self.kv_cache_config
)
draft_attn_metadata = self._build_draft_attn_metadata(
draft_attn_metadata = self._build_attn_metadata(
num_reqs=num_reqs,
num_reqs_padded=batch_desc.num_reqs or num_reqs,
num_tokens_padded=batch_desc.num_tokens,
batch_desc=batch_desc,
query_start_loc_np=input_batch.query_start_loc_np,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
step=0,
+45 -30
View File
@@ -23,6 +23,7 @@ from vllm.v1.worker.gpu.attn_utils import (
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.dp_utils import DPSyncState
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
@@ -149,9 +150,7 @@ class DraftModelSpeculator(BaseSpeculator):
dtype=torch.int64,
device=device,
)
self.arange = torch.arange(
self.max_num_reqs + 1, dtype=torch.int32, device="cpu"
)
self.arange_np = np.arange(self.max_num_reqs + 1, dtype=np.int32)
self.draft_is_prefilling = torch.zeros(self.max_num_reqs, dtype=torch.bool)
self.draft_logits: torch.Tensor | None = None
@@ -254,42 +253,37 @@ class DraftModelSpeculator(BaseSpeculator):
self.target_input_buffers = target_input_buffers
self.target_attn_groups = target_attn_groups
def _build_draft_attn_metadata(
def _build_attn_metadata(
self,
num_reqs: int,
num_reqs_padded: int,
num_tokens_padded: int,
batch_desc: BatchExecutionDescriptor,
query_start_loc_np: np.ndarray,
seq_lens_cpu_upper_bound: torch.Tensor,
step: int,
num_query_per_req: int = 1,
causal: bool | Mapping[int, bool] = True,
query_start_loc_np: np.ndarray | None = None,
dcp_local_seq_lens: torch.Tensor | None = None,
) -> dict[str, Any] | None:
if query_start_loc_np is not None:
# Non-uniform query layout (e.g. multi-module MTP's mixed
# prefill/decode queries); num_query_per_req is ignored.
query_start_loc_cpu = torch.empty(num_reqs_padded + 1, dtype=torch.int32)
query_start_loc_cpu[: num_reqs + 1] = torch.from_numpy(
query_start_loc_np[: num_reqs + 1]
)
query_start_loc_cpu[num_reqs:] = query_start_loc_cpu[num_reqs]
max_query_len = int(
(query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]).max()
)
else:
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
# Clamp keeps the series non-decreasing past num_reqs, which some
# attention backends require.
query_start_loc_cpu = (
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
* num_query_per_req
)
max_query_len = num_query_per_req
num_reqs_padded = batch_desc.num_reqs or num_reqs
# A FULL graph replays a captured shape whose padded requests each hold
# a full query width, so attention must see the padded token count.
# PIECEWISE/eager needs the actual token count, because batch_desc may
# carry graph or DP padding that no request owns, which would desync it
# from query_start_loc.
num_tokens = (
batch_desc.num_tokens
if batch_desc.cg_mode == CUDAGraphMode.FULL
else int(query_start_loc_np[-1])
)
query_start_loc_cpu = torch.empty(num_reqs_padded + 1, dtype=torch.int32)
query_start_loc_cpu[: num_reqs + 1] = torch.from_numpy(
query_start_loc_np[: num_reqs + 1]
)
query_start_loc_cpu[num_reqs:] = query_start_loc_cpu[num_reqs]
max_query_len = int((query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]).max())
block_tables = [
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
]
slot_mappings = self.block_tables.slot_mappings[:, :num_tokens_padded]
slot_mappings = self.block_tables.slot_mappings[:, :num_tokens]
draft_seq_lens_cpu_upper_bound = torch.zeros(
num_reqs_padded, dtype=torch.int32, device="cpu"
)
@@ -313,7 +307,7 @@ class DraftModelSpeculator(BaseSpeculator):
attn_metadata = build_attn_metadata(
attn_groups=self.attn_groups,
num_reqs=num_reqs_padded,
num_tokens=num_tokens_padded,
num_tokens=num_tokens,
query_start_loc_gpu=self.input_buffers.query_start_loc[
: num_reqs_padded + 1
],
@@ -434,3 +428,24 @@ class DraftModelSpeculator(BaseSpeculator):
uniform_token_count=num_query_per_req,
eager=False,
), num_batch_tokens
def _build_uniform_attn_metadata(
self,
batch_desc: BatchExecutionDescriptor,
num_reqs: int,
num_query_per_req: int,
seq_lens_cpu_upper_bound: torch.Tensor,
step: int,
causal: bool | Mapping[int, bool] = True,
dcp_local_seq_lens: torch.Tensor | None = None,
) -> dict[str, Any] | None:
query_start_loc_np = self.arange_np[: num_reqs + 1] * num_query_per_req
return self._build_attn_metadata(
num_reqs=num_reqs,
batch_desc=batch_desc,
query_start_loc_np=query_start_loc_np,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
step=step,
causal=causal,
dcp_local_seq_lens=dcp_local_seq_lens,
)