[BugFix] Fix DP token padding in dflash attention metadata (#56181)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the EAGLE speculator's draft attention metadata builder.
|
||||
|
||||
These tests guard the regression where ``_build_draft_attn_metadata`` did
|
||||
These tests guard the regression where ``_build_uniform_attn_metadata`` did
|
||||
not populate ``seq_lens_cpu_upper_bound`` on the per-step
|
||||
``CommonAttentionMetadata``. Several downstream attention backends and
|
||||
helpers (``split_decodes_prefills_and_extends``, the MLA indexer,
|
||||
@@ -13,11 +13,14 @@ backends (e.g. ``ROCM_AITER_FA`` with eagle/eagle3 spec decode):
|
||||
AssertionError: assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.spec_decode import speculator as base_speculator
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculator
|
||||
|
||||
@@ -30,7 +33,7 @@ def _make_fake_speculator(
|
||||
draft_max_seq_len: int = 1024,
|
||||
) -> SimpleNamespace:
|
||||
"""Build a fake EagleSpeculator with just the attributes used by
|
||||
``_build_draft_attn_metadata``. We deliberately avoid constructing a
|
||||
``_build_uniform_attn_metadata``. We deliberately avoid constructing a
|
||||
real ``EagleSpeculator`` because that requires a full ``VllmConfig``
|
||||
and a draft model.
|
||||
"""
|
||||
@@ -46,8 +49,8 @@ def _make_fake_speculator(
|
||||
cp_rank=0,
|
||||
cp_interleave=1,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
arange=torch.arange(max_num_reqs + 1, dtype=torch.int32, device="cpu"),
|
||||
fake = SimpleNamespace(
|
||||
arange_np=np.arange(max_num_reqs + 1, dtype=np.int32),
|
||||
draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool),
|
||||
block_tables=fake_block_tables,
|
||||
input_buffers=fake_input_buffers,
|
||||
@@ -56,21 +59,42 @@ def _make_fake_speculator(
|
||||
max_model_len=max_model_len,
|
||||
draft_max_seq_len=draft_max_seq_len,
|
||||
)
|
||||
# The uniform wrapper delegates through self; bind the real implementation.
|
||||
fake._build_attn_metadata = MethodType(EagleSpeculator._build_attn_metadata, fake)
|
||||
return fake
|
||||
|
||||
|
||||
def _run_build(fake, *, num_reqs, num_reqs_padded, num_tokens_padded, base, step):
|
||||
def _run_build(
|
||||
fake,
|
||||
*,
|
||||
num_reqs,
|
||||
num_reqs_padded,
|
||||
base,
|
||||
step,
|
||||
num_query_per_req=1,
|
||||
cg_mode=CUDAGraphMode.FULL,
|
||||
):
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_build_attn_metadata(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {}
|
||||
|
||||
# Request padding only occurs under FULL graphs, where the captured batch
|
||||
# is a uniform decode of num_reqs_padded * num_query_per_req tokens.
|
||||
batch_desc = BatchExecutionDescriptor(
|
||||
cg_mode=cg_mode,
|
||||
num_tokens=num_reqs_padded * num_query_per_req,
|
||||
num_reqs=num_reqs_padded,
|
||||
uniform_token_count=num_query_per_req,
|
||||
)
|
||||
|
||||
with patch.object(base_speculator, "build_attn_metadata", fake_build_attn_metadata):
|
||||
EagleSpeculator._build_draft_attn_metadata(
|
||||
EagleSpeculator._build_uniform_attn_metadata(
|
||||
fake, # type: ignore[arg-type]
|
||||
batch_desc=batch_desc,
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
num_tokens_padded=num_tokens_padded,
|
||||
num_query_per_req=num_query_per_req,
|
||||
seq_lens_cpu_upper_bound=base,
|
||||
step=step,
|
||||
)
|
||||
@@ -85,9 +109,7 @@ def test_build_draft_attn_metadata_sets_seq_lens_cpu_upper_bound():
|
||||
fake = _make_fake_speculator()
|
||||
base = torch.tensor([100, 200, 300, 0], dtype=torch.int32)
|
||||
|
||||
captured = _run_build(
|
||||
fake, num_reqs=3, num_reqs_padded=4, num_tokens_padded=4, base=base, step=2
|
||||
)
|
||||
captured = _run_build(fake, num_reqs=3, num_reqs_padded=4, base=base, step=2)
|
||||
|
||||
bound = captured["seq_lens_cpu_upper_bound"]
|
||||
assert isinstance(bound, torch.Tensor), (
|
||||
@@ -102,20 +124,19 @@ def test_build_draft_attn_metadata_sets_seq_lens_cpu_upper_bound():
|
||||
assert torch.equal(bound, torch.tensor([102, 202, 302, 0], dtype=torch.int32))
|
||||
|
||||
|
||||
def test_build_draft_attn_metadata_handles_zero_unpadded_reqs():
|
||||
"""Edge case: when ``num_reqs == 0`` the upper-bound tensor must
|
||||
still be a valid all-zero tensor of length ``num_reqs_padded``."""
|
||||
def test_build_draft_attn_metadata_zeroes_padded_upper_bound_tail():
|
||||
"""The padded tail of the upper-bound tensor is zeroed, so it stays a
|
||||
valid tensor of length ``num_reqs_padded`` regardless of padding."""
|
||||
fake = _make_fake_speculator()
|
||||
base = torch.zeros(2, dtype=torch.int32)
|
||||
|
||||
captured = _run_build(
|
||||
fake, num_reqs=0, num_reqs_padded=2, num_tokens_padded=2, base=base, step=1
|
||||
)
|
||||
captured = _run_build(fake, num_reqs=1, num_reqs_padded=2, base=base, step=1)
|
||||
|
||||
bound = captured["seq_lens_cpu_upper_bound"]
|
||||
assert isinstance(bound, torch.Tensor)
|
||||
assert bound.shape == (2,)
|
||||
assert torch.equal(bound, torch.zeros(2, dtype=torch.int32))
|
||||
# base is all zeros, so the real entry is 0 + step and the pad entry is 0.
|
||||
assert torch.equal(bound, torch.tensor([1, 0], dtype=torch.int32))
|
||||
|
||||
|
||||
def test_build_draft_attn_metadata_clamps_to_max_model_len():
|
||||
@@ -124,9 +145,7 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len():
|
||||
fake = _make_fake_speculator(max_model_len=1024)
|
||||
base = torch.tensor([1023, 500], dtype=torch.int32)
|
||||
|
||||
captured = _run_build(
|
||||
fake, num_reqs=2, num_reqs_padded=2, num_tokens_padded=2, base=base, step=3
|
||||
)
|
||||
captured = _run_build(fake, num_reqs=2, num_reqs_padded=2, base=base, step=3)
|
||||
|
||||
bound = captured["seq_lens_cpu_upper_bound"]
|
||||
# 1023 + 3 = 1026 -> clamped to 1024; 500 + 3 = 503 unaffected.
|
||||
@@ -154,7 +173,6 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
|
||||
fake,
|
||||
num_reqs=3,
|
||||
num_reqs_padded=4,
|
||||
num_tokens_padded=4,
|
||||
base=torch.tensor([5, 9, 16]),
|
||||
step=0,
|
||||
)
|
||||
|
||||
@@ -532,11 +532,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, self.kv_cache_config
|
||||
)
|
||||
attn_metadata = self._build_draft_attn_metadata(
|
||||
attn_metadata = self._build_uniform_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=batch_desc.num_reqs or num_reqs,
|
||||
# One query per request; exclude DP-only model padding.
|
||||
num_tokens_padded=batch_desc.num_reqs or num_reqs,
|
||||
batch_desc=batch_desc,
|
||||
num_query_per_req=1,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
step=step,
|
||||
)
|
||||
@@ -581,11 +580,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, self.kv_cache_config
|
||||
)
|
||||
attn_metadata = self._build_draft_attn_metadata(
|
||||
attn_metadata = self._build_uniform_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=batch_desc.num_reqs or num_reqs,
|
||||
# One query per request; exclude DP-only model padding.
|
||||
num_tokens_padded=batch_desc.num_reqs or num_reqs,
|
||||
batch_desc=batch_desc,
|
||||
num_query_per_req=1,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
step=1,
|
||||
)
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
@@ -299,33 +297,6 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_reqs, self.num_speculative_steps
|
||||
)
|
||||
|
||||
def _build_draft_attn_metadata(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_reqs_padded: int,
|
||||
num_tokens_padded: int,
|
||||
seq_lens_cpu_upper_bound: torch.Tensor,
|
||||
step: int,
|
||||
num_query_per_req: int | None = None,
|
||||
causal: bool | Mapping[int, bool] = False,
|
||||
query_start_loc_np: np.ndarray | None = None,
|
||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self.draft_attn_layer_names:
|
||||
return None
|
||||
assert num_query_per_req is None # Omitted for DFlash, read from self instead
|
||||
return super()._build_draft_attn_metadata(
|
||||
num_reqs,
|
||||
num_reqs_padded,
|
||||
num_tokens_padded,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
step=step,
|
||||
num_query_per_req=self.num_query_per_req,
|
||||
causal=causal,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def propose(
|
||||
self,
|
||||
@@ -471,7 +442,6 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
need_eager=is_profile,
|
||||
dp_sync=batch_sync,
|
||||
)
|
||||
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
||||
num_tokens_padded = batch_desc.num_tokens
|
||||
num_tokens_across_dp = (
|
||||
batch_sync.num_tokens_across_dp if batch_sync is not None else None
|
||||
@@ -479,10 +449,10 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
|
||||
# Rebuild the draft attention metadata even when replaying the FULL
|
||||
# graph so that any attention metadata builder state is updated.
|
||||
draft_attn_metadata = self._build_draft_attn_metadata(
|
||||
draft_attn_metadata = self._build_uniform_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
num_tokens_padded=num_tokens_padded,
|
||||
batch_desc=batch_desc,
|
||||
num_query_per_req=self.num_query_per_req,
|
||||
seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound,
|
||||
step=self.num_query_per_req,
|
||||
causal=self._group_causal,
|
||||
|
||||
@@ -226,10 +226,9 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
slot_mappings = build_slot_mappings_by_layer(
|
||||
slot_mappings_tensor, self.kv_cache_config
|
||||
)
|
||||
draft_attn_metadata = self._build_draft_attn_metadata(
|
||||
draft_attn_metadata = self._build_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=batch_desc.num_reqs or num_reqs,
|
||||
num_tokens_padded=batch_desc.num_tokens,
|
||||
batch_desc=batch_desc,
|
||||
query_start_loc_np=input_batch.query_start_loc_np,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
step=0,
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
@@ -149,9 +150,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
self.arange = torch.arange(
|
||||
self.max_num_reqs + 1, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
self.arange_np = np.arange(self.max_num_reqs + 1, dtype=np.int32)
|
||||
self.draft_is_prefilling = torch.zeros(self.max_num_reqs, dtype=torch.bool)
|
||||
|
||||
self.draft_logits: torch.Tensor | None = None
|
||||
@@ -254,42 +253,37 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.target_input_buffers = target_input_buffers
|
||||
self.target_attn_groups = target_attn_groups
|
||||
|
||||
def _build_draft_attn_metadata(
|
||||
def _build_attn_metadata(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_reqs_padded: int,
|
||||
num_tokens_padded: int,
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
query_start_loc_np: np.ndarray,
|
||||
seq_lens_cpu_upper_bound: torch.Tensor,
|
||||
step: int,
|
||||
num_query_per_req: int = 1,
|
||||
causal: bool | Mapping[int, bool] = True,
|
||||
query_start_loc_np: np.ndarray | None = None,
|
||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if query_start_loc_np is not None:
|
||||
# Non-uniform query layout (e.g. multi-module MTP's mixed
|
||||
# prefill/decode queries); num_query_per_req is ignored.
|
||||
query_start_loc_cpu = torch.empty(num_reqs_padded + 1, dtype=torch.int32)
|
||||
query_start_loc_cpu[: num_reqs + 1] = torch.from_numpy(
|
||||
query_start_loc_np[: num_reqs + 1]
|
||||
)
|
||||
query_start_loc_cpu[num_reqs:] = query_start_loc_cpu[num_reqs]
|
||||
max_query_len = int(
|
||||
(query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]).max()
|
||||
)
|
||||
else:
|
||||
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
|
||||
# Clamp keeps the series non-decreasing past num_reqs, which some
|
||||
# attention backends require.
|
||||
query_start_loc_cpu = (
|
||||
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
|
||||
* num_query_per_req
|
||||
)
|
||||
max_query_len = num_query_per_req
|
||||
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
||||
# A FULL graph replays a captured shape whose padded requests each hold
|
||||
# a full query width, so attention must see the padded token count.
|
||||
# PIECEWISE/eager needs the actual token count, because batch_desc may
|
||||
# carry graph or DP padding that no request owns, which would desync it
|
||||
# from query_start_loc.
|
||||
num_tokens = (
|
||||
batch_desc.num_tokens
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL
|
||||
else int(query_start_loc_np[-1])
|
||||
)
|
||||
query_start_loc_cpu = torch.empty(num_reqs_padded + 1, dtype=torch.int32)
|
||||
query_start_loc_cpu[: num_reqs + 1] = torch.from_numpy(
|
||||
query_start_loc_np[: num_reqs + 1]
|
||||
)
|
||||
query_start_loc_cpu[num_reqs:] = query_start_loc_cpu[num_reqs]
|
||||
max_query_len = int((query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]).max())
|
||||
block_tables = [
|
||||
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
|
||||
]
|
||||
slot_mappings = self.block_tables.slot_mappings[:, :num_tokens_padded]
|
||||
slot_mappings = self.block_tables.slot_mappings[:, :num_tokens]
|
||||
draft_seq_lens_cpu_upper_bound = torch.zeros(
|
||||
num_reqs_padded, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
@@ -313,7 +307,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=self.attn_groups,
|
||||
num_reqs=num_reqs_padded,
|
||||
num_tokens=num_tokens_padded,
|
||||
num_tokens=num_tokens,
|
||||
query_start_loc_gpu=self.input_buffers.query_start_loc[
|
||||
: num_reqs_padded + 1
|
||||
],
|
||||
@@ -434,3 +428,24 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
uniform_token_count=num_query_per_req,
|
||||
eager=False,
|
||||
), num_batch_tokens
|
||||
|
||||
def _build_uniform_attn_metadata(
|
||||
self,
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
num_reqs: int,
|
||||
num_query_per_req: int,
|
||||
seq_lens_cpu_upper_bound: torch.Tensor,
|
||||
step: int,
|
||||
causal: bool | Mapping[int, bool] = True,
|
||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
query_start_loc_np = self.arange_np[: num_reqs + 1] * num_query_per_req
|
||||
return self._build_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
batch_desc=batch_desc,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
step=step,
|
||||
causal=causal,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user