From 9dcf6bf344caa7793bae0b45a7896d3f8e03a01a Mon Sep 17 00:00:00 2001 From: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:33:53 -0700 Subject: [PATCH] [BugFix] Fix DP token padding in dflash attention metadata (#56181) Signed-off-by: Giancarlo Delfin --- .../test_eagle_draft_attn_metadata.py | 64 ++++++++++------ .../spec_decode/autoregressive/speculator.py | 14 ++-- .../gpu/spec_decode/dflash/speculator.py | 36 +-------- .../multi_module_mtp/speculator.py | 5 +- vllm/v1/worker/gpu/spec_decode/speculator.py | 75 +++++++++++-------- 5 files changed, 97 insertions(+), 97 deletions(-) diff --git a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py index 95e263f9de..0cc78f3371 100644 --- a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py +++ b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Unit tests for the EAGLE speculator's draft attention metadata builder. -These tests guard the regression where ``_build_draft_attn_metadata`` did +These tests guard the regression where ``_build_uniform_attn_metadata`` did not populate ``seq_lens_cpu_upper_bound`` on the per-step ``CommonAttentionMetadata``. Several downstream attention backends and helpers (``split_decodes_prefills_and_extends``, the MLA indexer, @@ -13,11 +13,14 @@ backends (e.g. ``ROCM_AITER_FA`` with eagle/eagle3 spec decode): AssertionError: assert common_attn_metadata.seq_lens_cpu_upper_bound is not None """ -from types import SimpleNamespace +from types import MethodType, SimpleNamespace from unittest.mock import patch +import numpy as np import torch +from vllm.config.compilation import CUDAGraphMode +from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.spec_decode import speculator as base_speculator from vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculator @@ -30,7 +33,7 @@ def _make_fake_speculator( draft_max_seq_len: int = 1024, ) -> SimpleNamespace: """Build a fake EagleSpeculator with just the attributes used by - ``_build_draft_attn_metadata``. We deliberately avoid constructing a + ``_build_uniform_attn_metadata``. We deliberately avoid constructing a real ``EagleSpeculator`` because that requires a full ``VllmConfig`` and a draft model. """ @@ -46,8 +49,8 @@ def _make_fake_speculator( cp_rank=0, cp_interleave=1, ) - return SimpleNamespace( - arange=torch.arange(max_num_reqs + 1, dtype=torch.int32, device="cpu"), + fake = SimpleNamespace( + arange_np=np.arange(max_num_reqs + 1, dtype=np.int32), draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool), block_tables=fake_block_tables, input_buffers=fake_input_buffers, @@ -56,21 +59,42 @@ def _make_fake_speculator( max_model_len=max_model_len, draft_max_seq_len=draft_max_seq_len, ) + # The uniform wrapper delegates through self; bind the real implementation. + fake._build_attn_metadata = MethodType(EagleSpeculator._build_attn_metadata, fake) + return fake -def _run_build(fake, *, num_reqs, num_reqs_padded, num_tokens_padded, base, step): +def _run_build( + fake, + *, + num_reqs, + num_reqs_padded, + base, + step, + num_query_per_req=1, + cg_mode=CUDAGraphMode.FULL, +): captured: dict[str, object] = {} def fake_build_attn_metadata(**kwargs): captured.update(kwargs) return {} + # Request padding only occurs under FULL graphs, where the captured batch + # is a uniform decode of num_reqs_padded * num_query_per_req tokens. + batch_desc = BatchExecutionDescriptor( + cg_mode=cg_mode, + num_tokens=num_reqs_padded * num_query_per_req, + num_reqs=num_reqs_padded, + uniform_token_count=num_query_per_req, + ) + with patch.object(base_speculator, "build_attn_metadata", fake_build_attn_metadata): - EagleSpeculator._build_draft_attn_metadata( + EagleSpeculator._build_uniform_attn_metadata( fake, # type: ignore[arg-type] + batch_desc=batch_desc, num_reqs=num_reqs, - num_reqs_padded=num_reqs_padded, - num_tokens_padded=num_tokens_padded, + num_query_per_req=num_query_per_req, seq_lens_cpu_upper_bound=base, step=step, ) @@ -85,9 +109,7 @@ def test_build_draft_attn_metadata_sets_seq_lens_cpu_upper_bound(): fake = _make_fake_speculator() base = torch.tensor([100, 200, 300, 0], dtype=torch.int32) - captured = _run_build( - fake, num_reqs=3, num_reqs_padded=4, num_tokens_padded=4, base=base, step=2 - ) + captured = _run_build(fake, num_reqs=3, num_reqs_padded=4, base=base, step=2) bound = captured["seq_lens_cpu_upper_bound"] assert isinstance(bound, torch.Tensor), ( @@ -102,20 +124,19 @@ def test_build_draft_attn_metadata_sets_seq_lens_cpu_upper_bound(): assert torch.equal(bound, torch.tensor([102, 202, 302, 0], dtype=torch.int32)) -def test_build_draft_attn_metadata_handles_zero_unpadded_reqs(): - """Edge case: when ``num_reqs == 0`` the upper-bound tensor must - still be a valid all-zero tensor of length ``num_reqs_padded``.""" +def test_build_draft_attn_metadata_zeroes_padded_upper_bound_tail(): + """The padded tail of the upper-bound tensor is zeroed, so it stays a + valid tensor of length ``num_reqs_padded`` regardless of padding.""" fake = _make_fake_speculator() base = torch.zeros(2, dtype=torch.int32) - captured = _run_build( - fake, num_reqs=0, num_reqs_padded=2, num_tokens_padded=2, base=base, step=1 - ) + captured = _run_build(fake, num_reqs=1, num_reqs_padded=2, base=base, step=1) bound = captured["seq_lens_cpu_upper_bound"] assert isinstance(bound, torch.Tensor) assert bound.shape == (2,) - assert torch.equal(bound, torch.zeros(2, dtype=torch.int32)) + # base is all zeros, so the real entry is 0 + step and the pad entry is 0. + assert torch.equal(bound, torch.tensor([1, 0], dtype=torch.int32)) def test_build_draft_attn_metadata_clamps_to_max_model_len(): @@ -124,9 +145,7 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len(): fake = _make_fake_speculator(max_model_len=1024) base = torch.tensor([1023, 500], dtype=torch.int32) - captured = _run_build( - fake, num_reqs=2, num_reqs_padded=2, num_tokens_padded=2, base=base, step=3 - ) + captured = _run_build(fake, num_reqs=2, num_reqs_padded=2, base=base, step=3) bound = captured["seq_lens_cpu_upper_bound"] # 1023 + 3 = 1026 -> clamped to 1024; 500 + 3 = 503 unaffected. @@ -154,7 +173,6 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(): fake, num_reqs=3, num_reqs_padded=4, - num_tokens_padded=4, base=torch.tensor([5, 9, 16]), step=0, ) diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index cab250ff4e..3d243b6ee5 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -532,11 +532,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): slot_mappings_by_layer = build_slot_mappings_by_layer( slot_mappings, self.kv_cache_config ) - attn_metadata = self._build_draft_attn_metadata( + attn_metadata = self._build_uniform_attn_metadata( num_reqs=num_reqs, - num_reqs_padded=batch_desc.num_reqs or num_reqs, - # One query per request; exclude DP-only model padding. - num_tokens_padded=batch_desc.num_reqs or num_reqs, + batch_desc=batch_desc, + num_query_per_req=1, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, step=step, ) @@ -581,11 +580,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): slot_mappings_by_layer = build_slot_mappings_by_layer( slot_mappings, self.kv_cache_config ) - attn_metadata = self._build_draft_attn_metadata( + attn_metadata = self._build_uniform_attn_metadata( num_reqs=num_reqs, - num_reqs_padded=batch_desc.num_reqs or num_reqs, - # One query per request; exclude DP-only model padding. - num_tokens_padded=batch_desc.num_reqs or num_reqs, + batch_desc=batch_desc, + num_query_per_req=1, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, step=1, ) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index 845781fab2..a5710829e0 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -1,10 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import copy -from collections.abc import Mapping from typing import Any -import numpy as np import torch import torch.nn as nn @@ -299,33 +297,6 @@ class DFlashSpeculator(DraftModelSpeculator): num_reqs, self.num_speculative_steps ) - def _build_draft_attn_metadata( - self, - num_reqs: int, - num_reqs_padded: int, - num_tokens_padded: int, - seq_lens_cpu_upper_bound: torch.Tensor, - step: int, - num_query_per_req: int | None = None, - causal: bool | Mapping[int, bool] = False, - query_start_loc_np: np.ndarray | None = None, - dcp_local_seq_lens: torch.Tensor | None = None, - ) -> dict[str, Any] | None: - if not self.draft_attn_layer_names: - return None - assert num_query_per_req is None # Omitted for DFlash, read from self instead - return super()._build_draft_attn_metadata( - num_reqs, - num_reqs_padded, - num_tokens_padded, - seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, - step=step, - num_query_per_req=self.num_query_per_req, - causal=causal, - query_start_loc_np=query_start_loc_np, - dcp_local_seq_lens=dcp_local_seq_lens, - ) - @torch.inference_mode() def propose( self, @@ -471,7 +442,6 @@ class DFlashSpeculator(DraftModelSpeculator): need_eager=is_profile, dp_sync=batch_sync, ) - num_reqs_padded = batch_desc.num_reqs or num_reqs num_tokens_padded = batch_desc.num_tokens num_tokens_across_dp = ( batch_sync.num_tokens_across_dp if batch_sync is not None else None @@ -479,10 +449,10 @@ class DFlashSpeculator(DraftModelSpeculator): # Rebuild the draft attention metadata even when replaying the FULL # graph so that any attention metadata builder state is updated. - draft_attn_metadata = self._build_draft_attn_metadata( + draft_attn_metadata = self._build_uniform_attn_metadata( num_reqs=num_reqs, - num_reqs_padded=num_reqs_padded, - num_tokens_padded=num_tokens_padded, + batch_desc=batch_desc, + num_query_per_req=self.num_query_per_req, seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound, step=self.num_query_per_req, causal=self._group_causal, diff --git a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py index 90572c03cc..1c6ae315cf 100644 --- a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py @@ -226,10 +226,9 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator): slot_mappings = build_slot_mappings_by_layer( slot_mappings_tensor, self.kv_cache_config ) - draft_attn_metadata = self._build_draft_attn_metadata( + draft_attn_metadata = self._build_attn_metadata( num_reqs=num_reqs, - num_reqs_padded=batch_desc.num_reqs or num_reqs, - num_tokens_padded=batch_desc.num_tokens, + batch_desc=batch_desc, query_start_loc_np=input_batch.query_start_loc_np, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, step=0, diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 2681677c76..4f20637d70 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -23,6 +23,7 @@ from vllm.v1.worker.gpu.attn_utils import ( ) from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens +from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.dp_utils import DPSyncState from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.interface import ModelState @@ -149,9 +150,7 @@ class DraftModelSpeculator(BaseSpeculator): dtype=torch.int64, device=device, ) - self.arange = torch.arange( - self.max_num_reqs + 1, dtype=torch.int32, device="cpu" - ) + self.arange_np = np.arange(self.max_num_reqs + 1, dtype=np.int32) self.draft_is_prefilling = torch.zeros(self.max_num_reqs, dtype=torch.bool) self.draft_logits: torch.Tensor | None = None @@ -254,42 +253,37 @@ class DraftModelSpeculator(BaseSpeculator): self.target_input_buffers = target_input_buffers self.target_attn_groups = target_attn_groups - def _build_draft_attn_metadata( + def _build_attn_metadata( self, num_reqs: int, - num_reqs_padded: int, - num_tokens_padded: int, + batch_desc: BatchExecutionDescriptor, + query_start_loc_np: np.ndarray, seq_lens_cpu_upper_bound: torch.Tensor, step: int, - num_query_per_req: int = 1, causal: bool | Mapping[int, bool] = True, - query_start_loc_np: np.ndarray | None = None, dcp_local_seq_lens: torch.Tensor | None = None, ) -> dict[str, Any] | None: - if query_start_loc_np is not None: - # Non-uniform query layout (e.g. multi-module MTP's mixed - # prefill/decode queries); num_query_per_req is ignored. - query_start_loc_cpu = torch.empty(num_reqs_padded + 1, dtype=torch.int32) - query_start_loc_cpu[: num_reqs + 1] = torch.from_numpy( - query_start_loc_np[: num_reqs + 1] - ) - query_start_loc_cpu[num_reqs:] = query_start_loc_cpu[num_reqs] - max_query_len = int( - (query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]).max() - ) - else: - # Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req. - # Clamp keeps the series non-decreasing past num_reqs, which some - # attention backends require. - query_start_loc_cpu = ( - torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs) - * num_query_per_req - ) - max_query_len = num_query_per_req + num_reqs_padded = batch_desc.num_reqs or num_reqs + # A FULL graph replays a captured shape whose padded requests each hold + # a full query width, so attention must see the padded token count. + # PIECEWISE/eager needs the actual token count, because batch_desc may + # carry graph or DP padding that no request owns, which would desync it + # from query_start_loc. + num_tokens = ( + batch_desc.num_tokens + if batch_desc.cg_mode == CUDAGraphMode.FULL + else int(query_start_loc_np[-1]) + ) + query_start_loc_cpu = torch.empty(num_reqs_padded + 1, dtype=torch.int32) + query_start_loc_cpu[: num_reqs + 1] = torch.from_numpy( + query_start_loc_np[: num_reqs + 1] + ) + query_start_loc_cpu[num_reqs:] = query_start_loc_cpu[num_reqs] + max_query_len = int((query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]).max()) block_tables = [ x[:num_reqs_padded] for x in self.block_tables.input_block_tables ] - slot_mappings = self.block_tables.slot_mappings[:, :num_tokens_padded] + slot_mappings = self.block_tables.slot_mappings[:, :num_tokens] draft_seq_lens_cpu_upper_bound = torch.zeros( num_reqs_padded, dtype=torch.int32, device="cpu" ) @@ -313,7 +307,7 @@ class DraftModelSpeculator(BaseSpeculator): attn_metadata = build_attn_metadata( attn_groups=self.attn_groups, num_reqs=num_reqs_padded, - num_tokens=num_tokens_padded, + num_tokens=num_tokens, query_start_loc_gpu=self.input_buffers.query_start_loc[ : num_reqs_padded + 1 ], @@ -434,3 +428,24 @@ class DraftModelSpeculator(BaseSpeculator): uniform_token_count=num_query_per_req, eager=False, ), num_batch_tokens + + def _build_uniform_attn_metadata( + self, + batch_desc: BatchExecutionDescriptor, + num_reqs: int, + num_query_per_req: int, + seq_lens_cpu_upper_bound: torch.Tensor, + step: int, + causal: bool | Mapping[int, bool] = True, + dcp_local_seq_lens: torch.Tensor | None = None, + ) -> dict[str, Any] | None: + query_start_loc_np = self.arange_np[: num_reqs + 1] * num_query_per_req + return self._build_attn_metadata( + num_reqs=num_reqs, + batch_desc=batch_desc, + query_start_loc_np=query_start_loc_np, + seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, + step=step, + causal=causal, + dcp_local_seq_lens=dcp_local_seq_lens, + )