[Bugfix][Attention] Normalize FlashInfer prefill LSE before merging (#52796)

Signed-off-by: yimdev <5779256+yimdev@users.noreply.github.com>
Co-authored-by: usberkeley <150880684+usberkeley@users.noreply.github.com>
This commit is contained in:
yimdev
2026-08-21 11:31:14 -04:00
committed by GitHub
co-authored by usberkeley
parent 2740c817ff
commit 574e6a00ef
5 changed files with 23 additions and 10 deletions
+6 -4
View File
@@ -1662,6 +1662,12 @@ def _run_backend_correctness(
kv_cache_per_block_size[block_size] = kv_cache
# 4. Run vLLM backends and compare
rtol = 1e-2
atol = {
"auto": 1e-2,
"fp8": 1.5e-1,
"fp8_e4m3": 1.5e-1,
}[kv_cache_dtype]
failures = []
for backend_idx, backend_name in enumerate(backends_to_test):
# Skip backends that don't support spec decode for spec decode tests
@@ -1726,10 +1732,6 @@ def _run_backend_correctness(
f"[{backend_name}] produced non-finite values"
)
# Check numerical similarity
rtol = 1e-2
atol = 5e-1
max_diff = torch.max(torch.abs(backend_output - expected_output)).item()
max_rel_diff = torch.max(
torch.abs(backend_output - expected_output) / torch.abs(expected_output)
+3 -2
View File
@@ -73,6 +73,7 @@ from vllm.v1.attention.backends.utils import (
get_num_attention_heads_from_layers,
get_per_layer_parameters,
infer_global_hyperparameters,
log2_lse_to_ln,
split_decodes_and_prefills,
)
from vllm.v1.attention.ops.dcp import (
@@ -370,7 +371,7 @@ class BatchDCPPrefillWrapper:
get_dcp_group(),
return_lse=True,
)
lse_context = lse_context.transpose(0, 1).contiguous()
lse_context = log2_lse_to_ln(lse_context.transpose(0, 1).contiguous())
output_query, lse_query = self._new_tokens.run(
prefill_query,
@@ -378,7 +379,7 @@ class BatchDCPPrefillWrapper:
value,
return_lse=True,
)
lse_query = lse_query.transpose(0, 1).contiguous()
lse_query = log2_lse_to_ln(lse_query.transpose(0, 1).contiguous())
merge_attn_states(
out,
@@ -15,6 +15,7 @@ from vllm.v1.attention.backends.utils import (
PerLayerParameters,
get_per_layer_parameters,
infer_global_hyperparameters,
log2_lse_to_ln,
)
from vllm.v1.worker.workspace import current_workspace_manager
@@ -219,7 +220,7 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
if isinstance(ret, tuple):
# Convert from (q_len, num_heads) to (num_heads, q_len)
return ret[0], ret[1].transpose(0, 1)
return ret[0], log2_lse_to_ln(ret[1].transpose(0, 1))
return ret
def run_prefill_context_chunk(
@@ -239,4 +240,4 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
)
# Convert from (q_len, num_heads) to (num_heads, q_len)
return attn_out, lse.transpose(0, 1)
return attn_out, log2_lse_to_ln(lse.transpose(0, 1))
@@ -11,6 +11,7 @@ from vllm.v1.attention.backends.mla.prefill.base import (
MLADimensions,
MLAPrefillBackend,
)
from vllm.v1.attention.backends.utils import log2_lse_to_ln
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
@@ -138,7 +139,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
if isinstance(ret, tuple):
# Convert from (q_len, num_heads) to (num_heads, q_len)
return ret[0], ret[1].transpose(0, 1)
return ret[0], log2_lse_to_ln(ret[1].transpose(0, 1))
return ret
def run_prefill_context_chunk(
@@ -182,4 +183,4 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
)
# Convert from (q_len, num_heads) to (num_heads, q_len)
return attn_out, lse.transpose(0, 1)
return attn_out, log2_lse_to_ln(lse.transpose(0, 1))
+8
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import math
from collections.abc import Callable
from dataclasses import dataclass, field, fields, make_dataclass
from typing import (
@@ -45,6 +46,13 @@ _KV_CACHE_LAYOUT_OVERRIDE: KVCacheLayoutType | None = None
PAD_SLOT_ID = -1
NULL_BLOCK_ID = 0
_LN_2 = math.log(2.0)
def log2_lse_to_ln(lse: torch.Tensor) -> torch.Tensor:
"""Convert a base-2 log-sum-exp tensor to natural-log units."""
return lse * _LN_2
def compute_mm_prefix_range_tensor(
mm_prefix_range: dict[int, list[tuple[int, int]]] | None,