[Bugfix][Attention] Normalize FlashInfer prefill LSE before merging (#52796)
Signed-off-by: yimdev <5779256+yimdev@users.noreply.github.com> Co-authored-by: usberkeley <150880684+usberkeley@users.noreply.github.com>
This commit is contained in:
@@ -1662,6 +1662,12 @@ def _run_backend_correctness(
|
||||
kv_cache_per_block_size[block_size] = kv_cache
|
||||
|
||||
# 4. Run vLLM backends and compare
|
||||
rtol = 1e-2
|
||||
atol = {
|
||||
"auto": 1e-2,
|
||||
"fp8": 1.5e-1,
|
||||
"fp8_e4m3": 1.5e-1,
|
||||
}[kv_cache_dtype]
|
||||
failures = []
|
||||
for backend_idx, backend_name in enumerate(backends_to_test):
|
||||
# Skip backends that don't support spec decode for spec decode tests
|
||||
@@ -1726,10 +1732,6 @@ def _run_backend_correctness(
|
||||
f"[{backend_name}] produced non-finite values"
|
||||
)
|
||||
|
||||
# Check numerical similarity
|
||||
rtol = 1e-2
|
||||
atol = 5e-1
|
||||
|
||||
max_diff = torch.max(torch.abs(backend_output - expected_output)).item()
|
||||
max_rel_diff = torch.max(
|
||||
torch.abs(backend_output - expected_output) / torch.abs(expected_output)
|
||||
|
||||
@@ -73,6 +73,7 @@ from vllm.v1.attention.backends.utils import (
|
||||
get_num_attention_heads_from_layers,
|
||||
get_per_layer_parameters,
|
||||
infer_global_hyperparameters,
|
||||
log2_lse_to_ln,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.attention.ops.dcp import (
|
||||
@@ -370,7 +371,7 @@ class BatchDCPPrefillWrapper:
|
||||
get_dcp_group(),
|
||||
return_lse=True,
|
||||
)
|
||||
lse_context = lse_context.transpose(0, 1).contiguous()
|
||||
lse_context = log2_lse_to_ln(lse_context.transpose(0, 1).contiguous())
|
||||
|
||||
output_query, lse_query = self._new_tokens.run(
|
||||
prefill_query,
|
||||
@@ -378,7 +379,7 @@ class BatchDCPPrefillWrapper:
|
||||
value,
|
||||
return_lse=True,
|
||||
)
|
||||
lse_query = lse_query.transpose(0, 1).contiguous()
|
||||
lse_query = log2_lse_to_ln(lse_query.transpose(0, 1).contiguous())
|
||||
|
||||
merge_attn_states(
|
||||
out,
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.v1.attention.backends.utils import (
|
||||
PerLayerParameters,
|
||||
get_per_layer_parameters,
|
||||
infer_global_hyperparameters,
|
||||
log2_lse_to_ln,
|
||||
)
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
@@ -219,7 +220,7 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
|
||||
if isinstance(ret, tuple):
|
||||
# Convert from (q_len, num_heads) to (num_heads, q_len)
|
||||
return ret[0], ret[1].transpose(0, 1)
|
||||
return ret[0], log2_lse_to_ln(ret[1].transpose(0, 1))
|
||||
return ret
|
||||
|
||||
def run_prefill_context_chunk(
|
||||
@@ -239,4 +240,4 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
)
|
||||
|
||||
# Convert from (q_len, num_heads) to (num_heads, q_len)
|
||||
return attn_out, lse.transpose(0, 1)
|
||||
return attn_out, log2_lse_to_ln(lse.transpose(0, 1))
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.v1.attention.backends.mla.prefill.base import (
|
||||
MLADimensions,
|
||||
MLAPrefillBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import log2_lse_to_ln
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -138,7 +139,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
|
||||
if isinstance(ret, tuple):
|
||||
# Convert from (q_len, num_heads) to (num_heads, q_len)
|
||||
return ret[0], ret[1].transpose(0, 1)
|
||||
return ret[0], log2_lse_to_ln(ret[1].transpose(0, 1))
|
||||
return ret
|
||||
|
||||
def run_prefill_context_chunk(
|
||||
@@ -182,4 +183,4 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
)
|
||||
|
||||
# Convert from (q_len, num_heads) to (num_heads, q_len)
|
||||
return attn_out, lse.transpose(0, 1)
|
||||
return attn_out, log2_lse_to_ln(lse.transpose(0, 1))
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field, fields, make_dataclass
|
||||
from typing import (
|
||||
@@ -45,6 +46,13 @@ _KV_CACHE_LAYOUT_OVERRIDE: KVCacheLayoutType | None = None
|
||||
PAD_SLOT_ID = -1
|
||||
NULL_BLOCK_ID = 0
|
||||
|
||||
_LN_2 = math.log(2.0)
|
||||
|
||||
|
||||
def log2_lse_to_ln(lse: torch.Tensor) -> torch.Tensor:
|
||||
"""Convert a base-2 log-sum-exp tensor to natural-log units."""
|
||||
return lse * _LN_2
|
||||
|
||||
|
||||
def compute_mm_prefix_range_tensor(
|
||||
mm_prefix_range: dict[int, list[tuple[int, int]]] | None,
|
||||
|
||||
Reference in New Issue
Block a user