diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index 950de76410..cfce3c8bcf 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -1662,6 +1662,12 @@ def _run_backend_correctness( kv_cache_per_block_size[block_size] = kv_cache # 4. Run vLLM backends and compare + rtol = 1e-2 + atol = { + "auto": 1e-2, + "fp8": 1.5e-1, + "fp8_e4m3": 1.5e-1, + }[kv_cache_dtype] failures = [] for backend_idx, backend_name in enumerate(backends_to_test): # Skip backends that don't support spec decode for spec decode tests @@ -1726,10 +1732,6 @@ def _run_backend_correctness( f"[{backend_name}] produced non-finite values" ) - # Check numerical similarity - rtol = 1e-2 - atol = 5e-1 - max_diff = torch.max(torch.abs(backend_output - expected_output)).item() max_rel_diff = torch.max( torch.abs(backend_output - expected_output) / torch.abs(expected_output) diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index fa8faf2a58..5085e26005 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -73,6 +73,7 @@ from vllm.v1.attention.backends.utils import ( get_num_attention_heads_from_layers, get_per_layer_parameters, infer_global_hyperparameters, + log2_lse_to_ln, split_decodes_and_prefills, ) from vllm.v1.attention.ops.dcp import ( @@ -370,7 +371,7 @@ class BatchDCPPrefillWrapper: get_dcp_group(), return_lse=True, ) - lse_context = lse_context.transpose(0, 1).contiguous() + lse_context = log2_lse_to_ln(lse_context.transpose(0, 1).contiguous()) output_query, lse_query = self._new_tokens.run( prefill_query, @@ -378,7 +379,7 @@ class BatchDCPPrefillWrapper: value, return_lse=True, ) - lse_query = lse_query.transpose(0, 1).contiguous() + lse_query = log2_lse_to_ln(lse_query.transpose(0, 1).contiguous()) merge_attn_states( out, diff --git a/vllm/v1/attention/backends/mla/prefill/flashinfer.py b/vllm/v1/attention/backends/mla/prefill/flashinfer.py index f2707aa970..936690ec56 100644 --- a/vllm/v1/attention/backends/mla/prefill/flashinfer.py +++ b/vllm/v1/attention/backends/mla/prefill/flashinfer.py @@ -15,6 +15,7 @@ from vllm.v1.attention.backends.utils import ( PerLayerParameters, get_per_layer_parameters, infer_global_hyperparameters, + log2_lse_to_ln, ) from vllm.v1.worker.workspace import current_workspace_manager @@ -219,7 +220,7 @@ class FlashInferPrefillBackend(MLAPrefillBackend): if isinstance(ret, tuple): # Convert from (q_len, num_heads) to (num_heads, q_len) - return ret[0], ret[1].transpose(0, 1) + return ret[0], log2_lse_to_ln(ret[1].transpose(0, 1)) return ret def run_prefill_context_chunk( @@ -239,4 +240,4 @@ class FlashInferPrefillBackend(MLAPrefillBackend): ) # Convert from (q_len, num_heads) to (num_heads, q_len) - return attn_out, lse.transpose(0, 1) + return attn_out, log2_lse_to_ln(lse.transpose(0, 1)) diff --git a/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py b/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py index 144a75aded..b8ae15b4e6 100644 --- a/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py +++ b/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py @@ -11,6 +11,7 @@ from vllm.v1.attention.backends.mla.prefill.base import ( MLADimensions, MLAPrefillBackend, ) +from vllm.v1.attention.backends.utils import log2_lse_to_ln from vllm.v1.worker.workspace import current_workspace_manager if TYPE_CHECKING: @@ -138,7 +139,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend): if isinstance(ret, tuple): # Convert from (q_len, num_heads) to (num_heads, q_len) - return ret[0], ret[1].transpose(0, 1) + return ret[0], log2_lse_to_ln(ret[1].transpose(0, 1)) return ret def run_prefill_context_chunk( @@ -182,4 +183,4 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend): ) # Convert from (q_len, num_heads) to (num_heads, q_len) - return attn_out, lse.transpose(0, 1) + return attn_out, log2_lse_to_ln(lse.transpose(0, 1)) diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index 28364b57ab..f3378806c8 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools +import math from collections.abc import Callable from dataclasses import dataclass, field, fields, make_dataclass from typing import ( @@ -45,6 +46,13 @@ _KV_CACHE_LAYOUT_OVERRIDE: KVCacheLayoutType | None = None PAD_SLOT_ID = -1 NULL_BLOCK_ID = 0 +_LN_2 = math.log(2.0) + + +def log2_lse_to_ln(lse: torch.Tensor) -> torch.Tensor: + """Convert a base-2 log-sum-exp tensor to natural-log units.""" + return lse * _LN_2 + def compute_mm_prefix_range_tensor( mm_prefix_range: dict[int, list[tuple[int, int]]] | None,