[Bugfix] Fix FlashInfer crash with kv_cache_dtype_skip_layers (#39002)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-04-10 18:50:47 +00:00
committed by GitHub
parent e281cb721c
commit e816a8811f
2 changed files with 15 additions and 11 deletions
+10 -3
View File
@@ -63,7 +63,11 @@ from vllm.v1.attention.backends.utils import (
from vllm.v1.attention.ops.common import cp_lse_ag_out_rs
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.kv_cache_interface import AttentionSpec, UniformTypeKVCacheSpecs
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVQuantMode,
UniformTypeKVCacheSpecs,
)
from vllm.v1.utils import CpuGpuBuffer
FLASHINFER_WORKSPACE_BUFFER_SIZE_BATCH_INVARIANT = 2048 * 1024 * 1024
@@ -600,12 +604,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self.head_dim = self.kv_cache_spec.head_size
self.page_size = self.kv_cache_spec.block_size
self.cache_dtype = self.cache_config.cache_dtype
if is_quantized_kv_cache(self.cache_dtype):
if self.kv_cache_spec.kv_quant_mode != KVQuantMode.NONE:
self.cache_dtype = self.cache_config.cache_dtype
# Cannot use self.kv_cache_spec.dtype here because kv_cache_spec
# storage dtype may not be the same as the op dtype (uint8 vs fp8_e4m3)
self.kv_cache_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer(
self.cache_dtype
)
else:
self.cache_dtype = "auto"
assert self.kv_cache_spec.dtype == self.model_config.dtype
self.kv_cache_dtype = self.kv_cache_spec.dtype