[Attention] Add head_dim=512 support for FlashInfer trtllm attention backend (#38822)

This commit is contained in:
Duncan Moss
2026-05-22 20:27:35 -04:00
committed by GitHub
parent 6d30655b13
commit 552bbe6f4e
2 changed files with 13 additions and 12 deletions
+11 -10
View File
@@ -402,7 +402,7 @@ class FlashInferBackend(AttentionBackend):
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
# https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157
return [64, 128, 256]
return [64, 128, 256, 512]
@classmethod
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
@@ -1451,15 +1451,16 @@ class FlashInferImpl(AttentionImpl):
num_actual_tokens = attn_metadata.num_actual_tokens
# The FlashInfer api requires data to be in fp8_e4m3 or fp8_e5m2
# to process the cache when the kv_cache_dtype is fp8
if self.kv_sharing_target_layer_name is None and is_quantized_kv_cache(
self.kv_cache_dtype
):
torch_dtype = FlashInferBackend.get_dtype_for_flashinfer(
self.kv_cache_dtype
)
kv_cache = kv_cache.view(torch_dtype)
# FlashInfer treats uint8 KV cache as NVFP4. vLLM stores FP8 KV cache
# as uint8 bytes, so pass FP8 caches with their logical dtype.
if not self.is_kvcache_nvfp4 and kv_cache.dtype == torch.uint8:
fp8_view_dtype = None
if self.kv_cache_dtype in ("fp8", "fp8_e4m3", torch.float8_e4m3fn):
fp8_view_dtype = torch.float8_e4m3fn
elif self.kv_cache_dtype in ("fp8_e5m2", torch.float8_e5m2):
fp8_view_dtype = torch.float8_e5m2
if fp8_view_dtype is not None:
kv_cache = kv_cache.view(fp8_view_dtype)
# Inputs and outputs may be padded for CUDA graphs
query = query[:num_actual_tokens]