[DSv4] Adding TRTLLM gen attention kernel (#43827)
This commit is contained in:
@@ -155,13 +155,17 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
|
||||
raise ValueError(f"Invalid compress ratio: {compress_ratio}")
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# FlashMLA's UE8M0 paged layout needs 576B alignment; the FlashInfer
|
||||
# full-cache path shares state pages with contiguous KV pages, so
|
||||
# padding would break page matching.
|
||||
is_flashmla = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
return SlidingWindowMLASpec( # only has one vector instead of K + V
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.state_dim,
|
||||
dtype=self.dtype,
|
||||
sliding_window=self.sliding_window,
|
||||
alignment=576, # NOTE: FlashMLA requires 576B alignment
|
||||
alignment=576 if is_flashmla else None,
|
||||
)
|
||||
|
||||
def forward(self): ...
|
||||
@@ -333,26 +337,40 @@ class DeepseekCompressor(nn.Module):
|
||||
# - position used: (positions // compress_ratio) * compress_ratio
|
||||
cos_sin_cache = rotary_emb.cos_sin_cache
|
||||
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
|
||||
kv_cache = self._static_forward_context[self.k_cache_prefix].kv_cache
|
||||
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
|
||||
kv_cache = k_cache_layer.kv_cache
|
||||
|
||||
if current_platform.is_cuda():
|
||||
# NVIDIA GPUs.
|
||||
if self.head_dim == 512:
|
||||
from .nvidia.ops.sparse_attn_compress_cutedsl import (
|
||||
compress_norm_rope_store_cutedsl,
|
||||
)
|
||||
# FlashInfer V4 reads a contiguous bf16 / per-tensor fp8 cache row; the
|
||||
# legacy FlashMLA path uses the UE8M0 paged uint8 layout.
|
||||
store_full_kv = self.head_dim == 512 and kv_cache.dtype != torch.uint8
|
||||
store_full_fp8 = kv_cache.dtype == torch.float8_e4m3fn
|
||||
fp8_scale = (
|
||||
getattr(k_cache_layer, "_flashinfer_fp8_kv_scale", None)
|
||||
if store_full_fp8
|
||||
else None
|
||||
)
|
||||
|
||||
# Main compressor path.
|
||||
# Use a cutedsl kernel for better performance.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_cutedsl
|
||||
else:
|
||||
# Indexer path (head_dim == 128).
|
||||
# Use a triton kernel.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_triton
|
||||
# cutedsl (head=512) accepts the full-cache flags; triton (indexer/AMD)
|
||||
# does not, so the two callables have different signatures.
|
||||
compress_norm_rope_store_fn: Any
|
||||
if current_platform.is_cuda() and self.head_dim == 512:
|
||||
from .nvidia.ops.sparse_attn_compress_cutedsl import (
|
||||
compress_norm_rope_store_cutedsl,
|
||||
)
|
||||
|
||||
# head=512 on CUDA always uses cutedsl, for both the legacy UE8M0
|
||||
# layout and the FlashInfer full-cache layout. The full-cache flags
|
||||
# are consumed only here.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_cutedsl
|
||||
extra_kwargs: dict[str, Any] = dict(
|
||||
store_full_kv=store_full_kv,
|
||||
store_full_fp8=store_full_fp8,
|
||||
fp8_scale=fp8_scale,
|
||||
)
|
||||
else:
|
||||
# AMD GPUs.
|
||||
# Always use a triton kernel.
|
||||
# Indexer path (head_dim == 128) or AMD: triton, legacy UE8M0 only.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_triton
|
||||
extra_kwargs = {}
|
||||
|
||||
compress_norm_rope_store_fn(
|
||||
state_cache=state_cache,
|
||||
@@ -377,4 +395,5 @@ class DeepseekCompressor(nn.Module):
|
||||
quant_block=self._quant_block,
|
||||
token_stride=self._token_stride,
|
||||
scale_dim=self._scale_dim,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user