[Performance][DSR1]: Fused RoPE+KVCache+q_concat for MLA (#40392)
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com> Signed-off-by: Rohan Potdar <66227218+Rohan138@users.noreply.github.com> Co-authored-by: ElizaWszola <ewszola@redhat.com>
This commit is contained in:
co-authored by
ElizaWszola
parent
8415bf2cdb
commit
a51376b3f0
@@ -345,6 +345,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
attn_backend: type[AttentionBackend] | None = None,
|
||||
use_sparse: bool = False,
|
||||
indexer: object | None = None,
|
||||
**extra_impl_args,
|
||||
@@ -374,14 +375,21 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.quant_config = quant_config
|
||||
|
||||
dtype = torch.get_default_dtype()
|
||||
self.attn_backend = get_attn_backend(
|
||||
self.head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
use_mla=True,
|
||||
use_sparse=use_sparse,
|
||||
num_heads=self.num_heads,
|
||||
)
|
||||
if attn_backend is not None:
|
||||
assert attn_backend.is_mla(), (
|
||||
f"MLAAttention: attn_backend must be an MLA backend, "
|
||||
f"got {attn_backend.get_name()} instead"
|
||||
)
|
||||
self.attn_backend = attn_backend
|
||||
else:
|
||||
self.attn_backend = get_attn_backend(
|
||||
self.head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
use_mla=True,
|
||||
use_sparse=use_sparse,
|
||||
num_heads=self.num_heads,
|
||||
)
|
||||
|
||||
# FlashMLA Sparse Attention fp8 backend uses "fp8_ds_mla" kv-cache format
|
||||
# Automatically convert fp8 kv-cache format to "fp8_ds_mla"
|
||||
@@ -1008,23 +1016,9 @@ def unified_mla_kv_cache_update(
|
||||
the data dependency between them to ensure torch.compile preserves ordering.
|
||||
"""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context = get_forward_context()
|
||||
attn_layer = forward_context.no_compile_layers[layer_name]
|
||||
kv_cache = attn_layer.kv_cache
|
||||
|
||||
# This needs to run even when we don't have metadata yet, so that the op
|
||||
# is correctly captured.
|
||||
if kv_cache.numel() == 0:
|
||||
# Can't update an empty KV cache.
|
||||
return torch.empty(0, device=kv_c_normed.device, dtype=kv_c_normed.dtype)
|
||||
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
assert isinstance(slot_mapping, dict), (
|
||||
f"Expected slot_mapping to be a dict, got {type(slot_mapping)}. "
|
||||
)
|
||||
layer_slot_mapping = slot_mapping.get(layer_name)
|
||||
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
|
||||
if layer_slot_mapping is not None:
|
||||
attn_layer.impl.do_kv_cache_update(
|
||||
attn_layer.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_cache,
|
||||
@@ -1113,6 +1107,7 @@ direct_register_custom_op(
|
||||
mutates_args=["output", "output_block_scale"],
|
||||
fake_impl=unified_mla_attention_with_output_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
tags=(torch.Tag.flexible_layout,),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -127,29 +127,52 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""PyTorch-native implementation equivalent to forward()."""
|
||||
assert key is not None
|
||||
cos_sin_cache = self._match_cos_sin_cache_dtype(query)
|
||||
query_rot = query[..., : self.rotary_dim]
|
||||
key_rot = key[..., : self.rotary_dim]
|
||||
if self.rotary_dim < self.head_size:
|
||||
query_pass = query[..., self.rotary_dim :]
|
||||
key_pass = key[..., self.rotary_dim :]
|
||||
return self.forward_static(
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
self.head_size,
|
||||
self.rotary_dim,
|
||||
self.cos_sin_cache,
|
||||
self.is_neox_style,
|
||||
offsets,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def forward_static(
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox_style: bool,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""A static implementation of forward()."""
|
||||
assert key is not None
|
||||
query_rot = query[..., :rotary_dim]
|
||||
key_rot = key[..., :rotary_dim]
|
||||
if rotary_dim < head_size:
|
||||
query_pass = query[..., rotary_dim:]
|
||||
key_pass = key[..., rotary_dim:]
|
||||
|
||||
cos_sin = cos_sin_cache[
|
||||
torch.add(positions, offsets) if offsets is not None else positions
|
||||
]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
if self.is_neox_style:
|
||||
if is_neox_style:
|
||||
cos = torch.cat((cos, cos), dim=-1).unsqueeze(-2)
|
||||
sin = torch.cat((sin, sin), dim=-1).unsqueeze(-2)
|
||||
else:
|
||||
cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
|
||||
sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
|
||||
|
||||
rotate_fn = rotate_neox if self.is_neox_style else rotate_gptj
|
||||
rotate_fn = rotate_neox if is_neox_style else rotate_gptj
|
||||
query_rot = query_rot * cos + rotate_fn(query_rot) * sin
|
||||
key_rot = key_rot * cos + rotate_fn(key_rot) * sin
|
||||
|
||||
if self.rotary_dim < self.head_size:
|
||||
if rotary_dim < head_size:
|
||||
query = torch.cat((query_rot, query_pass), dim=-1)
|
||||
key = torch.cat((key_rot, key_pass), dim=-1)
|
||||
else:
|
||||
|
||||
@@ -195,10 +195,8 @@ class DualChunkRotaryEmbedding(CustomOp):
|
||||
def _apply_rotary_embedding(self, cos_sin, hidden_rot, hidden_pass):
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
if self.is_neox_style:
|
||||
# NOTE(woosuk): Here we assume that the positions tensor has the
|
||||
# shape [batch_size, seq_len].
|
||||
cos = cos.repeat(1, 1, 2).unsqueeze(-2)
|
||||
sin = sin.repeat(1, 1, 2).unsqueeze(-2)
|
||||
cos = torch.cat((cos, cos), dim=-1).unsqueeze(-2)
|
||||
sin = torch.cat((sin, sin), dim=-1).unsqueeze(-2)
|
||||
else:
|
||||
cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
|
||||
sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
|
||||
|
||||
Reference in New Issue
Block a user