[Performance][DSR1]: Fused RoPE+KVCache+q_concat for MLA (#40392)
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com> Signed-off-by: Rohan Potdar <66227218+Rohan138@users.noreply.github.com> Co-authored-by: ElizaWszola <ewszola@redhat.com>
This commit is contained in:
co-authored by
ElizaWszola
parent
8415bf2cdb
commit
a51376b3f0
@@ -23,6 +23,9 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
|
||||
DeepseekScalingRotaryEmbedding,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
RMS_ADD_OP = torch.ops._C.fused_add_rms_norm.default
|
||||
@@ -158,6 +161,87 @@ class MatcherRotaryEmbedding(MatcherCustomOp):
|
||||
return result
|
||||
|
||||
|
||||
class MatcherDeepseekScalingRotaryEmbedding(MatcherCustomOp):
|
||||
def __init__(
|
||||
self,
|
||||
is_neox: bool,
|
||||
head_size: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
use_flashinfer: bool = False,
|
||||
enabled: bool | None = None,
|
||||
) -> None:
|
||||
if enabled is None:
|
||||
enabled = DeepseekScalingRotaryEmbedding.enabled()
|
||||
|
||||
super().__init__(enabled)
|
||||
self.is_neox = is_neox
|
||||
self.head_size = head_size
|
||||
self.num_heads = num_heads
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.q_size = self.num_heads * self.head_size
|
||||
self.kv_size = self.num_kv_heads * self.head_size
|
||||
self.rotary_dim = head_size
|
||||
self.use_flashinfer = use_flashinfer
|
||||
|
||||
def inputs(self) -> list[torch.Tensor]:
|
||||
positions = self.empty_int64(5)
|
||||
query = self.empty(5, self.num_heads, self.head_size)
|
||||
key = self.empty(5, self.num_kv_heads, self.head_size)
|
||||
cos_sin_cache = self.empty(4096, self.rotary_dim)
|
||||
return [positions, query, key, cos_sin_cache]
|
||||
|
||||
def forward_custom(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if self.use_flashinfer:
|
||||
torch.ops.vllm.flashinfer_rotary_embedding(
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
self.head_size,
|
||||
cos_sin_cache,
|
||||
self.is_neox,
|
||||
)
|
||||
return query, key
|
||||
result: tuple[torch.Tensor, torch.Tensor | None] = (
|
||||
DeepseekScalingRotaryEmbedding.forward_static(
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
self.head_size,
|
||||
self.rotary_dim,
|
||||
cos_sin_cache,
|
||||
self.is_neox,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
result: tuple[torch.Tensor, torch.Tensor | None] = (
|
||||
DeepseekScalingRotaryEmbedding.forward_static(
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
self.head_size,
|
||||
self.rotary_dim,
|
||||
cos_sin_cache,
|
||||
self.is_neox,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class MatcherQuantFP8(MatcherCustomOp):
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.attention.attention import get_attention_context
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.utils.torch_utils import (
|
||||
_USE_LAYERNAME,
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
|
||||
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
|
||||
from .matcher_utils import MatcherDeepseekScalingRotaryEmbedding, MatcherRotaryEmbedding
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def fused_rope_unified_mla_kv_cache_update_impl(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
attn_metadata, _, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
|
||||
if layer_slot_mapping is not None:
|
||||
ops.concat_and_cache_mla_rope_fused(
|
||||
positions,
|
||||
q_pe,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cos_sin_cache,
|
||||
is_neox,
|
||||
layer_slot_mapping,
|
||||
kv_cache,
|
||||
kv_cache_dtype,
|
||||
kv_cache_scale,
|
||||
)
|
||||
return torch.empty(0, device=kv_c.device, dtype=kv_c.dtype)
|
||||
|
||||
|
||||
def fused_rope_unified_mla_kv_cache_update_fake(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(0, dtype=kv_c.dtype, device=kv_c.device)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_rope_unified_mla_kv_cache_update",
|
||||
op_func=fused_rope_unified_mla_kv_cache_update_impl,
|
||||
fake_impl=fused_rope_unified_mla_kv_cache_update_fake,
|
||||
mutates_args=["q_pe", "k_pe"],
|
||||
)
|
||||
|
||||
|
||||
class MLARoPEKVCacheCatPattern(VllmPatternReplacement):
|
||||
FUSED_OP = torch.ops.vllm.fused_rope_unified_mla_kv_cache_update.default
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer: MLAAttention,
|
||||
is_neox: bool,
|
||||
use_flashinfer: bool = False,
|
||||
use_deepseek_scaling: bool = False,
|
||||
) -> None:
|
||||
self.layer_name = layer.layer_name
|
||||
self.kv_cache_dtype = layer.kv_cache_dtype
|
||||
self.num_heads = layer.num_heads
|
||||
self.num_kv_heads = layer.num_kv_heads
|
||||
self.kv_lora_rank = layer.kv_lora_rank
|
||||
self.qk_rope_head_dim = layer.qk_rope_head_dim
|
||||
self.is_neox = is_neox
|
||||
self.use_flashinfer = use_flashinfer
|
||||
self._ln = _encode_layer_name(self.layer_name)
|
||||
|
||||
if use_deepseek_scaling:
|
||||
self.rope_matcher = MatcherDeepseekScalingRotaryEmbedding(
|
||||
is_neox=self.is_neox,
|
||||
head_size=self.qk_rope_head_dim,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
use_flashinfer=self.use_flashinfer,
|
||||
)
|
||||
else:
|
||||
self.rope_matcher = MatcherRotaryEmbedding( # type: ignore
|
||||
is_neox=self.is_neox,
|
||||
head_size=self.qk_rope_head_dim,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
use_flashinfer=self.use_flashinfer,
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
T = 5
|
||||
L = 4096
|
||||
q_pe = self.empty_bf16(T, self.num_heads, self.qk_rope_head_dim)
|
||||
k_pe = self.empty_bf16(T, self.qk_rope_head_dim)
|
||||
kv_c_normed = self.empty_bf16(T, self.kv_lora_rank)
|
||||
cos_sin_cache = self.empty_bf16(L, self.qk_rope_head_dim)
|
||||
positions = self.empty(T, dtype=torch.int64)
|
||||
k_scale = self.empty(0, dtype=torch.float32)
|
||||
inputs = [
|
||||
q_pe,
|
||||
k_pe,
|
||||
kv_c_normed,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
k_scale,
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(self._ln)
|
||||
return inputs
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
_ln = self._ln
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
k_pe_unsqueezed = k_pe.unsqueeze(1)
|
||||
q_pe, k_pe = self.rope_matcher(
|
||||
positions, q_pe, k_pe_unsqueezed, cos_sin_cache
|
||||
)
|
||||
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
|
||||
kv_c_normed, k_pe, layer_name, self.kv_cache_dtype, k_scale
|
||||
)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
k_pe_unsqueezed = k_pe.unsqueeze(1)
|
||||
q_pe, k_pe = self.rope_matcher(
|
||||
positions, q_pe, k_pe_unsqueezed, cos_sin_cache
|
||||
)
|
||||
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
|
||||
kv_c_normed, k_pe, _ln, self.kv_cache_dtype, k_scale
|
||||
)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
_ln = self._ln
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
at = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
positions=positions,
|
||||
q_pe=q_pe,
|
||||
k_pe=k_pe,
|
||||
kv_c=kv_c_normed,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
kv_cache_scale=k_scale,
|
||||
layer_name=layer_name,
|
||||
)
|
||||
dummy, q_pe, k_pe_squeezed = at
|
||||
k_pe = k_pe_squeezed.unsqueeze(1)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
at = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
positions=positions,
|
||||
q_pe=q_pe,
|
||||
k_pe=k_pe,
|
||||
kv_c=kv_c_normed,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
kv_cache_scale=k_scale,
|
||||
layer_name=_ln,
|
||||
)
|
||||
dummy, q_pe, k_pe_squeezed = at
|
||||
k_pe = k_pe_squeezed.unsqueeze(1)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class MLARoPEKVCacheCatFusionPass(VllmFusionPatternMatcherPass):
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "mla_rope_kv_cache_fusion_pass")
|
||||
|
||||
attn_layers = get_layers_from_vllm_config(config, MLAAttention)
|
||||
|
||||
for _, layer in attn_layers.items():
|
||||
for is_neox in [False, True]:
|
||||
for use_deepseek_scaling in [False, True]:
|
||||
if RotaryEmbedding.enabled():
|
||||
for use_flashinfer in [False, True]:
|
||||
self.register(
|
||||
MLARoPEKVCacheCatPattern(
|
||||
layer,
|
||||
is_neox,
|
||||
use_flashinfer,
|
||||
use_deepseek_scaling,
|
||||
)
|
||||
)
|
||||
else:
|
||||
self.register(
|
||||
MLARoPEKVCacheCatPattern(
|
||||
layer,
|
||||
is_neox,
|
||||
use_deepseek_scaling=use_deepseek_scaling,
|
||||
)
|
||||
)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
@@ -33,6 +33,7 @@ if current_platform.is_cuda_alike():
|
||||
from .fusion.act_quant_fusion import ActivationQuantFusionPass
|
||||
from .fusion.attn_quant_fusion import AttnQuantFusionPass
|
||||
from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass
|
||||
from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass
|
||||
from .fusion.qk_norm_rope_fusion import QKNormRoPEFusionPass
|
||||
from .fusion.rms_quant_fusion import RMSNormQuantFusionPass
|
||||
from .fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
|
||||
@@ -174,6 +175,9 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
self.passes += [ScatterSplitReplacementPass(config)]
|
||||
self.passes += [RopeKVCacheFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_rope_kvcache_cat_mla:
|
||||
self.passes += [MLARoPEKVCacheCatFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_attn_quant:
|
||||
self.passes += [AttnQuantFusionPass(config)]
|
||||
self.passes += [MLAAttnQuantFusionPass(config)]
|
||||
|
||||
@@ -181,6 +181,45 @@ class FixFunctionalizationPass(VllmInductorPass):
|
||||
2: "key",
|
||||
}
|
||||
self.defunctionalize(graph, node, mutated_args=mutated_args)
|
||||
elif (
|
||||
hasattr(torch.ops.vllm, "fused_rope_unified_mla_kv_cache_update")
|
||||
and at_target
|
||||
== torch.ops.vllm.fused_rope_unified_mla_kv_cache_update.default
|
||||
):
|
||||
# AOTAutograd functionalizes `q[..., nope_dim:] = rope_result` into
|
||||
# a sequence of aten ops on q: view+slice+copy+slice_scatter.
|
||||
# Since the fused MLA RoPE op mutates q_pe in-place, we can remove
|
||||
# the redundant copy and slice_scatter ops during defunctionalization.
|
||||
getitem_nodes = self.getitem_users(node)
|
||||
q_pe_out = getitem_nodes[1]
|
||||
|
||||
for user in list(q_pe_out.users):
|
||||
if is_func(user, torch.ops.aten.copy.default):
|
||||
copy_temp = user
|
||||
slice_temp = copy_temp.args[0]
|
||||
for user in list(copy_temp.users):
|
||||
if is_func(user, torch.ops.aten.slice_scatter.default):
|
||||
slice_scatter_temp = user
|
||||
view_temp = slice_scatter_temp.args[0]
|
||||
|
||||
view_orig = slice_temp.args[0]
|
||||
slice_scatter_temp.replace_all_uses_with(view_orig)
|
||||
self._remove(slice_scatter_temp)
|
||||
self._remove(copy_temp)
|
||||
self._remove(slice_temp)
|
||||
self._remove(view_temp)
|
||||
self._remove(q_pe_out)
|
||||
|
||||
# defunctionalize k_pe manually; self.replace_users_with_mutated_args
|
||||
# does not support only replacing specific kwargs
|
||||
k_pe_in = node.kwargs["k_pe"]
|
||||
k_pe_out = getitem_nodes[2]
|
||||
k_pe_out.replace_all_uses_with(k_pe_in)
|
||||
self._remove(k_pe_out)
|
||||
|
||||
self.insert_defunctionalized(graph, node)
|
||||
self._remove(node)
|
||||
|
||||
# only used for test_functionalization::TestFunctionWithMutatedArgsAndReturn
|
||||
elif (
|
||||
hasattr(torch.ops.vllm, "function_with_mutated_args_and_return")
|
||||
|
||||
Reference in New Issue
Block a user