[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
@@ -1,629 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from torch._higher_order_ops.auto_functionalize import auto_functionalized
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic64Sym,
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
from .matcher_utils import MatcherQuantFP8
from .rms_quant_fusion import QUANT_OPS
logger = init_logger(__name__)
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
MLA_ATTN_OP = torch.ops.vllm.unified_mla_attention_with_output.default
class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
"""Fusion for MLA Attention+Fp8StaticQuant.
Matches the pattern: MLA attention -> static FP8 quant, and replaces
it with MLA attention(output_scale=scale, output=fp8_buffer).
"""
def __init__(self, layer: MLAAttention, dtype: torch.dtype) -> None:
self._layer_name = layer.layer_name
self._num_heads = layer.num_heads
self._v_head_dim = layer.v_head_dim
self._kv_lora_rank = layer.kv_lora_rank
self._qk_rope_head_dim = layer.qk_rope_head_dim
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
self._output_dim = layer.num_heads * layer.v_head_dim
self._dtype = dtype
self._quant_matcher = MatcherQuantFP8(kFp8StaticTensorSym)
@property
def pattern(self) -> Callable[..., torch.Tensor]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _pattern_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
scale,
kv_cache_dummy_dep,
layer_name,
):
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
# MLA output is already 2D (T, N*V), no reshape needed
return self._quant_matcher(at1[1], scale)[0]
return _pattern_with_ln
def _pattern(q, kv_c_normed, k_pe, output_attn, scale, kv_cache_dummy_dep):
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
# MLA output is already 2D (T, N*V), no reshape needed
return self._quant_matcher(at1[1], scale)[0]
return _pattern
@property
def replacement(self) -> Callable[..., torch.Tensor]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _replacement_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
scale,
kv_cache_dummy_dep,
layer_name,
):
# MLA output in quant_dtype
output_attn = torch.empty(
[q.shape[0], self._output_dim],
dtype=FP8_DTYPE,
device=q.device,
)
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=scale,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
return at1[1]
return _replacement_with_ln
def _replacement(q, kv_c_normed, k_pe, output_attn, scale, kv_cache_dummy_dep):
# MLA output in quant_dtype
output_attn = torch.empty(
[q.shape[0], self._output_dim],
dtype=FP8_DTYPE,
device=q.device,
)
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=scale,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
return at1[1]
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
inputs: list = [
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
self.empty(5, self._output_dim, dtype=self._dtype),
self.empty_fp32(1, 1),
self.empty(0, dtype=self._dtype),
]
if _USE_LAYERNAME:
inputs.append(_encode_layer_name(self._layer_name))
return inputs
class MLAAttnNvfp4QuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
"""Fusion for MLA Attention+Nvfp4Quant.
Matches the pattern: MLA attention -> NVFP4 quant, and replaces
it with MLA attention(output_scale=scale, output_block_scale=block_scale,
output=fp4_buffer).
"""
def __init__(self, layer: MLAAttention, dtype: torch.dtype) -> None:
self._layer_name = layer.layer_name
self._num_heads = layer.num_heads
self._v_head_dim = layer.v_head_dim
self._kv_lora_rank = layer.kv_lora_rank
self._qk_rope_head_dim = layer.qk_rope_head_dim
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
self._output_dim = layer.num_heads * layer.v_head_dim
self._dtype = dtype
self._QUANT_OP = QUANT_OPS[kNvfp4Dynamic]
@property
def pattern(
self,
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _pattern_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
output_quant,
output_scale,
input_scale,
kv_cache_dummy_dep,
layer_name,
):
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
at2 = auto_functionalized(
self._QUANT_OP,
input=at1[1],
input_scale=input_scale,
is_sf_swizzled_layout=True,
output=output_quant,
output_scale=output_scale,
)
output_scale_view = torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
return at2[1], output_scale_view
return _pattern_with_ln
def _pattern(
q,
kv_c_normed,
k_pe,
output_attn,
output_quant,
output_scale,
input_scale,
kv_cache_dummy_dep,
):
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
at2 = auto_functionalized(
self._QUANT_OP,
input=at1[1],
input_scale=input_scale,
is_sf_swizzled_layout=True,
output=output_quant,
output_scale=output_scale,
)
output_scale_view = torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
return at2[1], output_scale_view
return _pattern
@property
def replacement(
self,
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _replacement_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
_output_quant,
output_scale,
input_scale,
kv_cache_dummy_dep,
layer_name,
):
# MLA output in quant_dtype (FP4 packed as uint8)
output_attn = torch.empty(
[q.shape[0], self._output_dim // 2],
dtype=FP4_DTYPE,
device=q.device,
)
output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
at2 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=input_scale,
output_block_scale=output_scale_view,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
return at2[1], at2[2]
return _replacement_with_ln
def _replacement(
q,
kv_c_normed,
k_pe,
output_attn,
_output_quant,
output_scale,
input_scale,
kv_cache_dummy_dep,
):
# MLA output in quant_dtype (FP4 packed as uint8)
output_attn = torch.empty(
[q.shape[0], self._output_dim // 2],
dtype=FP4_DTYPE,
device=q.device,
)
output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
at2 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=input_scale,
output_block_scale=output_scale_view,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
return at2[1], at2[2]
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
inputs: list = [
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
self.empty(5, self._output_dim, dtype=self._dtype),
self.empty(5, self._output_dim // 2, dtype=FP4_DTYPE),
self.empty_i32(128, round_up(self._output_dim // 16, 4)),
self.empty_fp32(1, 1),
self.empty(0, dtype=self._dtype),
]
if _USE_LAYERNAME:
inputs.append(_encode_layer_name(self._layer_name))
return inputs
class MLAAttnFp8GroupQuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
"""Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
Matches the pattern: MLA attention -> per_token_group_fp8_quant, and
replaces it with MLA attention(output_block_scale=group_scale_buffer).
Used by models with block FP8 quantization (e.g. DeepSeek V3).
"""
def __init__(
self,
layer: MLAAttention,
dtype: torch.dtype,
quant_key: QuantKey,
has_col_major_scales: bool,
is_e8m0: bool,
is_tma_aligned: bool,
) -> None:
self._layer_name = layer.layer_name
self._num_heads = layer.num_heads
self._v_head_dim = layer.v_head_dim
self._kv_lora_rank = layer.kv_lora_rank
self._qk_rope_head_dim = layer.qk_rope_head_dim
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
self._output_dim = layer.num_heads * layer.v_head_dim
self._dtype = dtype
self._layer = layer
self._group_size = quant_key.scale.group_shape[1]
self._has_col_major_scales = has_col_major_scales
self._is_e8m0 = is_e8m0
self._is_tma_aligned = is_tma_aligned
self._quant_matcher = MatcherQuantFP8(
quant_key,
has_col_major_scales=has_col_major_scales,
is_e8m0=is_e8m0,
is_tma_aligned=is_tma_aligned,
)
@property
def pattern(
self,
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _pattern_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
kv_cache_dummy_dep,
scale,
layer_name,
):
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
attn_out = at1[1]
result = torch.empty(
attn_out.shape, device=attn_out.device, dtype=FP8_DTYPE
)
finfo = torch.finfo(FP8_DTYPE)
_, result, scale = auto_functionalized(
self._quant_matcher.QUANT_OP,
input=attn_out,
output_q=result,
output_s=scale,
group_size=self._group_size,
eps=1e-10,
fp8_min=finfo.min,
fp8_max=finfo.max,
scale_ue8m0=self._is_e8m0,
dummy_is_scale_transposed=self._has_col_major_scales,
dummy_is_tma_aligned=self._is_tma_aligned,
)
return result, scale
return _pattern_with_ln
def _pattern(
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
output_attn: torch.Tensor,
kv_cache_dummy_dep: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=None,
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
attn_out = at1[1]
result = torch.empty(
attn_out.shape, device=attn_out.device, dtype=FP8_DTYPE
)
finfo = torch.finfo(FP8_DTYPE)
_, result, scale = auto_functionalized(
self._quant_matcher.QUANT_OP,
input=attn_out,
output_q=result,
output_s=scale,
group_size=self._group_size,
eps=1e-10,
fp8_min=finfo.min,
fp8_max=finfo.max,
scale_ue8m0=self._is_e8m0,
dummy_is_scale_transposed=self._has_col_major_scales,
dummy_is_tma_aligned=self._is_tma_aligned,
)
return result, scale
return _pattern
@property
def replacement(
self,
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
def _replacement_with_ln( # type: ignore[misc]
q,
kv_c_normed,
k_pe,
output_attn,
kv_cache_dummy_dep,
scale,
layer_name,
):
output_attn = torch.empty(
[q.shape[0], self._output_dim],
dtype=FP8_DTYPE,
device=q.device,
)
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=layer_name,
output_scale=None,
output_block_scale=scale,
kv_cache_dummy_dep=kv_cache_dummy_dep,
quant_group_size=self._group_size,
quant_scale_ue8m0=self._is_e8m0,
quant_col_major=self._has_col_major_scales,
quant_tma_aligned=self._is_tma_aligned,
)
return at1[1], at1[2]
return _replacement_with_ln
def _replacement(q, kv_c_normed, k_pe, output_attn, kv_cache_dummy_dep, scale):
output_attn = torch.empty(
[q.shape[0], self._output_dim],
dtype=FP8_DTYPE,
device=q.device,
)
at1 = auto_functionalized(
MLA_ATTN_OP,
q=q,
kv_c_normed=kv_c_normed,
k_pe=k_pe,
output=output_attn,
layer_name=_ln,
output_scale=None,
output_block_scale=scale,
kv_cache_dummy_dep=kv_cache_dummy_dep,
quant_group_size=self._group_size,
quant_scale_ue8m0=self._is_e8m0,
quant_col_major=self._has_col_major_scales,
quant_tma_aligned=self._is_tma_aligned,
)
return at1[1], at1[2]
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
inputs: list = [
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
self.empty(5, self._output_dim, dtype=self._dtype),
self.empty(0, dtype=self._dtype),
self._quant_matcher.empty_f32(1, 1),
]
if _USE_LAYERNAME:
inputs.append(_encode_layer_name(self._layer_name))
return inputs
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
"""This pass fuses post-attention quantization onto MLA attention if supported.
It uses the pattern matcher and matches each MLA layer manually, as strings
cannot be wildcarded. This also lets us check support on attention layers
upon registration instead of during pattern matching.
"""
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "mla_attn_quant_fusion")
dtype = config.model_config.dtype
layers = list(get_layers_from_vllm_config(config, MLAAttention).values())
if len(layers) == 0:
logger.warning(
"MLA attention + quant fusion is enabled, but no MLA "
"attention layers were found in "
"CompilationConfig.static_forward_context "
"so no fusion patterns were registered."
)
# When _USE_LAYERNAME is enabled, layer_name is a wildcard so all
# layers produce the same pattern — register once then break.
for layer in layers:
if layer.impl.fused_output_quant_supported(kFp8StaticTensorSym):
self.register(MLAAttnFp8StaticQuantPattern(layer, dtype))
if _USE_LAYERNAME:
break
if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
for layer in layers:
if layer.impl.fused_output_quant_supported(kNvfp4Dynamic):
self.register(MLAAttnNvfp4QuantPattern(layer, dtype))
if _USE_LAYERNAME:
break
# Per-group FP8 (block quant) — register all flag combinations.
if current_platform.is_cuda():
for quant_key in [kFp8Dynamic128Sym, kFp8Dynamic64Sym]:
for col_major in [True, False]:
for is_e8m0 in [True, False]:
for tma_aligned in [False, True]:
for layer in layers:
if layer.impl.fused_output_quant_supported(quant_key):
self.register(
MLAAttnFp8GroupQuantPattern(
layer,
dtype,
quant_key,
col_major,
is_e8m0,
tma_aligned,
)
)
if _USE_LAYERNAME:
break
self.dump_patterns(config, self.pm_pass)
@@ -1,274 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch._higher_order_ops.auto_functionalize import auto_functionalized
import vllm._custom_ops as ops
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.attention.attention import get_attention_context
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.utils.torch_utils import (
_USE_LAYERNAME,
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
from .matcher_utils import MatcherDeepseekScalingRotaryEmbedding, MatcherRotaryEmbedding
logger = init_logger(__name__)
def fused_rope_unified_mla_kv_cache_update_impl(
positions: torch.Tensor,
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c: torch.Tensor,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache_dtype: str,
kv_cache_scale: torch.Tensor,
layer_name: LayerNameType,
) -> torch.Tensor:
layer_name = _resolve_layer_name(layer_name)
attn_metadata, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(
layer_name
)
attn_layer.prepare_kv_cache_update(attn_metadata)
if layer_slot_mapping is not None:
ops.concat_and_cache_mla_rope_fused(
positions,
q_pe,
k_pe,
kv_c,
cos_sin_cache,
is_neox,
layer_slot_mapping,
kv_cache.squeeze(1),
kv_cache_dtype,
kv_cache_scale,
)
return torch.empty(0, device=kv_c.device, dtype=kv_c.dtype)
def fused_rope_unified_mla_kv_cache_update_fake(
positions: torch.Tensor,
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c: torch.Tensor,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache_dtype: str,
kv_cache_scale: torch.Tensor,
layer_name: LayerNameType,
) -> torch.Tensor:
return torch.empty(0, dtype=kv_c.dtype, device=kv_c.device)
direct_register_custom_op(
op_name="fused_rope_unified_mla_kv_cache_update",
op_func=fused_rope_unified_mla_kv_cache_update_impl,
fake_impl=fused_rope_unified_mla_kv_cache_update_fake,
mutates_args=["q_pe", "k_pe"],
)
class MLARoPEKVCacheCatPattern(VllmPatternReplacement):
FUSED_OP = torch.ops.vllm.fused_rope_unified_mla_kv_cache_update.default
def __init__(
self,
layer: MLAAttention,
is_neox: bool,
use_flashinfer: bool = False,
use_deepseek_scaling: bool = False,
) -> None:
self.layer_name = layer.layer_name
self.kv_cache_dtype = layer.kv_cache_dtype
self.num_heads = layer.num_heads
self.num_kv_heads = layer.num_kv_heads
self.kv_lora_rank = layer.kv_lora_rank
self.qk_rope_head_dim = layer.qk_rope_head_dim
self.is_neox = is_neox
self.use_flashinfer = use_flashinfer
self._ln = _encode_layer_name(self.layer_name)
if use_deepseek_scaling:
self.rope_matcher = MatcherDeepseekScalingRotaryEmbedding(
is_neox=self.is_neox,
head_size=self.qk_rope_head_dim,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
use_flashinfer=self.use_flashinfer,
)
else:
self.rope_matcher = MatcherRotaryEmbedding( # type: ignore
is_neox=self.is_neox,
head_size=self.qk_rope_head_dim,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
use_flashinfer=self.use_flashinfer,
)
def get_inputs(self) -> list[torch.Tensor]:
T = 5
L = 4096
q_pe = self.empty_bf16(T, self.num_heads, self.qk_rope_head_dim)
k_pe = self.empty_bf16(T, self.qk_rope_head_dim)
kv_c_normed = self.empty_bf16(T, self.kv_lora_rank)
cos_sin_cache = self.empty_bf16(L, self.qk_rope_head_dim)
positions = self.empty(T, dtype=torch.int64)
k_scale = self.empty(0, dtype=torch.float32)
inputs = [
q_pe,
k_pe,
kv_c_normed,
positions,
cos_sin_cache,
k_scale,
]
if _USE_LAYERNAME:
inputs.append(self._ln)
return inputs
@property
def pattern(self):
_ln = self._ln
if _USE_LAYERNAME:
def _pattern_with_ln(
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c_normed: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
k_scale: torch.Tensor,
layer_name: LayerNameType,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
k_pe_unsqueezed = k_pe.unsqueeze(1)
q_pe, k_pe = self.rope_matcher(
positions, q_pe, k_pe_unsqueezed, cos_sin_cache
)
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
kv_c_normed, k_pe, layer_name, self.kv_cache_dtype, k_scale
)
return dummy, q_pe, k_pe
return _pattern_with_ln
def _pattern(
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c_normed: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
k_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
k_pe_unsqueezed = k_pe.unsqueeze(1)
q_pe, k_pe = self.rope_matcher(
positions, q_pe, k_pe_unsqueezed, cos_sin_cache
)
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
kv_c_normed, k_pe, _ln, self.kv_cache_dtype, k_scale
)
return dummy, q_pe, k_pe
return _pattern
@property
def replacement(self):
_ln = self._ln
if _USE_LAYERNAME:
def _replacement_with_ln(
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c_normed: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
k_scale: torch.Tensor,
layer_name: LayerNameType,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
at = auto_functionalized(
self.FUSED_OP,
positions=positions,
q_pe=q_pe,
k_pe=k_pe,
kv_c=kv_c_normed,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
kv_cache_dtype=self.kv_cache_dtype,
kv_cache_scale=k_scale,
layer_name=layer_name,
)
dummy, q_pe, k_pe_squeezed = at
k_pe = k_pe_squeezed.unsqueeze(1)
return dummy, q_pe, k_pe
return _replacement_with_ln
def _replacement(
q_pe: torch.Tensor,
k_pe: torch.Tensor,
kv_c_normed: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
k_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
at = auto_functionalized(
self.FUSED_OP,
positions=positions,
q_pe=q_pe,
k_pe=k_pe,
kv_c=kv_c_normed,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
kv_cache_dtype=self.kv_cache_dtype,
kv_cache_scale=k_scale,
layer_name=_ln,
)
dummy, q_pe, k_pe_squeezed = at
k_pe = k_pe_squeezed.unsqueeze(1)
return dummy, q_pe, k_pe
return _replacement
class MLARoPEKVCacheCatFusionPass(VllmFusionPatternMatcherPass):
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "mla_rope_kv_cache_fusion_pass")
attn_layers = get_layers_from_vllm_config(config, MLAAttention)
for _, layer in attn_layers.items():
for is_neox in [False, True]:
for use_deepseek_scaling in [False, True]:
if RotaryEmbedding.enabled():
for use_flashinfer in [False, True]:
self.register(
MLARoPEKVCacheCatPattern(
layer,
is_neox,
use_flashinfer,
use_deepseek_scaling,
)
)
else:
self.register(
MLARoPEKVCacheCatPattern(
layer,
is_neox,
use_deepseek_scaling=use_deepseek_scaling,
)
)
if _USE_LAYERNAME:
break
self.dump_patterns(config, self.pm_pass)
-6
View File
@@ -40,8 +40,6 @@ if current_platform.is_cuda_alike() or current_platform.is_xpu():
if current_platform.is_cuda_alike():
from .fusion.act_quant_fusion import ActivationQuantFusionPass
from .fusion.attn_quant_fusion import AttnQuantFusionPass
from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass
from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass
from .fusion.qk_norm_rope_kvcache_fusion import QkNormRopeKvCacheFusionPass
from .fusion.rms_quant_fusion import RMSNormQuantFusionPass
from .fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
@@ -206,12 +204,8 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
self.passes += [ScatterSplitReplacementPass(config)]
self.passes += [RopeKVCacheFusionPass(config)]
if self.pass_config.fuse_rope_kvcache_cat_mla:
self.passes += [MLARoPEKVCacheCatFusionPass(config)]
if self.pass_config.fuse_attn_quant:
self.passes += [AttnQuantFusionPass(config)]
self.passes += [MLAAttnQuantFusionPass(config)]
if self.pass_config.enable_qk_norm_rope_fusion:
self.passes += [SplitCoalescingPass(config)]