[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,629 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic64Sym,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
|
||||
|
||||
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
|
||||
from .matcher_utils import MatcherQuantFP8
|
||||
from .rms_quant_fusion import QUANT_OPS
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP4_DTYPE = torch.uint8
|
||||
|
||||
MLA_ATTN_OP = torch.ops.vllm.unified_mla_attention_with_output.default
|
||||
|
||||
|
||||
class MLAAttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
"""Fusion for MLA Attention+Fp8StaticQuant.
|
||||
|
||||
Matches the pattern: MLA attention -> static FP8 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output=fp8_buffer).
|
||||
"""
|
||||
|
||||
def __init__(self, layer: MLAAttention, dtype: torch.dtype) -> None:
|
||||
self._layer_name = layer.layer_name
|
||||
self._num_heads = layer.num_heads
|
||||
self._v_head_dim = layer.v_head_dim
|
||||
self._kv_lora_rank = layer.kv_lora_rank
|
||||
self._qk_rope_head_dim = layer.qk_rope_head_dim
|
||||
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
|
||||
self._output_dim = layer.num_heads * layer.v_head_dim
|
||||
self._dtype = dtype
|
||||
self._quant_matcher = MatcherQuantFP8(kFp8StaticTensorSym)
|
||||
|
||||
@property
|
||||
def pattern(self) -> Callable[..., torch.Tensor]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
# MLA output is already 2D (T, N*V), no reshape needed
|
||||
return self._quant_matcher(at1[1], scale)[0]
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(q, kv_c_normed, k_pe, output_attn, scale, kv_cache_dummy_dep):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
# MLA output is already 2D (T, N*V), no reshape needed
|
||||
return self._quant_matcher(at1[1], scale)[0]
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self) -> Callable[..., torch.Tensor]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
# MLA output in quant_dtype
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim],
|
||||
dtype=FP8_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=scale,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return at1[1]
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(q, kv_c_normed, k_pe, output_attn, scale, kv_cache_dummy_dep):
|
||||
# MLA output in quant_dtype
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim],
|
||||
dtype=FP8_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=scale,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return at1[1]
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
inputs: list = [
|
||||
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
|
||||
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._output_dim, dtype=self._dtype),
|
||||
self.empty_fp32(1, 1),
|
||||
self.empty(0, dtype=self._dtype),
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self._layer_name))
|
||||
return inputs
|
||||
|
||||
|
||||
class MLAAttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""Fusion for MLA Attention+Nvfp4Quant.
|
||||
|
||||
Matches the pattern: MLA attention -> NVFP4 quant, and replaces
|
||||
it with MLA attention(output_scale=scale, output_block_scale=block_scale,
|
||||
output=fp4_buffer).
|
||||
"""
|
||||
|
||||
def __init__(self, layer: MLAAttention, dtype: torch.dtype) -> None:
|
||||
self._layer_name = layer.layer_name
|
||||
self._num_heads = layer.num_heads
|
||||
self._v_head_dim = layer.v_head_dim
|
||||
self._kv_lora_rank = layer.kv_lora_rank
|
||||
self._qk_rope_head_dim = layer.qk_rope_head_dim
|
||||
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
|
||||
self._output_dim = layer.num_heads * layer.v_head_dim
|
||||
self._dtype = dtype
|
||||
self._QUANT_OP = QUANT_OPS[kNvfp4Dynamic]
|
||||
|
||||
@property
|
||||
def pattern(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
at2 = auto_functionalized(
|
||||
self._QUANT_OP,
|
||||
input=at1[1],
|
||||
input_scale=input_scale,
|
||||
is_sf_swizzled_layout=True,
|
||||
output=output_quant,
|
||||
output_scale=output_scale,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
|
||||
return at2[1], output_scale_view
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
at2 = auto_functionalized(
|
||||
self._QUANT_OP,
|
||||
input=at1[1],
|
||||
input_scale=input_scale,
|
||||
is_sf_swizzled_layout=True,
|
||||
output=output_quant,
|
||||
output_scale=output_scale,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(at2[2], FP8_DTYPE)
|
||||
return at2[1], output_scale_view
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
_output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
layer_name,
|
||||
):
|
||||
# MLA output in quant_dtype (FP4 packed as uint8)
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim // 2],
|
||||
dtype=FP4_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
|
||||
at2 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=input_scale,
|
||||
output_block_scale=output_scale_view,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return at2[1], at2[2]
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
_output_quant,
|
||||
output_scale,
|
||||
input_scale,
|
||||
kv_cache_dummy_dep,
|
||||
):
|
||||
# MLA output in quant_dtype (FP4 packed as uint8)
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim // 2],
|
||||
dtype=FP4_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE)
|
||||
at2 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=input_scale,
|
||||
output_block_scale=output_scale_view,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return at2[1], at2[2]
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
inputs: list = [
|
||||
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
|
||||
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._output_dim, dtype=self._dtype),
|
||||
self.empty(5, self._output_dim // 2, dtype=FP4_DTYPE),
|
||||
self.empty_i32(128, round_up(self._output_dim // 16, 4)),
|
||||
self.empty_fp32(1, 1),
|
||||
self.empty(0, dtype=self._dtype),
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self._layer_name))
|
||||
return inputs
|
||||
|
||||
|
||||
class MLAAttnFp8GroupQuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8).
|
||||
|
||||
Matches the pattern: MLA attention -> per_token_group_fp8_quant, and
|
||||
replaces it with MLA attention(output_block_scale=group_scale_buffer).
|
||||
Used by models with block FP8 quantization (e.g. DeepSeek V3).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer: MLAAttention,
|
||||
dtype: torch.dtype,
|
||||
quant_key: QuantKey,
|
||||
has_col_major_scales: bool,
|
||||
is_e8m0: bool,
|
||||
is_tma_aligned: bool,
|
||||
) -> None:
|
||||
self._layer_name = layer.layer_name
|
||||
self._num_heads = layer.num_heads
|
||||
self._v_head_dim = layer.v_head_dim
|
||||
self._kv_lora_rank = layer.kv_lora_rank
|
||||
self._qk_rope_head_dim = layer.qk_rope_head_dim
|
||||
self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim
|
||||
self._output_dim = layer.num_heads * layer.v_head_dim
|
||||
self._dtype = dtype
|
||||
self._layer = layer
|
||||
self._group_size = quant_key.scale.group_shape[1]
|
||||
self._has_col_major_scales = has_col_major_scales
|
||||
self._is_e8m0 = is_e8m0
|
||||
self._is_tma_aligned = is_tma_aligned
|
||||
|
||||
self._quant_matcher = MatcherQuantFP8(
|
||||
quant_key,
|
||||
has_col_major_scales=has_col_major_scales,
|
||||
is_e8m0=is_e8m0,
|
||||
is_tma_aligned=is_tma_aligned,
|
||||
)
|
||||
|
||||
@property
|
||||
def pattern(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
kv_cache_dummy_dep,
|
||||
scale,
|
||||
layer_name,
|
||||
):
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
attn_out = at1[1]
|
||||
result = torch.empty(
|
||||
attn_out.shape, device=attn_out.device, dtype=FP8_DTYPE
|
||||
)
|
||||
finfo = torch.finfo(FP8_DTYPE)
|
||||
_, result, scale = auto_functionalized(
|
||||
self._quant_matcher.QUANT_OP,
|
||||
input=attn_out,
|
||||
output_q=result,
|
||||
output_s=scale,
|
||||
group_size=self._group_size,
|
||||
eps=1e-10,
|
||||
fp8_min=finfo.min,
|
||||
fp8_max=finfo.max,
|
||||
scale_ue8m0=self._is_e8m0,
|
||||
dummy_is_scale_transposed=self._has_col_major_scales,
|
||||
dummy_is_tma_aligned=self._is_tma_aligned,
|
||||
)
|
||||
return result, scale
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
output_attn: torch.Tensor,
|
||||
kv_cache_dummy_dep: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
attn_out = at1[1]
|
||||
result = torch.empty(
|
||||
attn_out.shape, device=attn_out.device, dtype=FP8_DTYPE
|
||||
)
|
||||
finfo = torch.finfo(FP8_DTYPE)
|
||||
_, result, scale = auto_functionalized(
|
||||
self._quant_matcher.QUANT_OP,
|
||||
input=attn_out,
|
||||
output_q=result,
|
||||
output_s=scale,
|
||||
group_size=self._group_size,
|
||||
eps=1e-10,
|
||||
fp8_min=finfo.min,
|
||||
fp8_max=finfo.max,
|
||||
scale_ue8m0=self._is_e8m0,
|
||||
dummy_is_scale_transposed=self._has_col_major_scales,
|
||||
dummy_is_tma_aligned=self._is_tma_aligned,
|
||||
)
|
||||
return result, scale
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln( # type: ignore[misc]
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_attn,
|
||||
kv_cache_dummy_dep,
|
||||
scale,
|
||||
layer_name,
|
||||
):
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim],
|
||||
dtype=FP8_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=layer_name,
|
||||
output_scale=None,
|
||||
output_block_scale=scale,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
quant_group_size=self._group_size,
|
||||
quant_scale_ue8m0=self._is_e8m0,
|
||||
quant_col_major=self._has_col_major_scales,
|
||||
quant_tma_aligned=self._is_tma_aligned,
|
||||
)
|
||||
return at1[1], at1[2]
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(q, kv_c_normed, k_pe, output_attn, kv_cache_dummy_dep, scale):
|
||||
output_attn = torch.empty(
|
||||
[q.shape[0], self._output_dim],
|
||||
dtype=FP8_DTYPE,
|
||||
device=q.device,
|
||||
)
|
||||
at1 = auto_functionalized(
|
||||
MLA_ATTN_OP,
|
||||
q=q,
|
||||
kv_c_normed=kv_c_normed,
|
||||
k_pe=k_pe,
|
||||
output=output_attn,
|
||||
layer_name=_ln,
|
||||
output_scale=None,
|
||||
output_block_scale=scale,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
quant_group_size=self._group_size,
|
||||
quant_scale_ue8m0=self._is_e8m0,
|
||||
quant_col_major=self._has_col_major_scales,
|
||||
quant_tma_aligned=self._is_tma_aligned,
|
||||
)
|
||||
return at1[1], at1[2]
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
inputs: list = [
|
||||
self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._kv_lora_rank, dtype=self._dtype),
|
||||
self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype),
|
||||
self.empty(5, self._output_dim, dtype=self._dtype),
|
||||
self.empty(0, dtype=self._dtype),
|
||||
self._quant_matcher.empty_f32(1, 1),
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(_encode_layer_name(self._layer_name))
|
||||
return inputs
|
||||
|
||||
|
||||
class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""This pass fuses post-attention quantization onto MLA attention if supported.
|
||||
|
||||
It uses the pattern matcher and matches each MLA layer manually, as strings
|
||||
cannot be wildcarded. This also lets us check support on attention layers
|
||||
upon registration instead of during pattern matching.
|
||||
"""
|
||||
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "mla_attn_quant_fusion")
|
||||
|
||||
dtype = config.model_config.dtype
|
||||
layers = list(get_layers_from_vllm_config(config, MLAAttention).values())
|
||||
|
||||
if len(layers) == 0:
|
||||
logger.warning(
|
||||
"MLA attention + quant fusion is enabled, but no MLA "
|
||||
"attention layers were found in "
|
||||
"CompilationConfig.static_forward_context "
|
||||
"so no fusion patterns were registered."
|
||||
)
|
||||
|
||||
# When _USE_LAYERNAME is enabled, layer_name is a wildcard so all
|
||||
# layers produce the same pattern — register once then break.
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(kFp8StaticTensorSym):
|
||||
self.register(MLAAttnFp8StaticQuantPattern(layer, dtype))
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"):
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(kNvfp4Dynamic):
|
||||
self.register(MLAAttnNvfp4QuantPattern(layer, dtype))
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
# Per-group FP8 (block quant) — register all flag combinations.
|
||||
if current_platform.is_cuda():
|
||||
for quant_key in [kFp8Dynamic128Sym, kFp8Dynamic64Sym]:
|
||||
for col_major in [True, False]:
|
||||
for is_e8m0 in [True, False]:
|
||||
for tma_aligned in [False, True]:
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(quant_key):
|
||||
self.register(
|
||||
MLAAttnFp8GroupQuantPattern(
|
||||
layer,
|
||||
dtype,
|
||||
quant_key,
|
||||
col_major,
|
||||
is_e8m0,
|
||||
tma_aligned,
|
||||
)
|
||||
)
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
@@ -1,274 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.attention.attention import get_attention_context
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.utils.torch_utils import (
|
||||
_USE_LAYERNAME,
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
|
||||
from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement
|
||||
from .matcher_utils import MatcherDeepseekScalingRotaryEmbedding, MatcherRotaryEmbedding
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def fused_rope_unified_mla_kv_cache_update_impl(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
attn_metadata, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(
|
||||
layer_name
|
||||
)
|
||||
attn_layer.prepare_kv_cache_update(attn_metadata)
|
||||
if layer_slot_mapping is not None:
|
||||
ops.concat_and_cache_mla_rope_fused(
|
||||
positions,
|
||||
q_pe,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cos_sin_cache,
|
||||
is_neox,
|
||||
layer_slot_mapping,
|
||||
kv_cache.squeeze(1),
|
||||
kv_cache_dtype,
|
||||
kv_cache_scale,
|
||||
)
|
||||
return torch.empty(0, device=kv_c.device, dtype=kv_c.dtype)
|
||||
|
||||
|
||||
def fused_rope_unified_mla_kv_cache_update_fake(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(0, dtype=kv_c.dtype, device=kv_c.device)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_rope_unified_mla_kv_cache_update",
|
||||
op_func=fused_rope_unified_mla_kv_cache_update_impl,
|
||||
fake_impl=fused_rope_unified_mla_kv_cache_update_fake,
|
||||
mutates_args=["q_pe", "k_pe"],
|
||||
)
|
||||
|
||||
|
||||
class MLARoPEKVCacheCatPattern(VllmPatternReplacement):
|
||||
FUSED_OP = torch.ops.vllm.fused_rope_unified_mla_kv_cache_update.default
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer: MLAAttention,
|
||||
is_neox: bool,
|
||||
use_flashinfer: bool = False,
|
||||
use_deepseek_scaling: bool = False,
|
||||
) -> None:
|
||||
self.layer_name = layer.layer_name
|
||||
self.kv_cache_dtype = layer.kv_cache_dtype
|
||||
self.num_heads = layer.num_heads
|
||||
self.num_kv_heads = layer.num_kv_heads
|
||||
self.kv_lora_rank = layer.kv_lora_rank
|
||||
self.qk_rope_head_dim = layer.qk_rope_head_dim
|
||||
self.is_neox = is_neox
|
||||
self.use_flashinfer = use_flashinfer
|
||||
self._ln = _encode_layer_name(self.layer_name)
|
||||
|
||||
if use_deepseek_scaling:
|
||||
self.rope_matcher = MatcherDeepseekScalingRotaryEmbedding(
|
||||
is_neox=self.is_neox,
|
||||
head_size=self.qk_rope_head_dim,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
use_flashinfer=self.use_flashinfer,
|
||||
)
|
||||
else:
|
||||
self.rope_matcher = MatcherRotaryEmbedding( # type: ignore
|
||||
is_neox=self.is_neox,
|
||||
head_size=self.qk_rope_head_dim,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
use_flashinfer=self.use_flashinfer,
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
T = 5
|
||||
L = 4096
|
||||
q_pe = self.empty_bf16(T, self.num_heads, self.qk_rope_head_dim)
|
||||
k_pe = self.empty_bf16(T, self.qk_rope_head_dim)
|
||||
kv_c_normed = self.empty_bf16(T, self.kv_lora_rank)
|
||||
cos_sin_cache = self.empty_bf16(L, self.qk_rope_head_dim)
|
||||
positions = self.empty(T, dtype=torch.int64)
|
||||
k_scale = self.empty(0, dtype=torch.float32)
|
||||
inputs = [
|
||||
q_pe,
|
||||
k_pe,
|
||||
kv_c_normed,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
k_scale,
|
||||
]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(self._ln)
|
||||
return inputs
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
_ln = self._ln
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _pattern_with_ln(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
k_pe_unsqueezed = k_pe.unsqueeze(1)
|
||||
q_pe, k_pe = self.rope_matcher(
|
||||
positions, q_pe, k_pe_unsqueezed, cos_sin_cache
|
||||
)
|
||||
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
|
||||
kv_c_normed, k_pe, layer_name, self.kv_cache_dtype, k_scale
|
||||
)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
def _pattern(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
k_pe_unsqueezed = k_pe.unsqueeze(1)
|
||||
q_pe, k_pe = self.rope_matcher(
|
||||
positions, q_pe, k_pe_unsqueezed, cos_sin_cache
|
||||
)
|
||||
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
|
||||
kv_c_normed, k_pe, _ln, self.kv_cache_dtype, k_scale
|
||||
)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
_ln = self._ln
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def _replacement_with_ln(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
at = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
positions=positions,
|
||||
q_pe=q_pe,
|
||||
k_pe=k_pe,
|
||||
kv_c=kv_c_normed,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
kv_cache_scale=k_scale,
|
||||
layer_name=layer_name,
|
||||
)
|
||||
dummy, q_pe, k_pe_squeezed = at
|
||||
k_pe = k_pe_squeezed.unsqueeze(1)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
def _replacement(
|
||||
q_pe: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
at = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
positions=positions,
|
||||
q_pe=q_pe,
|
||||
k_pe=k_pe,
|
||||
kv_c=kv_c_normed,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
kv_cache_scale=k_scale,
|
||||
layer_name=_ln,
|
||||
)
|
||||
dummy, q_pe, k_pe_squeezed = at
|
||||
k_pe = k_pe_squeezed.unsqueeze(1)
|
||||
return dummy, q_pe, k_pe
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class MLARoPEKVCacheCatFusionPass(VllmFusionPatternMatcherPass):
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "mla_rope_kv_cache_fusion_pass")
|
||||
|
||||
attn_layers = get_layers_from_vllm_config(config, MLAAttention)
|
||||
|
||||
for _, layer in attn_layers.items():
|
||||
for is_neox in [False, True]:
|
||||
for use_deepseek_scaling in [False, True]:
|
||||
if RotaryEmbedding.enabled():
|
||||
for use_flashinfer in [False, True]:
|
||||
self.register(
|
||||
MLARoPEKVCacheCatPattern(
|
||||
layer,
|
||||
is_neox,
|
||||
use_flashinfer,
|
||||
use_deepseek_scaling,
|
||||
)
|
||||
)
|
||||
else:
|
||||
self.register(
|
||||
MLARoPEKVCacheCatPattern(
|
||||
layer,
|
||||
is_neox,
|
||||
use_deepseek_scaling=use_deepseek_scaling,
|
||||
)
|
||||
)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
@@ -40,8 +40,6 @@ if current_platform.is_cuda_alike() or current_platform.is_xpu():
|
||||
if current_platform.is_cuda_alike():
|
||||
from .fusion.act_quant_fusion import ActivationQuantFusionPass
|
||||
from .fusion.attn_quant_fusion import AttnQuantFusionPass
|
||||
from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass
|
||||
from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass
|
||||
from .fusion.qk_norm_rope_kvcache_fusion import QkNormRopeKvCacheFusionPass
|
||||
from .fusion.rms_quant_fusion import RMSNormQuantFusionPass
|
||||
from .fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
|
||||
@@ -206,12 +204,8 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
self.passes += [ScatterSplitReplacementPass(config)]
|
||||
self.passes += [RopeKVCacheFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_rope_kvcache_cat_mla:
|
||||
self.passes += [MLARoPEKVCacheCatFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_attn_quant:
|
||||
self.passes += [AttnQuantFusionPass(config)]
|
||||
self.passes += [MLAAttnQuantFusionPass(config)]
|
||||
|
||||
if self.pass_config.enable_qk_norm_rope_fusion:
|
||||
self.passes += [SplitCoalescingPass(config)]
|
||||
|
||||
Reference in New Issue
Block a user