[Refactor][Quantization] Make FP8 and MLA weight transforms reusable pure functions (#57732)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -58,6 +58,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
accumulate_mla_context_chunk,
|
||||
init_mla_context_partial,
|
||||
neutralize_empty_context_partials,
|
||||
split_kv_b_proj,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -73,9 +74,6 @@ from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_and_maybe_dequant_weights,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
@@ -441,20 +439,13 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
projected into latent space by ``W_UK_T`` and the attention output is
|
||||
projected back to ``v`` by ``W_UV`` -- avoiding materializing full K/V.
|
||||
"""
|
||||
kv_b_proj_weight = get_and_maybe_dequant_weights(
|
||||
self.kv_b_proj, out_dtype=act_dtype
|
||||
).T
|
||||
assert kv_b_proj_weight.shape == (
|
||||
self.kv_lora_rank,
|
||||
self.num_local_heads * (self.qk_nope_head_dim + self.v_head_dim),
|
||||
), f"{kv_b_proj_weight.shape=}"
|
||||
kv_b_proj_weight = kv_b_proj_weight.view(
|
||||
W_UK, W_UV = split_kv_b_proj(
|
||||
self.kv_b_proj,
|
||||
act_dtype,
|
||||
self.kv_lora_rank,
|
||||
self.num_local_heads,
|
||||
self.qk_nope_head_dim + self.v_head_dim,
|
||||
)
|
||||
W_UK, W_UV = kv_b_proj_weight.split(
|
||||
[self.qk_nope_head_dim, self.v_head_dim], dim=-1
|
||||
self.qk_nope_head_dim,
|
||||
self.v_head_dim,
|
||||
)
|
||||
# (L, N, V) -> (N, L, V)
|
||||
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
|
||||
|
||||
Reference in New Issue
Block a user