[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn (#25968)
Signed-off-by: Pavani Majety <pmajety@nvidia.com>
This commit is contained in:
@@ -1003,12 +1003,18 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]:
|
||||
return None
|
||||
return remapped_name
|
||||
|
||||
if any("mla_attn" in key for key in params_dict):
|
||||
attn_str = "mla_attn.mla_attn"
|
||||
logger.debug_once(f"Found mla_attn with k_scale and v_scale in "
|
||||
f"the checkpoint, using {attn_str} as attn_str")
|
||||
else:
|
||||
attn_str = "attn"
|
||||
# Define scale name mapping patterns in order of precedence
|
||||
scale_mapping_patterns = [
|
||||
# ModelOpt format: .self_attn.{k,v}_proj.{k,v}_scale ->
|
||||
# .self_attn.attn.{k,v}_scale
|
||||
(r"\.self_attn\.([kv])_proj\.([kv])_scale$",
|
||||
r".self_attn.attn.\2_scale"),
|
||||
rf".self_attn.{attn_str}.\2_scale"),
|
||||
# QKV proj format: .self_attn.qkv_proj.{k,v}_scale ->
|
||||
# .self_attn.attn.{k,v}_scale
|
||||
(r"\.self_attn\.qkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"),
|
||||
|
||||
Reference in New Issue
Block a user