Support DeepSeek-V4 AMD Quark NVFP4 with emulation kernel (#47972)
Signed-off-by: Jimmy Adams <Adam_ji@outlook.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: huji <huidong.ji@amd.com> Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: huji <huidong.ji@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Andreas Karatzas
huji
Cursor
parent
0de0362ea1
commit
da788334bc
@@ -750,6 +750,12 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
def _resolve_param_name(name: str) -> str:
|
||||
inv_name = f"{name}_inv"
|
||||
if name not in params_dict and inv_name in params_dict:
|
||||
return inv_name
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
@@ -803,10 +809,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
break
|
||||
param = params_dict[name]
|
||||
param_name = _resolve_param_name(name)
|
||||
param = params_dict[param_name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(name)
|
||||
loaded_params.add(param_name)
|
||||
break
|
||||
else:
|
||||
if ".experts." in name:
|
||||
@@ -857,12 +864,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
param_name = _resolve_param_name(name)
|
||||
param = params_dict[param_name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
loaded_params.add(param_name)
|
||||
continue
|
||||
|
||||
return loaded_params
|
||||
@@ -934,6 +942,7 @@ def _make_deepseek_v4_weights_mapper(
|
||||
"head.weight": "lm_head.weight",
|
||||
"embed.weight": "embed_tokens.weight",
|
||||
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
|
||||
".input_scale": ".input_scale_2",
|
||||
},
|
||||
orig_to_new_substr=substr_map,
|
||||
)
|
||||
@@ -945,6 +954,10 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
# Default mapper assumes the original FP4-expert checkpoint layout.
|
||||
# Overridden per-instance in __init__ when expert_dtype != "fp4".
|
||||
hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4")
|
||||
packed_modules_mapping = {
|
||||
"gate_up_proj": ["w1", "w3"],
|
||||
"fused_wqa_wkv": ["wq_a", "wkv"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
Reference in New Issue
Block a user