Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -335,8 +335,8 @@ class AriaTextModel(LlamaModel, SupportsQuant):
|
|||||||
packed_modules_mapping = {
|
packed_modules_mapping = {
|
||||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||||
"experts.w13_weight": ["experts.fc1.weight"],
|
"experts.routed_experts.w13_weight": ["experts.fc1.weight"],
|
||||||
"experts.w2_weight": ["experts.fc2.weight"],
|
"experts.routed_experts.w2_weight": ["experts.fc2.weight"],
|
||||||
}
|
}
|
||||||
|
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||||
|
|||||||
@@ -187,8 +187,18 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
|||||||
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
|
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
|
||||||
)
|
)
|
||||||
fused_expert_params_mapping = [
|
fused_expert_params_mapping = [
|
||||||
(f"experts.{base_layer}w13_weight", "experts.gate_up_proj", 0, "w1"),
|
(
|
||||||
(f"experts.{base_layer}w2_weight", "experts.down_proj", 0, "w2"),
|
f"experts.routed_experts.{base_layer}w13_weight",
|
||||||
|
"experts.gate_up_proj",
|
||||||
|
0,
|
||||||
|
"w1",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f"experts.routed_experts.{base_layer}w2_weight",
|
||||||
|
"experts.down_proj",
|
||||||
|
0,
|
||||||
|
"w2",
|
||||||
|
),
|
||||||
]
|
]
|
||||||
num_experts = self.config.num_experts
|
num_experts = self.config.num_experts
|
||||||
for name, loaded_weight in weights:
|
for name, loaded_weight in weights:
|
||||||
|
|||||||
@@ -422,9 +422,21 @@ class Step3TextModel(nn.Module):
|
|||||||
)
|
)
|
||||||
|
|
||||||
expert_params_mapping = [
|
expert_params_mapping = [
|
||||||
(f".moe.experts.{base_layer}w13_weight", ".moe.gate_proj.weight", "w1"),
|
(
|
||||||
(f".moe.experts.{base_layer}w13_weight", ".moe.up_proj.weight", "w3"),
|
f".moe.experts.routed_experts.{base_layer}w13_weight",
|
||||||
(f".moe.experts.{base_layer}w2_weight", ".moe.down_proj.weight", "w2"),
|
".moe.gate_proj.weight",
|
||||||
|
"w1",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f".moe.experts.routed_experts.{base_layer}w13_weight",
|
||||||
|
".moe.up_proj.weight",
|
||||||
|
"w3",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f".moe.experts.routed_experts.{base_layer}w2_weight",
|
||||||
|
".moe.down_proj.weight",
|
||||||
|
"w2",
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
disable_moe_stacked_params = [data[1] for data in expert_params_mapping]
|
disable_moe_stacked_params = [data[1] for data in expert_params_mapping]
|
||||||
|
|||||||
@@ -635,36 +635,48 @@ class Step3p5Model(nn.Module):
|
|||||||
|
|
||||||
# Old packed 3D format: .moe.gate_proj.weight [num_experts, out, in]
|
# Old packed 3D format: .moe.gate_proj.weight [num_experts, out, in]
|
||||||
expert_params_mapping = [
|
expert_params_mapping = [
|
||||||
(f".moe.experts.{base_layer}w13_weight", ".moe.gate_proj.weight", "w1"),
|
|
||||||
(f".moe.experts.{base_layer}w13_weight", ".moe.up_proj.weight", "w3"),
|
|
||||||
(f".moe.experts.{base_layer}w2_weight", ".moe.down_proj.weight", "w2"),
|
|
||||||
(
|
(
|
||||||
f".moe.experts.{base_layer}w13_weight_scale_2",
|
f".moe.experts.routed_experts.{base_layer}w13_weight",
|
||||||
|
".moe.gate_proj.weight",
|
||||||
|
"w1",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f".moe.experts.routed_experts.{base_layer}w13_weight",
|
||||||
|
".moe.up_proj.weight",
|
||||||
|
"w3",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f".moe.experts.routed_experts.{base_layer}w2_weight",
|
||||||
|
".moe.down_proj.weight",
|
||||||
|
"w2",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f".moe.experts.routed_experts.{base_layer}w13_weight_scale_2",
|
||||||
".moe.gate_proj.weight_scale_2",
|
".moe.gate_proj.weight_scale_2",
|
||||||
"w1",
|
"w1",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
f".moe.experts.{base_layer}w13_weight_scale_2",
|
f".moe.experts.routed_experts.{base_layer}w13_weight_scale_2",
|
||||||
".moe.up_proj.weight_scale_2",
|
".moe.up_proj.weight_scale_2",
|
||||||
"w3",
|
"w3",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
f".moe.experts.{base_layer}w2_weight_scale_2",
|
f".moe.experts.routed_experts.{base_layer}w2_weight_scale_2",
|
||||||
".moe.down_proj.weight_scale_2",
|
".moe.down_proj.weight_scale_2",
|
||||||
"w2",
|
"w2",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
f".moe.experts.{base_layer}w13_weight_scale",
|
f".moe.experts.routed_experts.{base_layer}w13_weight_scale",
|
||||||
".moe.gate_proj.weight_scale",
|
".moe.gate_proj.weight_scale",
|
||||||
"w1",
|
"w1",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
f".moe.experts.{base_layer}w13_weight_scale",
|
f".moe.experts.routed_experts.{base_layer}w13_weight_scale",
|
||||||
".moe.up_proj.weight_scale",
|
".moe.up_proj.weight_scale",
|
||||||
"w3",
|
"w3",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
f".moe.experts.{base_layer}w2_weight_scale",
|
f".moe.experts.routed_experts.{base_layer}w2_weight_scale",
|
||||||
".moe.down_proj.weight_scale",
|
".moe.down_proj.weight_scale",
|
||||||
"w2",
|
"w2",
|
||||||
),
|
),
|
||||||
@@ -672,17 +684,17 @@ class Step3p5Model(nn.Module):
|
|||||||
# input scales are stored as moe.{gate,up,down}_proj.input_scale
|
# input scales are stored as moe.{gate,up,down}_proj.input_scale
|
||||||
# rather than the standard per-expert format handled generically.
|
# rather than the standard per-expert format handled generically.
|
||||||
(
|
(
|
||||||
f".moe.experts.{base_layer}w13_input_scale",
|
f".moe.experts.routed_experts.{base_layer}w13_input_scale",
|
||||||
".moe.gate_proj.input_scale",
|
".moe.gate_proj.input_scale",
|
||||||
"w1",
|
"w1",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
f".moe.experts.{base_layer}w13_input_scale",
|
f".moe.experts.routed_experts.{base_layer}w13_input_scale",
|
||||||
".moe.up_proj.input_scale",
|
".moe.up_proj.input_scale",
|
||||||
"w3",
|
"w3",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
f".moe.experts.{base_layer}w2_input_scale",
|
f".moe.experts.routed_experts.{base_layer}w2_input_scale",
|
||||||
".moe.down_proj.input_scale",
|
".moe.down_proj.input_scale",
|
||||||
"w2",
|
"w2",
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ from typing import TYPE_CHECKING
|
|||||||
|
|
||||||
from vllm.config import get_current_vllm_config
|
from vllm.config import get_current_vllm_config
|
||||||
from vllm.model_executor.layers.fused_moe import (
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
MoERunner,
|
|
||||||
RoutedExperts,
|
RoutedExperts,
|
||||||
UnquantizedFusedMoEMethod,
|
UnquantizedFusedMoEMethod,
|
||||||
)
|
)
|
||||||
@@ -133,7 +132,7 @@ class DeepseekV4FP8Config(Fp8Config):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
def get_quant_method(self, layer, prefix):
|
def get_quant_method(self, layer, prefix):
|
||||||
if isinstance(layer, (MoERunner, RoutedExperts)):
|
if isinstance(layer, RoutedExperts):
|
||||||
if is_layer_skipped(
|
if is_layer_skipped(
|
||||||
prefix=prefix,
|
prefix=prefix,
|
||||||
ignored_layers=self.ignored_layers,
|
ignored_layers=self.ignored_layers,
|
||||||
@@ -156,9 +155,6 @@ class DeepseekV4FP8Config(Fp8Config):
|
|||||||
return super().get_quant_method(layer, prefix)
|
return super().get_quant_method(layer, prefix)
|
||||||
|
|
||||||
def is_mxfp4_quant(self, prefix, layer):
|
def is_mxfp4_quant(self, prefix, layer):
|
||||||
if (
|
if not isinstance(layer, RoutedExperts) or self.expert_dtype != "fp4":
|
||||||
not isinstance(layer, (MoERunner, RoutedExperts))
|
|
||||||
or self.expert_dtype != "fp4"
|
|
||||||
):
|
|
||||||
return False
|
return False
|
||||||
return self.moe_quant_algo != "NVFP4"
|
return self.moe_quant_algo != "NVFP4"
|
||||||
|
|||||||
Reference in New Issue
Block a user