[Model]Support Step-3.7-Flash (#43859)
Signed-off-by: luotingdan <luotingdan@stepfun.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: luotingdan <luotingdan@stepfun.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Yu Huang <yuhuang@nvidia.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
luotingdan
Isotr0py
Yu Huang
Jee Jee Li
parent
325a1ec4fb
commit
b690b2bb67
@@ -260,7 +260,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
return router_logits_dtype != torch.float32
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
|
||||
@@ -363,7 +363,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
return router_logits_dtype != torch.float32
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -974,6 +974,19 @@ class FusedMoE(PluggableLayer):
|
||||
# this is needed for compressed-tensors only
|
||||
loaded_weight = loaded_weight.to(param.data.device)
|
||||
|
||||
# ModelOpt NVFP4 stores w13 input scales as two logical shards.
|
||||
# The generic assignment below would broadcast w1/w3 into the
|
||||
# whole expert row, so the second shard would overwrite the first.
|
||||
if (
|
||||
"ModelOpt" in quant_method_name
|
||||
and param.data.ndim == 2
|
||||
and shard_id in ("w1", "w3")
|
||||
):
|
||||
scale_expert_id = global_expert_id if use_global_sf else expert_id
|
||||
scale_shard_id = 0 if shard_id == "w1" else 1
|
||||
param.data[scale_expert_id][scale_shard_id] = loaded_weight.reshape(())
|
||||
return True if return_success else None
|
||||
|
||||
if (
|
||||
"compressed" in quant_method_name.lower()
|
||||
and param.data[expert_id] != 1
|
||||
|
||||
@@ -192,7 +192,11 @@ class ModelOptQuantConfigBase(QuantizationConfig):
|
||||
# exclude_modules config. But need to keep them for loading quantized
|
||||
# checkpoints generated by older versions. Then check substring matching
|
||||
# for patterns not caught by exact match
|
||||
if "vision_tower" in prefix or "vision_model" in prefix:
|
||||
if (
|
||||
"vision_tower" in prefix
|
||||
or "vision_model" in prefix
|
||||
or "vit_large_projector" in prefix
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
# now, the layer is quantized, handle it here
|
||||
@@ -2340,6 +2344,14 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
if key.startswith(prefix_dot):
|
||||
return info["quant_algo"].upper()
|
||||
|
||||
# FusedMoE expert prefix is e.g. "...moe.experts", while ModelOpt's
|
||||
# quantized_layers entries use "...moe.gate_proj" / "...moe.up_proj".
|
||||
if prefix.endswith(".experts"):
|
||||
parent_dot = prefix.rsplit(".experts", 1)[0] + "."
|
||||
for key, info in self.quantized_layers.items():
|
||||
if key.startswith(parent_dot):
|
||||
return info["quant_algo"].upper()
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user