[Model]Support Step-3.7-Flash (#43859)
Signed-off-by: luotingdan <luotingdan@stepfun.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: luotingdan <luotingdan@stepfun.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Yu Huang <yuhuang@nvidia.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
luotingdan
Isotr0py
Yu Huang
Jee Jee Li
parent
325a1ec4fb
commit
b690b2bb67
@@ -260,7 +260,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
return router_logits_dtype != torch.float32
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
|
||||
@@ -363,7 +363,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
return router_logits_dtype != torch.float32
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -974,6 +974,19 @@ class FusedMoE(PluggableLayer):
|
||||
# this is needed for compressed-tensors only
|
||||
loaded_weight = loaded_weight.to(param.data.device)
|
||||
|
||||
# ModelOpt NVFP4 stores w13 input scales as two logical shards.
|
||||
# The generic assignment below would broadcast w1/w3 into the
|
||||
# whole expert row, so the second shard would overwrite the first.
|
||||
if (
|
||||
"ModelOpt" in quant_method_name
|
||||
and param.data.ndim == 2
|
||||
and shard_id in ("w1", "w3")
|
||||
):
|
||||
scale_expert_id = global_expert_id if use_global_sf else expert_id
|
||||
scale_shard_id = 0 if shard_id == "w1" else 1
|
||||
param.data[scale_expert_id][scale_shard_id] = loaded_weight.reshape(())
|
||||
return True if return_success else None
|
||||
|
||||
if (
|
||||
"compressed" in quant_method_name.lower()
|
||||
and param.data[expert_id] != 1
|
||||
|
||||
@@ -192,7 +192,11 @@ class ModelOptQuantConfigBase(QuantizationConfig):
|
||||
# exclude_modules config. But need to keep them for loading quantized
|
||||
# checkpoints generated by older versions. Then check substring matching
|
||||
# for patterns not caught by exact match
|
||||
if "vision_tower" in prefix or "vision_model" in prefix:
|
||||
if (
|
||||
"vision_tower" in prefix
|
||||
or "vision_model" in prefix
|
||||
or "vit_large_projector" in prefix
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
# now, the layer is quantized, handle it here
|
||||
@@ -2340,6 +2344,14 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
if key.startswith(prefix_dot):
|
||||
return info["quant_algo"].upper()
|
||||
|
||||
# FusedMoE expert prefix is e.g. "...moe.experts", while ModelOpt's
|
||||
# quantized_layers entries use "...moe.gate_proj" / "...moe.up_proj".
|
||||
if prefix.endswith(".experts"):
|
||||
parent_dot = prefix.rsplit(".experts", 1)[0] + "."
|
||||
for key, info in self.quantized_layers.items():
|
||||
if key.startswith(parent_dot):
|
||||
return info["quant_algo"].upper()
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -567,6 +567,7 @@ _MULTIMODAL_MODELS = {
|
||||
"SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
|
||||
"StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
|
||||
"Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
|
||||
"Step3p7ForConditionalGeneration": ("step3p7", "Step3p7ForConditionalGeneration"),
|
||||
"TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),
|
||||
"Tarsier2ForConditionalGeneration": (
|
||||
"qwen2_vl",
|
||||
|
||||
@@ -637,6 +637,54 @@ class Step3p5Model(nn.Module):
|
||||
(f".moe.experts.{base_layer}w13_weight", ".moe.gate_proj.weight", "w1"),
|
||||
(f".moe.experts.{base_layer}w13_weight", ".moe.up_proj.weight", "w3"),
|
||||
(f".moe.experts.{base_layer}w2_weight", ".moe.down_proj.weight", "w2"),
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_weight_scale_2",
|
||||
".moe.gate_proj.weight_scale_2",
|
||||
"w1",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_weight_scale_2",
|
||||
".moe.up_proj.weight_scale_2",
|
||||
"w3",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w2_weight_scale_2",
|
||||
".moe.down_proj.weight_scale_2",
|
||||
"w2",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_weight_scale",
|
||||
".moe.gate_proj.weight_scale",
|
||||
"w1",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_weight_scale",
|
||||
".moe.up_proj.weight_scale",
|
||||
"w3",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w2_weight_scale",
|
||||
".moe.down_proj.weight_scale",
|
||||
"w2",
|
||||
),
|
||||
# Required due to the Step3 HF model's packed expert format:
|
||||
# input scales are stored as moe.{gate,up,down}_proj.input_scale
|
||||
# rather than the standard per-expert format handled generically.
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_input_scale",
|
||||
".moe.gate_proj.input_scale",
|
||||
"w1",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w13_input_scale",
|
||||
".moe.up_proj.input_scale",
|
||||
"w3",
|
||||
),
|
||||
(
|
||||
f".moe.experts.{base_layer}w2_input_scale",
|
||||
".moe.down_proj.input_scale",
|
||||
"w2",
|
||||
),
|
||||
]
|
||||
|
||||
# New per-expert format: .moe.experts.E.gate_proj.weight_packed [out, in]
|
||||
@@ -756,7 +804,11 @@ class Step3p5Model(nn.Module):
|
||||
# Per-tensor global scales (e.g. weight_global_scale)
|
||||
# have shape [1] in compressed-tensors NVFP4 checkpoints.
|
||||
# Expand to per-expert before the iteration loop.
|
||||
if (
|
||||
if loaded_weight.ndim == 0:
|
||||
loaded_weight = loaded_weight.unsqueeze(0).expand(
|
||||
moe_expert_num
|
||||
)
|
||||
elif (
|
||||
loaded_weight.shape[0] == 1
|
||||
and loaded_weight.shape[0] != moe_expert_num
|
||||
):
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Jurassic model."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
|
||||
from .step3_vl import Step3VLForConditionalGeneration
|
||||
from .step_vl import PerceptionEncoder
|
||||
from .utils import WeightsMapper, init_vllm_registered_model, maybe_prefix
|
||||
from .vision import run_dp_sharded_vision_model
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Step3p7ForConditionalGeneration(Step3VLForConditionalGeneration):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"model.vision_model.": "vision_model.",
|
||||
"model.vit_large_projector.": "vit_large_projector.",
|
||||
"model.vit_large_projector": "vit_large_projector",
|
||||
"model.language_model.": "language_model.model.",
|
||||
"model.language_model": "language_model.model",
|
||||
"model.": "language_model.model.",
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
"lm_head": "language_model.lm_head",
|
||||
},
|
||||
orig_to_new_substr={
|
||||
".attn.in_proj_weight": ".attn.qkv_proj.weight",
|
||||
".attn.in_proj_bias": ".attn.qkv_proj.bias",
|
||||
".mlp.c_fc": ".mlp.fc1",
|
||||
".mlp.c_proj": ".mlp.fc2",
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super(Step3VLForConditionalGeneration, self).__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
self.multimodal_config = multimodal_config
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
|
||||
with self._mark_tower_model(vllm_config, "image"):
|
||||
self.vision_model = PerceptionEncoder(
|
||||
config.vision_config,
|
||||
get_act_fn(config.vision_config.hidden_act),
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "vision_model"),
|
||||
)
|
||||
self.vit_large_projector = ColumnParallelLinear(
|
||||
config.vision_config.width * 4,
|
||||
config.text_config.hidden_size,
|
||||
bias=config.projector_bias,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "vit_large_projector"),
|
||||
disable_tp=self.use_data_parallel,
|
||||
)
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config.text_config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
)
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _get_vision_model_output(
|
||||
self, input_tensor: torch.Tensor | None
|
||||
) -> torch.Tensor | None:
|
||||
if input_tensor is None:
|
||||
return None
|
||||
if self.use_data_parallel:
|
||||
return run_dp_sharded_vision_model(input_tensor, self.vision_model)
|
||||
return self.vision_model(input_tensor)
|
||||
|
||||
def _process_image_features(self, image_features: torch.Tensor) -> torch.Tensor:
|
||||
image_features, _ = self.vit_large_projector(image_features)
|
||||
return image_features
|
||||
Reference in New Issue
Block a user