[Model]Support Step-3.7-Flash (#43859)

Signed-off-by: luotingdan <luotingdan@stepfun.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: luotingdan <luotingdan@stepfun.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Yu Huang <yuhuang@nvidia.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
ltd0924
2026-05-28 17:01:48 -07:00
committed by GitHub
co-authored by luotingdan Isotr0py Yu Huang Jee Jee Li
parent 325a1ec4fb
commit b690b2bb67
15 changed files with 698 additions and 27 deletions
+1
View File
@@ -567,6 +567,7 @@ _MULTIMODAL_MODELS = {
"SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
"StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
"Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
"Step3p7ForConditionalGeneration": ("step3p7", "Step3p7ForConditionalGeneration"),
"TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),
"Tarsier2ForConditionalGeneration": (
"qwen2_vl",
+53 -1
View File
@@ -637,6 +637,54 @@ class Step3p5Model(nn.Module):
(f".moe.experts.{base_layer}w13_weight", ".moe.gate_proj.weight", "w1"),
(f".moe.experts.{base_layer}w13_weight", ".moe.up_proj.weight", "w3"),
(f".moe.experts.{base_layer}w2_weight", ".moe.down_proj.weight", "w2"),
(
f".moe.experts.{base_layer}w13_weight_scale_2",
".moe.gate_proj.weight_scale_2",
"w1",
),
(
f".moe.experts.{base_layer}w13_weight_scale_2",
".moe.up_proj.weight_scale_2",
"w3",
),
(
f".moe.experts.{base_layer}w2_weight_scale_2",
".moe.down_proj.weight_scale_2",
"w2",
),
(
f".moe.experts.{base_layer}w13_weight_scale",
".moe.gate_proj.weight_scale",
"w1",
),
(
f".moe.experts.{base_layer}w13_weight_scale",
".moe.up_proj.weight_scale",
"w3",
),
(
f".moe.experts.{base_layer}w2_weight_scale",
".moe.down_proj.weight_scale",
"w2",
),
# Required due to the Step3 HF model's packed expert format:
# input scales are stored as moe.{gate,up,down}_proj.input_scale
# rather than the standard per-expert format handled generically.
(
f".moe.experts.{base_layer}w13_input_scale",
".moe.gate_proj.input_scale",
"w1",
),
(
f".moe.experts.{base_layer}w13_input_scale",
".moe.up_proj.input_scale",
"w3",
),
(
f".moe.experts.{base_layer}w2_input_scale",
".moe.down_proj.input_scale",
"w2",
),
]
# New per-expert format: .moe.experts.E.gate_proj.weight_packed [out, in]
@@ -756,7 +804,11 @@ class Step3p5Model(nn.Module):
# Per-tensor global scales (e.g. weight_global_scale)
# have shape [1] in compressed-tensors NVFP4 checkpoints.
# Expand to per-expert before the iteration loop.
if (
if loaded_weight.ndim == 0:
loaded_weight = loaded_weight.unsqueeze(0).expand(
moe_expert_num
)
elif (
loaded_weight.shape[0] == 1
and loaded_weight.shape[0] != moe_expert_num
):
+90
View File
@@ -0,0 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Jurassic model."""
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.linear import ColumnParallelLinear
from .step3_vl import Step3VLForConditionalGeneration
from .step_vl import PerceptionEncoder
from .utils import WeightsMapper, init_vllm_registered_model, maybe_prefix
from .vision import run_dp_sharded_vision_model
logger = init_logger(__name__)
class Step3p7ForConditionalGeneration(Step3VLForConditionalGeneration):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"model.vision_model.": "vision_model.",
"model.vit_large_projector.": "vit_large_projector.",
"model.vit_large_projector": "vit_large_projector",
"model.language_model.": "language_model.model.",
"model.language_model": "language_model.model",
"model.": "language_model.model.",
"lm_head.": "language_model.lm_head.",
"lm_head": "language_model.lm_head",
},
orig_to_new_substr={
".attn.in_proj_weight": ".attn.qkv_proj.weight",
".attn.in_proj_bias": ".attn.qkv_proj.bias",
".mlp.c_fc": ".mlp.fc1",
".mlp.c_proj": ".mlp.fc2",
},
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super(Step3VLForConditionalGeneration, self).__init__()
config = vllm_config.model_config.hf_config
multimodal_config = vllm_config.model_config.multimodal_config
quant_config = vllm_config.quant_config
self.config = config
self.multimodal_config = multimodal_config
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
with self._mark_tower_model(vllm_config, "image"):
self.vision_model = PerceptionEncoder(
config.vision_config,
get_act_fn(config.vision_config.hidden_act),
quant_config=quant_config,
prefix=maybe_prefix(prefix, "vision_model"),
)
self.vit_large_projector = ColumnParallelLinear(
config.vision_config.width * 4,
config.text_config.hidden_size,
bias=config.projector_bias,
gather_output=True,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "vit_large_projector"),
disable_tp=self.use_data_parallel,
)
with self._mark_language_model(vllm_config):
self.language_model = init_vllm_registered_model(
vllm_config=vllm_config,
hf_config=config.text_config,
prefix=maybe_prefix(prefix, "language_model"),
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
def _get_vision_model_output(
self, input_tensor: torch.Tensor | None
) -> torch.Tensor | None:
if input_tensor is None:
return None
if self.use_data_parallel:
return run_dp_sharded_vision_model(input_tensor, self.vision_model)
return self.vision_model(input_tensor)
def _process_image_features(self, image_features: torch.Tensor) -> torch.Tensor:
image_features, _ = self.vit_large_projector(image_features)
return image_features