[Model] Support Qwen3.5 text-only dense and MoE models (#50210)
Signed-off-by: Perkz Zheng <PerkzZheng@users.noreply.github.com> Co-authored-by: Perkz Zheng <PerkzZheng@users.noreply.github.com>
This commit is contained in:
co-authored by
Perkz Zheng
parent
f5a7cce9b6
commit
ad5d29db70
@@ -497,6 +497,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
|||||||
"Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
|
"Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
|
||||||
"Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"),
|
"Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"),
|
||||||
"Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
|
"Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
|
||||||
|
"Qwen3_5ForCausalLM": _HfExamplesInfo("codecho/Qwen3.5-0.8B-text-only"),
|
||||||
|
"Qwen3_5MoeForCausalLM": _HfExamplesInfo("codecho/Qwen3.5-35B-A3B-text-only"),
|
||||||
"MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"),
|
"MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"),
|
||||||
"Qwen3NextForCausalLM": _HfExamplesInfo(
|
"Qwen3NextForCausalLM": _HfExamplesInfo(
|
||||||
"Qwen/Qwen3-Next-80B-A3B-Instruct",
|
"Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||||
|
|||||||
@@ -768,6 +768,20 @@ class Qwen3_5ForConditionalGenerationConfig(VerifyAndUpdateConfig):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Qwen3_5ForCausalLMConfig(Qwen3_5ForConditionalGenerationConfig):
|
||||||
|
@staticmethod
|
||||||
|
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||||
|
Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||||
|
|
||||||
|
# Text-only Qwen3.5 models use one-dimensional positions. Remove the
|
||||||
|
# M-RoPE fields inherited from the multimodal configuration.
|
||||||
|
hf_text_config = vllm_config.model_config.hf_text_config
|
||||||
|
rope_parameters = getattr(hf_text_config, "rope_parameters", None)
|
||||||
|
if rope_parameters is not None:
|
||||||
|
rope_parameters.pop("mrope_section", None)
|
||||||
|
rope_parameters.pop("mrope_interleaved", None)
|
||||||
|
|
||||||
|
|
||||||
class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
|
class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
|
||||||
"""Apply the attention contract declared by a ColQwen3.5 checkpoint."""
|
"""Apply the attention contract declared by a ColQwen3.5 checkpoint."""
|
||||||
|
|
||||||
@@ -884,7 +898,9 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
|||||||
"Qwen2ForRewardModel": Qwen2ForRewardModelConfig,
|
"Qwen2ForRewardModel": Qwen2ForRewardModelConfig,
|
||||||
"Qwen3ForSequenceClassification": Qwen3ForSequenceClassificationConfig,
|
"Qwen3ForSequenceClassification": Qwen3ForSequenceClassificationConfig,
|
||||||
"Qwen3VLForSequenceClassification": Qwen3VLForSequenceClassificationConfig,
|
"Qwen3VLForSequenceClassification": Qwen3VLForSequenceClassificationConfig,
|
||||||
|
"Qwen3_5ForCausalLM": Qwen3_5ForCausalLMConfig,
|
||||||
"Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
"Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
||||||
|
"Qwen3_5MoeForCausalLM": Qwen3_5ForCausalLMConfig,
|
||||||
"Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
"Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
||||||
"UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig,
|
"UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig,
|
||||||
"VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
|
"VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
|
||||||
|
|||||||
@@ -282,6 +282,7 @@ class Qwen3_5Model(Qwen3NextModel):
|
|||||||
class Qwen3_5ForCausalLMBase(
|
class Qwen3_5ForCausalLMBase(
|
||||||
nn.Module,
|
nn.Module,
|
||||||
HasInnerState,
|
HasInnerState,
|
||||||
|
IsHybrid,
|
||||||
SupportsEagle3,
|
SupportsEagle3,
|
||||||
SupportsLoRA,
|
SupportsLoRA,
|
||||||
SupportsPP,
|
SupportsPP,
|
||||||
@@ -361,6 +362,45 @@ class Qwen3_5ForCausalLMBase(
|
|||||||
|
|
||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_mamba_state_dtype_from_config(
|
||||||
|
cls,
|
||||||
|
vllm_config: "VllmConfig",
|
||||||
|
) -> tuple[torch.dtype, torch.dtype]:
|
||||||
|
return MambaStateDtypeCalculator.gated_delta_net_state_dtype(
|
||||||
|
vllm_config.model_config.dtype,
|
||||||
|
vllm_config.cache_config.mamba_cache_dtype,
|
||||||
|
vllm_config.cache_config.mamba_ssm_cache_dtype,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_mamba_state_shape_from_config(
|
||||||
|
cls, vllm_config: "VllmConfig"
|
||||||
|
) -> tuple[tuple[int, int], tuple[int, int]]:
|
||||||
|
parallel_config = vllm_config.parallel_config
|
||||||
|
hf_config = vllm_config.model_config.hf_text_config
|
||||||
|
tp_size = parallel_config.tensor_parallel_size
|
||||||
|
num_spec = (
|
||||||
|
vllm_config.speculative_config.num_speculative_tokens
|
||||||
|
if vllm_config.speculative_config
|
||||||
|
else 0
|
||||||
|
)
|
||||||
|
return MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||||
|
tp_size,
|
||||||
|
hf_config.linear_num_key_heads,
|
||||||
|
hf_config.linear_num_value_heads,
|
||||||
|
hf_config.linear_key_head_dim,
|
||||||
|
hf_config.linear_value_head_dim,
|
||||||
|
hf_config.linear_conv_kernel_dim,
|
||||||
|
num_spec,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_mamba_state_copy_func(
|
||||||
|
cls,
|
||||||
|
) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
|
||||||
|
return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func()
|
||||||
|
|
||||||
def compute_logits(
|
def compute_logits(
|
||||||
self,
|
self,
|
||||||
hidden_states: torch.Tensor,
|
hidden_states: torch.Tensor,
|
||||||
|
|||||||
@@ -195,6 +195,8 @@ _TEXT_GENERATION_MODELS = {
|
|||||||
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
|
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
|
||||||
"Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
|
"Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
|
||||||
"Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
|
"Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
|
||||||
|
"Qwen3_5ForCausalLM": ("qwen3_5", "Qwen3_5ForCausalLM"),
|
||||||
|
"Qwen3_5MoeForCausalLM": ("qwen3_5", "Qwen3_5MoeForCausalLM"),
|
||||||
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
|
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
|
||||||
"SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
|
"SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
|
||||||
"SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
|
"SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
|
||||||
|
|||||||
@@ -124,7 +124,9 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
|||||||
qwen3_asr="Qwen3ASRConfig",
|
qwen3_asr="Qwen3ASRConfig",
|
||||||
qwen3_next="Qwen3NextConfig",
|
qwen3_next="Qwen3NextConfig",
|
||||||
qwen3_5="Qwen3_5Config",
|
qwen3_5="Qwen3_5Config",
|
||||||
|
qwen3_5_text="Qwen3_5TextConfig",
|
||||||
qwen3_5_moe="Qwen3_5MoeConfig",
|
qwen3_5_moe="Qwen3_5MoeConfig",
|
||||||
|
qwen3_5_moe_text="Qwen3_5MoeTextConfig",
|
||||||
laguna="LagunaConfig",
|
laguna="LagunaConfig",
|
||||||
lfm2_moe="Lfm2MoeConfig",
|
lfm2_moe="Lfm2MoeConfig",
|
||||||
**{"unlimited-ocr": "UnlimitedOCRConfig"},
|
**{"unlimited-ocr": "UnlimitedOCRConfig"},
|
||||||
|
|||||||
Reference in New Issue
Block a user