[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
+7 -597
View File
@@ -42,7 +42,6 @@ else:
SequencePoolingType = Any
TokenPoolingType = Any
from .interfaces import (
has_inner_state,
has_noops,
@@ -70,639 +69,50 @@ from .interfaces_base import (
logger = init_logger(__name__)
_TEXT_GENERATION_MODELS = {
# [Decoder-only]
"AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
"AliceAIForCausalLM": ("alice_ai", "AliceAIForCausalLM"),
"ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
"AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
"BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
"BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
"BailingMoeV2_5ForCausalLM": ("bailing_moe_linear", "BailingMoeV25ForCausalLM"),
"BailingMoeV3ForCausalLM": ("bailing_moe_v3", "BailingMoeV3ForCausalLM"),
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
"ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
"CohereForCausalLM": ("commandr", "CohereForCausalLM"),
"Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
"Cohere2MoeForCausalLM": ("cohere2_moe", "Cohere2MoeForCausalLM"),
"DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
"DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
"DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
"DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
"DeepseekV32ForCausalLM": (
"vllm.models.deepseek_v32",
"DeepseekV32ForCausalLM",
),
"DeepseekV4ForCausalLM": ("vllm.models.deepseek_v4", "DeepseekV4ForCausalLM"),
"Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
"Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
"Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
"ExaoneMoeForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
"ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
"FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
"FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
"GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
"Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
"Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
"Rnj1ForCausalLM": ("rnj1", "Rnj1ForCausalLM"),
"Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
"Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
"Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
"Qwen4ExpForCausalLM": (
"vllm.models.qwen4_exp",
"Qwen4ExpForCausalLM",
),
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
"GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"),
"Glm5NextForCausalLM": ("vllm.models.glm5next", "Glm5NextForCausalLM"),
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
"GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
"GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
"GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
"GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
"GraniteMoeSWAForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
"HYV4ForCausalLM": ("vllm.models.hy_v4", "HYV4ForCausalLM"),
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
"InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
"IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
"IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
"K2HorizonForCausalLM": ("k2_horizon", "K2HorizonForCausalLM"),
"KimiLinearForCausalLM": (
"vllm.models.kimi_k3",
"KimiLinearForCausalLM",
),
"Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
"Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
"LagunaForCausalLM": ("laguna", "LagunaForCausalLM"),
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
"Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),
# For decapoda-research/llama-*
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
"LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
"LongcatFlashNgramForCausalLM": (
"longcat_flash_ngram",
"LongcatFlashNgramForCausalLM",
),
"MambaForCausalLM": ("mamba", "MambaForCausalLM"),
"Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
"MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
"MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
"MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
"MiniMaxM3SparseForCausalLM": (
"vllm.models.minimax_m3",
"MiniMaxM3SparseForCausalLM",
),
"InklingForCausalLM": ("vllm.models.inkling", "InklingForCausalLM"),
"InklingForConditionalGeneration": (
"vllm.models.inkling",
"InklingForConditionalGeneration",
),
"Ministral3ForCausalLM": ("mistral", "MistralForCausalLM"),
"MistralForCausalLM": ("mistral", "MistralForCausalLM"),
"MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
"MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
"MiMoV2FlashForCausalLM": ("mimo_v2", "MiMoV2FlashForCausalLM"),
"MiMoV2ForCausalLM": ("mimo_v2", "MiMoV2ForCausalLM"),
"NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
"NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
"NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
"MuseGlimmerForCausalLM": ("muse_glimmer", "MuseGlimmerForCausalLM"),
"OlmoHybridForCausalLM": ("olmo_hybrid", "OlmoHybridForCausalLM"),
"OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
"PanguEmbeddedForCausalLM": ("openpangu", "PanguEmbeddedForCausalLM"),
"PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
"PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
"Param2MoEForCausalLM": ("param2moe", "Param2MoEForCausalLM"),
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
"Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
"Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
"Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
"Qwen3_5ForCausalLM": ("qwen3_5", "Qwen3_5ForCausalLM"),
"Qwen3_5MoeForCausalLM": ("qwen3_5", "Qwen3_5MoeForCausalLM"),
"SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
"SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
"Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
"Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
"Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
"SolarForCausalLM": ("solar", "SolarForCausalLM"),
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
"TeleChat3ForCausalLM": ("llama", "LlamaForCausalLM"),
"TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
"Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
}
_EMBEDDING_MODELS = {
# [Text-only]
"BertModel": ("bert", "BertEmbeddingModel"),
"BertForMaskedLM": ("bert", "BertForMaskedLM"),
"BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
"BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"),
"DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
"DeepseekV3BidirectionalModel": ("deepseek_v2", "DeepseekV3ForCausalLM"),
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
"Gemma3TextModel": ("gemma3", "Gemma3Model"),
"GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
"GteNewModel": ("bert_with_rope", "GteNewModel"),
"JinaEmbeddingsV5Model": ("jina", "JinaEmbeddingsV5Model"),
"LlamaBidirectionalModel": ("llama", "LlamaBidirectionalModel"),
"LlamaModel": ("llama", "LlamaForCausalLM"),
"Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
**{
# Multiple models share the same architecture, so we include them all
k: (mod, arch)
for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
if arch == "LlamaForCausalLM"
},
"MistralModel": ("llama", "LlamaForCausalLM"),
"ModernBertModel": ("modernbert", "ModernBertModel"),
"NomicBertModel": ("bert_with_rope", "NomicBertModel"),
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
"Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
"RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
"RobertaModel": ("roberta", "RobertaEmbeddingModel"),
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
"VoyageQwen3BidirectionalEmbedModel": (
"voyage",
"VoyageQwen3BidirectionalEmbedModel",
),
"XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
# [Multimodal]
"CLIPModel": ("clip", "CLIPEmbeddingModel"),
"ColPaliForRetrieval": ("colpali", "ColPaliModel"),
"EmbeddingGemma2Model": ("embedding_gemma2", "EmbeddingGemma2Model"),
"LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
"LlavaNextForConditionalGeneration": (
"llava_next",
"LlavaNextForConditionalGeneration",
),
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
"SiglipModel": ("siglip", "SiglipEmbeddingModel"),
# Technically Terratorch models work on images, both in
# input and output. I am adding it here because it piggy-backs on embedding
# models for the time being.
"Terratorch": ("terratorch", "Terratorch"),
}
_LATE_INTERACTION_MODELS = {
# [Text-only]
"HF_ColBERT": ("colbert", "ColBERTModel"),
"ColBERTModernBertModel": ("colbert", "ColBERTModernBertModel"),
"ColBERTJinaRobertaModel": ("colbert", "ColBERTJinaRobertaModel"),
"ColBERTLfm2Model": ("colbert", "ColBERTLfm2Model"),
"JinaForRanking": ("jina", "JinaForRanking"),
# [Multimodal]
"ColModernVBertForRetrieval": ("colmodernvbert", "ColModernVBertForRetrieval"),
"ColPaliForRetrieval": ("colpali", "ColPaliModel"),
"ColQwen3": ("colqwen3", "ColQwen3Model"),
"OpsColQwen3Model": ("colqwen3", "ColQwen3Model"),
"ColQwen3_5": ("colqwen3_5", "ColQwen3_5Model"),
"Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"),
}
_LATE_INTERACTION_MODELS = {}
_REWARD_MODELS = {
"InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
"Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
"Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
}
_REWARD_MODELS = {}
_TOKEN_CLASSIFICATION_MODELS = {
"BertForTokenClassification": ("bert", "BertForTokenClassification"),
"ClefFlashForJointSchema": ("clef_flash", "ClefFlashForJointSchema"),
"ClefOmniForJointSchema": ("clef_omni", "ClefOmniForJointSchema"),
"ModernBertForTokenClassification": (
"modernbert",
"ModernBertForTokenClassification",
),
"OpenAIPrivacyFilterForTokenClassification": (
"openai_privacy_filter",
"OpenAIPrivacyFilterForTokenClassification",
),
"Qwen3ASRForcedAlignerForTokenClassification": (
"qwen3_asr_forced_aligner",
"Qwen3ASRForcedAlignerForTokenClassification",
),
"RobertaForTokenClassification": ("roberta", "RobertaForTokenClassification"),
"XLMRobertaForTokenClassification": (
"roberta",
"RobertaForTokenClassification",
),
}
_TOKEN_CLASSIFICATION_MODELS = {}
_SEQUENCE_CLASSIFICATION_MODELS = {
"BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
"GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
"GteNewForSequenceClassification": (
"bert_with_rope",
"GteNewForSequenceClassification",
),
"JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),
"LlamaBidirectionalForSequenceClassification": (
"llama",
"LlamaBidirectionalForSequenceClassification",
),
"ModernBertForSequenceClassification": (
"modernbert",
"ModernBertForSequenceClassification",
),
"RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
"XLMRobertaForSequenceClassification": (
"roberta",
"RobertaForSequenceClassification",
),
# [Multimodal]
"D3ForSequenceClassification": ("d3", "D3ForSequenceClassification"),
"JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
"LlamaNemotronVLForSequenceClassification": (
"nemotron_vl",
"LlamaNemotronVLForSequenceClassification",
),
}
_SEQUENCE_CLASSIFICATION_MODELS = {}
_MULTIMODAL_MODELS = {
# [Decoder-only]
"AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
"AudioFlamingo3ForConditionalGeneration": (
"audioflamingo3",
"AudioFlamingo3ForConditionalGeneration",
),
"BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
"BailingMoeV3VLForConditionalGeneration": (
"bailing_moe_v3_vl",
"BailingMoeV3VLForConditionalGeneration",
),
"BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
"Cohere2VisionForConditionalGeneration": (
"cohere2_vision",
"Cohere2VisionForConditionalGeneration",
),
"CohereCompassForConditionalGeneration": (
"cohere_compass",
"CohereCompassForConditionalGeneration",
),
"Cosmos3ForConditionalGeneration": ("cosmos3", "Cosmos3ForConditionalGeneration"),
"Cosmos3EdgeForConditionalGeneration": (
"cosmos3_edge",
"Cosmos3EdgeForConditionalGeneration",
),
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
"DeepseekV4ForConditionalGeneration": (
"vllm.models.deepseek_v4",
"DeepseekV4ForConditionalGeneration",
),
"DeepseekV41ForCausalLM": (
"vllm.models.deepseek_v41",
"DeepseekV41ForCausalLM",
),
"Dots3NoteForCausalLM": (
"vllm.models.dots3_note",
"Dots3NoteForCausalLM",
),
"UnlimitedOCRForCausalLM": ("unlimited_ocr", "UnlimitedOCRForCausalLM"),
"DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
"Eagle2_5_VLForConditionalGeneration": (
"eagle2_5_vl",
"Eagle2_5_VLForConditionalGeneration",
),
"Ernie4_5_VLMoeForConditionalGeneration": (
"ernie45_vl",
"Ernie4_5_VLMoeForConditionalGeneration",
),
"Exaone4_5_ForConditionalGeneration": (
"exaone4_5",
"Exaone4_5_ForConditionalGeneration",
), # noqa: E501
"FireRedASR2ForConditionalGeneration": (
"fireredasr2",
"FireRedASR2ForConditionalGeneration",
),
"FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),
"FunAudioChatForConditionalGeneration": (
"funaudiochat",
"FunAudioChatForConditionalGeneration",
),
"Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),
"Gemma3nForConditionalGeneration": (
"gemma3n_mm",
"Gemma3nForConditionalGeneration",
),
"DiffusionGemmaForBlockDiffusion": (
"diffusion_gemma",
"DiffusionGemmaForConditionalGeneration",
),
"Gemma4ForConditionalGeneration": ("gemma4_mm", "Gemma4ForConditionalGeneration"),
"Gemma4UnifiedForConditionalGeneration": (
"gemma4_unified",
"Gemma4UnifiedForConditionalGeneration",
),
"GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
"GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
"Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
"Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
"GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"),
"Glm5NextForConditionalGeneration": (
"vllm.models.glm5next",
"Glm5NextForConditionalGeneration",
),
"GraniteSpeechForConditionalGeneration": (
"granite_speech",
"GraniteSpeechForConditionalGeneration",
),
"GraniteSpeechPlusForConditionalGeneration": (
"granite_speech_plus",
"GraniteSpeechPlusForConditionalGeneration",
),
"Granite4VisionForConditionalGeneration": (
"granite4_vision",
"Granite4VisionForConditionalGeneration",
),
"H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
"InternVLChatModel": ("internvl", "InternVLChatModel"),
"InternS1ForConditionalGeneration": (
"interns1",
"InternS1ForConditionalGeneration",
),
"InternVLForConditionalGeneration": (
"interns1",
"InternS1ForConditionalGeneration",
),
"InternS1ProForConditionalGeneration": (
"interns1_pro",
"InternS1ProForConditionalGeneration",
),
"InternS2MobiusForConditionalGeneration": (
"interns2_mobius",
"InternS2MobiusForConditionalGeneration",
),
"InternS2PreviewForConditionalGeneration": (
"interns2_preview",
"InternS2PreviewForConditionalGeneration",
),
"Idefics3ForConditionalGeneration": (
"idefics3",
"Idefics3ForConditionalGeneration",
),
"IsaacForConditionalGeneration": ("isaac", "IsaacForConditionalGeneration"),
"KananaVForConditionalGeneration": ("kanana_v", "KananaVForConditionalGeneration"),
"KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
"KeyeVL1_5ForConditionalGeneration": (
"keye_vl1_5",
"KeyeVL1_5ForConditionalGeneration",
),
"KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),
"KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),
"KimiK3ForConditionalGeneration": (
"vllm.models.kimi_k3",
"KimiK3ForConditionalGeneration",
),
"MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),
"MossTranscribeDiarizeForConditionalGeneration": (
"moss_transcribe_diarize",
"MossTranscribeDiarizeForConditionalGeneration",
),
"LightOnOCRForConditionalGeneration": (
"lightonocr",
"LightOnOCRForConditionalGeneration",
),
"Lfm2VlForConditionalGeneration": ("lfm2_vl", "Lfm2VLForConditionalGeneration"),
"Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),
"Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
"LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
"LlavaNextForConditionalGeneration": (
"llava_next",
"LlavaNextForConditionalGeneration",
),
"LlavaNextVideoForConditionalGeneration": (
"llava_next_video",
"LlavaNextVideoForConditionalGeneration",
),
"LlavaOnevisionForConditionalGeneration": (
"llava_onevision",
"LlavaOnevisionForConditionalGeneration",
),
"LlavaOnevision2ForConditionalGeneration": (
"llava_onevision2",
"LlavaOnevision2ForConditionalGeneration",
),
"MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
"MiMoV2OmniForCausalLM": ("mimo_v2_omni", "MiMoV2OmniForCausalLM"),
"MiniMaxM3SparseForConditionalGeneration": (
"vllm.models.minimax_m3",
"MiniMaxM3SparseForConditionalGeneration",
),
"MiniCPMO": ("minicpmo", "MiniCPMO"),
"MiniCPMV": ("minicpmv", "MiniCPMV"),
"MiniCPMV4_6ForConditionalGeneration": (
"minicpmv4_6",
"MiniCPMV4_6ForConditionalGeneration",
),
"MiniCPMV4_7ForConditionalGeneration": (
"minicpmv4_7",
"MiniCPMV4_7ForConditionalGeneration",
),
"Mistral3ForConditionalGeneration": (
"mistral3",
"Mistral3ForConditionalGeneration",
),
"MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
"Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
"Moondream3ForCausalLM": ("moondream3", "Moondream3ForCausalLM"),
"MossAudioModel": ("moss_audio", "MossAudioModel"),
"HfMoondream": ("moondream3", "Moondream3ForCausalLM"),
"NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NemotronH_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NVLM_D": ("nvlm_d", "NVLM_D_Model"),
"MuseGlimmerForConditionalGeneration": ("muse_glimmer", "MuseGlimmerForCausalLM"),
"OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),
"OpenPanguVLForConditionalGeneration": (
"openpangu_vl",
"OpenPanguVLForConditionalGeneration",
),
"OpenVLAForActionPrediction": ("openvla", "OpenVLAForActionPrediction"),
"Ovis": ("ovis", "Ovis"),
"Ovis2_5": ("ovis2_5", "Ovis2_5"),
"Ovis2_6ForCausalLM": ("ovis2_5", "Ovis2_5"),
"Ovis2_6_MoeForCausalLM": ("ovis2_5", "Ovis2_5"),
"PaddleOCRVLForConditionalGeneration": (
"paddleocr_vl",
"PaddleOCRVLForConditionalGeneration",
),
"PaliGemmaForConditionalGeneration": (
"paligemma",
"PaliGemmaForConditionalGeneration",
),
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
"Phi4ForCausalLMV": ("phi4siglip", "Phi4ForCausalLMV"),
"Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
"PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),
"QianfanOCRForConditionalGeneration": (
"qianfan_ocr",
"QianfanOCRForConditionalGeneration",
),
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
"Qwen2_5_VLForConditionalGeneration": (
"qwen2_5_vl",
"Qwen2_5_VLForConditionalGeneration",
),
"Qwen2AudioForConditionalGeneration": (
"qwen2_audio",
"Qwen2AudioForConditionalGeneration",
),
"Qwen2_5OmniModel": (
"qwen2_5_omni_thinker",
"Qwen2_5OmniThinkerForConditionalGeneration",
),
"Qwen2_5OmniForConditionalGeneration": (
"qwen2_5_omni_thinker",
"Qwen2_5OmniThinkerForConditionalGeneration",
),
"Qwen3OmniMoeForConditionalGeneration": (
"qwen3_omni_moe_thinker",
"Qwen3OmniMoeThinkerForConditionalGeneration",
),
"Qwen3ASRForConditionalGeneration": (
"qwen3_asr",
"Qwen3ASRForConditionalGeneration",
),
"Qwen3ASRRealtimeGeneration": ("qwen3_asr_realtime", "Qwen3ASRRealtimeGeneration"),
"Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),
"Qwen3VLMoeForConditionalGeneration": (
"qwen3_vl_moe",
"Qwen3VLMoeForConditionalGeneration",
),
"Qwen3_5ForConditionalGeneration": ("qwen3_5", "Qwen3_5ForConditionalGeneration"),
"Qwen3_5MoeForConditionalGeneration": (
"qwen3_5",
"Qwen3_5MoeForConditionalGeneration",
),
"Qwen4ExpForConditionalGeneration": (
"vllm.models.qwen4_exp",
"Qwen4ExpForConditionalGeneration",
),
"RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
"SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
"SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
"StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
"Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
"Step3p7ForConditionalGeneration": ("step3p7", "Step3p7ForConditionalGeneration"),
"UltravoxModel": ("ultravox", "UltravoxModel"),
"VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),
"VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),
# [Encoder-decoder]
"CohereAsrForConditionalGeneration": (
"cohere_asr",
"CohereAsrForConditionalGeneration",
),
"NemotronParseForConditionalGeneration": (
"nemotron_parse",
"NemotronParseForConditionalGeneration",
),
"WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),
}
_SPECULATIVE_DECODING_MODELS = {
"ExtractHiddenStatesModel": ("extract_hidden_states", "ExtractHiddenStatesModel"),
"MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
"MiMoV2MTPModel": ("mimo_v2_mtp", "MiMoV2MTP"),
"MiMoV2OmniMTPModel": ("mimo_v2_mtp", "MiMoV2OmniMTP"),
"EagleCohereForCausalLM": ("cohere_eagle", "EagleCohereForCausalLM"),
"EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"LiLiCorrDraftModel": ("lilicorr", "LiLiCorrForCausalLM"),
"DFlash2DraftModel": ("qwen3_dflash2", "DFlash2Qwen3ForCausalLM"),
# Muse Glimmer's DFlash draft head, reusing the generic qwen3_dflash
# implementation. EAGLEConfig rewrites a dflash draft's architecture to
# DFlash{arch} unless it already starts or ends with "DFlash" (see
# transformers_utils/configs/eagle.py), so the name the registry is asked
# for is DFlashMuseGlimmerAssistantModel -- same convention as
# DFlashLagunaForCausalLM. The bare name is kept as a defensive alias.
"MuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"DFlashMuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
"DSparkV41DraftModel": (
"vllm.models.deepseek_v41",
"DSparkDeepseekV4ForCausalLM",
),
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
"Qwen3OmniDSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
"K3DSparkModel": (
"vllm.models.kimi_k3.nvidia.dspark_mla",
"K3DSparkForCausalLM",
),
"DFlashLagunaForCausalLM": ("laguna_dflash", "DFlashLagunaForCausalLM"),
"Gemma4DSparkModel": ("gemma4_dspark", "Gemma4DSparkForCausalLM"),
"PEagleDraftModel": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"PeagleLlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"Eagle3MiniMaxM2ForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
"Eagle3Qwen3ForCausalLM": ("qwen3_eagle3", "Eagle3Qwen3ForCausalLM"),
"PeagleQwen3ForCausalLM": ("qwen3_eagle3", "Eagle3Qwen3ForCausalLM"),
"EagleMistralForCausalLM": ("mistral_eagle", "EagleMistralForCausalLM"),
"EagleMistralLarge3ForCausalLM": (
"mistral_large_3_eagle",
"EagleMistralLarge3ForCausalLM",
),
"Eagle3DeepseekV2ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
"Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
"DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"),
"Dots3NoteMTPModel": ("vllm.models.dots3_note", "Dots3NoteMTP"),
"DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"),
"BailingMoeV3MTPModel": ("bailing_moe_v3_mtp", "BailingMoeV3MTPModel"),
"MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"),
"BailingMoeV25MTPModel": ("bailing_moe_mtp", "BailingMoeV25MTPModel"),
"InklingMTPModel": ("vllm.models.inkling", "InklingMTP"),
"Gemma4MTPModel": ("gemma4_mtp", "Gemma4MTP"),
"ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
"ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
"Exaone4_5_MTP": ("exaone4_5_mtp", "Exaone4_5_MTP"),
"NemotronHMTPModel": ("nemotron_h_mtp", "NemotronHMTP"),
"LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
"Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
"Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
"GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
"Glm5NextMTPModel": ("vllm.models.glm5next", "Glm5NextMTP"),
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
"Qwen4ExpMTP": ("vllm.models.qwen4_exp", "Qwen4ExpMTP"),
"Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
"InternS2MobiusMTP": ("interns2_mobius", "InternS2MobiusMTP"),
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
"HYV4MTPModel": ("vllm.models.hy_v4", "HYV4MTP"),
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
}
_TRANSFORMERS_SUPPORTED_MODELS = {