[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -42,7 +42,6 @@ else:
|
||||
SequencePoolingType = Any
|
||||
TokenPoolingType = Any
|
||||
|
||||
|
||||
from .interfaces import (
|
||||
has_inner_state,
|
||||
has_noops,
|
||||
@@ -70,639 +69,50 @@ from .interfaces_base import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_TEXT_GENERATION_MODELS = {
|
||||
# [Decoder-only]
|
||||
"AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
|
||||
"AliceAIForCausalLM": ("alice_ai", "AliceAIForCausalLM"),
|
||||
"ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
|
||||
"AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
|
||||
"BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
|
||||
"BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
|
||||
"BailingMoeV2_5ForCausalLM": ("bailing_moe_linear", "BailingMoeV25ForCausalLM"),
|
||||
"BailingMoeV3ForCausalLM": ("bailing_moe_v3", "BailingMoeV3ForCausalLM"),
|
||||
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
|
||||
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
|
||||
"ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
|
||||
"CohereForCausalLM": ("commandr", "CohereForCausalLM"),
|
||||
"Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
|
||||
"Cohere2MoeForCausalLM": ("cohere2_moe", "Cohere2MoeForCausalLM"),
|
||||
"DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
|
||||
"DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
|
||||
"DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
|
||||
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
|
||||
"DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
|
||||
"DeepseekV32ForCausalLM": (
|
||||
"vllm.models.deepseek_v32",
|
||||
"DeepseekV32ForCausalLM",
|
||||
),
|
||||
"DeepseekV4ForCausalLM": ("vllm.models.deepseek_v4", "DeepseekV4ForCausalLM"),
|
||||
"Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
|
||||
"Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
|
||||
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
|
||||
"Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
|
||||
"ExaoneMoeForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
|
||||
"ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
|
||||
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
|
||||
"FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
|
||||
"FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
|
||||
"GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
|
||||
"Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
|
||||
"Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"),
|
||||
"Rnj1ForCausalLM": ("rnj1", "Rnj1ForCausalLM"),
|
||||
"Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
|
||||
"Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
|
||||
"Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
|
||||
"Qwen4ExpForCausalLM": (
|
||||
"vllm.models.qwen4_exp",
|
||||
"Qwen4ExpForCausalLM",
|
||||
),
|
||||
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
|
||||
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
|
||||
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
|
||||
"GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"),
|
||||
"Glm5NextForCausalLM": ("vllm.models.glm5next", "Glm5NextForCausalLM"),
|
||||
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
|
||||
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
|
||||
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
|
||||
"GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
|
||||
"GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
|
||||
"GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
|
||||
"GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
|
||||
"GraniteMoeSWAForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
|
||||
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
|
||||
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
|
||||
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
|
||||
"HYV4ForCausalLM": ("vllm.models.hy_v4", "HYV4ForCausalLM"),
|
||||
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
|
||||
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
|
||||
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
|
||||
"InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
|
||||
"IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
|
||||
"IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
|
||||
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
|
||||
"K2HorizonForCausalLM": ("k2_horizon", "K2HorizonForCausalLM"),
|
||||
"KimiLinearForCausalLM": (
|
||||
"vllm.models.kimi_k3",
|
||||
"KimiLinearForCausalLM",
|
||||
),
|
||||
"Lfm2ForCausalLM": ("lfm2", "Lfm2ForCausalLM"),
|
||||
"Lfm2MoeForCausalLM": ("lfm2_moe", "Lfm2MoeForCausalLM"),
|
||||
"LagunaForCausalLM": ("laguna", "LagunaForCausalLM"),
|
||||
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
|
||||
"Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"),
|
||||
# For decapoda-research/llama-*
|
||||
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
|
||||
"LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
|
||||
"LongcatFlashNgramForCausalLM": (
|
||||
"longcat_flash_ngram",
|
||||
"LongcatFlashNgramForCausalLM",
|
||||
),
|
||||
"MambaForCausalLM": ("mamba", "MambaForCausalLM"),
|
||||
"Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
|
||||
"MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
|
||||
"MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
|
||||
"MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
|
||||
"MiniMaxM3SparseForCausalLM": (
|
||||
"vllm.models.minimax_m3",
|
||||
"MiniMaxM3SparseForCausalLM",
|
||||
),
|
||||
"InklingForCausalLM": ("vllm.models.inkling", "InklingForCausalLM"),
|
||||
"InklingForConditionalGeneration": (
|
||||
"vllm.models.inkling",
|
||||
"InklingForConditionalGeneration",
|
||||
),
|
||||
"Ministral3ForCausalLM": ("mistral", "MistralForCausalLM"),
|
||||
"MistralForCausalLM": ("mistral", "MistralForCausalLM"),
|
||||
"MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
|
||||
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
|
||||
"MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
|
||||
"MiMoV2FlashForCausalLM": ("mimo_v2", "MiMoV2FlashForCausalLM"),
|
||||
"MiMoV2ForCausalLM": ("mimo_v2", "MiMoV2ForCausalLM"),
|
||||
"NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
|
||||
"NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
|
||||
"NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
|
||||
"MuseGlimmerForCausalLM": ("muse_glimmer", "MuseGlimmerForCausalLM"),
|
||||
"OlmoHybridForCausalLM": ("olmo_hybrid", "OlmoHybridForCausalLM"),
|
||||
"OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
|
||||
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
|
||||
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
|
||||
"PanguEmbeddedForCausalLM": ("openpangu", "PanguEmbeddedForCausalLM"),
|
||||
"PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
|
||||
"PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
|
||||
"Param2MoEForCausalLM": ("param2moe", "Param2MoEForCausalLM"),
|
||||
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
|
||||
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
|
||||
"Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
|
||||
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
|
||||
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
|
||||
"Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
|
||||
"Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
|
||||
"Qwen3_5ForCausalLM": ("qwen3_5", "Qwen3_5ForCausalLM"),
|
||||
"Qwen3_5MoeForCausalLM": ("qwen3_5", "Qwen3_5MoeForCausalLM"),
|
||||
"SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
|
||||
"SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
|
||||
"Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
|
||||
"Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
|
||||
"Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
|
||||
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
|
||||
"SolarForCausalLM": ("solar", "SolarForCausalLM"),
|
||||
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
|
||||
"TeleChat3ForCausalLM": ("llama", "LlamaForCausalLM"),
|
||||
"TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
|
||||
"Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
|
||||
}
|
||||
|
||||
_EMBEDDING_MODELS = {
|
||||
# [Text-only]
|
||||
"BertModel": ("bert", "BertEmbeddingModel"),
|
||||
"BertForMaskedLM": ("bert", "BertForMaskedLM"),
|
||||
"BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
|
||||
"BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"),
|
||||
"DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
|
||||
"DeepseekV3BidirectionalModel": ("deepseek_v2", "DeepseekV3ForCausalLM"),
|
||||
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
|
||||
"Gemma3TextModel": ("gemma3", "Gemma3Model"),
|
||||
"GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
|
||||
"GteNewModel": ("bert_with_rope", "GteNewModel"),
|
||||
"JinaEmbeddingsV5Model": ("jina", "JinaEmbeddingsV5Model"),
|
||||
"LlamaBidirectionalModel": ("llama", "LlamaBidirectionalModel"),
|
||||
"LlamaModel": ("llama", "LlamaForCausalLM"),
|
||||
"Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
|
||||
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
|
||||
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
|
||||
**{
|
||||
# Multiple models share the same architecture, so we include them all
|
||||
k: (mod, arch)
|
||||
for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
|
||||
if arch == "LlamaForCausalLM"
|
||||
},
|
||||
"MistralModel": ("llama", "LlamaForCausalLM"),
|
||||
"ModernBertModel": ("modernbert", "ModernBertModel"),
|
||||
"NomicBertModel": ("bert_with_rope", "NomicBertModel"),
|
||||
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
|
||||
"Qwen2Model": ("qwen2", "Qwen2ForCausalLM"),
|
||||
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
|
||||
"RobertaForMaskedLM": ("roberta", "RobertaEmbeddingModel"),
|
||||
"RobertaModel": ("roberta", "RobertaEmbeddingModel"),
|
||||
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
|
||||
"VoyageQwen3BidirectionalEmbedModel": (
|
||||
"voyage",
|
||||
"VoyageQwen3BidirectionalEmbedModel",
|
||||
),
|
||||
"XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
|
||||
# [Multimodal]
|
||||
"CLIPModel": ("clip", "CLIPEmbeddingModel"),
|
||||
"ColPaliForRetrieval": ("colpali", "ColPaliModel"),
|
||||
"EmbeddingGemma2Model": ("embedding_gemma2", "EmbeddingGemma2Model"),
|
||||
"LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
|
||||
"LlavaNextForConditionalGeneration": (
|
||||
"llava_next",
|
||||
"LlavaNextForConditionalGeneration",
|
||||
),
|
||||
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
|
||||
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
|
||||
"SiglipModel": ("siglip", "SiglipEmbeddingModel"),
|
||||
# Technically Terratorch models work on images, both in
|
||||
# input and output. I am adding it here because it piggy-backs on embedding
|
||||
# models for the time being.
|
||||
"Terratorch": ("terratorch", "Terratorch"),
|
||||
}
|
||||
|
||||
_LATE_INTERACTION_MODELS = {
|
||||
# [Text-only]
|
||||
"HF_ColBERT": ("colbert", "ColBERTModel"),
|
||||
"ColBERTModernBertModel": ("colbert", "ColBERTModernBertModel"),
|
||||
"ColBERTJinaRobertaModel": ("colbert", "ColBERTJinaRobertaModel"),
|
||||
"ColBERTLfm2Model": ("colbert", "ColBERTLfm2Model"),
|
||||
"JinaForRanking": ("jina", "JinaForRanking"),
|
||||
# [Multimodal]
|
||||
"ColModernVBertForRetrieval": ("colmodernvbert", "ColModernVBertForRetrieval"),
|
||||
"ColPaliForRetrieval": ("colpali", "ColPaliModel"),
|
||||
"ColQwen3": ("colqwen3", "ColQwen3Model"),
|
||||
"OpsColQwen3Model": ("colqwen3", "ColQwen3Model"),
|
||||
"ColQwen3_5": ("colqwen3_5", "ColQwen3_5Model"),
|
||||
"Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"),
|
||||
}
|
||||
_LATE_INTERACTION_MODELS = {}
|
||||
|
||||
_REWARD_MODELS = {
|
||||
"InternLM2ForRewardModel": ("internlm2", "InternLM2ForRewardModel"),
|
||||
"Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
|
||||
"Qwen2ForProcessRewardModel": ("qwen2_rm", "Qwen2ForProcessRewardModel"),
|
||||
}
|
||||
_REWARD_MODELS = {}
|
||||
|
||||
_TOKEN_CLASSIFICATION_MODELS = {
|
||||
"BertForTokenClassification": ("bert", "BertForTokenClassification"),
|
||||
"ClefFlashForJointSchema": ("clef_flash", "ClefFlashForJointSchema"),
|
||||
"ClefOmniForJointSchema": ("clef_omni", "ClefOmniForJointSchema"),
|
||||
"ModernBertForTokenClassification": (
|
||||
"modernbert",
|
||||
"ModernBertForTokenClassification",
|
||||
),
|
||||
"OpenAIPrivacyFilterForTokenClassification": (
|
||||
"openai_privacy_filter",
|
||||
"OpenAIPrivacyFilterForTokenClassification",
|
||||
),
|
||||
"Qwen3ASRForcedAlignerForTokenClassification": (
|
||||
"qwen3_asr_forced_aligner",
|
||||
"Qwen3ASRForcedAlignerForTokenClassification",
|
||||
),
|
||||
"RobertaForTokenClassification": ("roberta", "RobertaForTokenClassification"),
|
||||
"XLMRobertaForTokenClassification": (
|
||||
"roberta",
|
||||
"RobertaForTokenClassification",
|
||||
),
|
||||
}
|
||||
_TOKEN_CLASSIFICATION_MODELS = {}
|
||||
|
||||
_SEQUENCE_CLASSIFICATION_MODELS = {
|
||||
"BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
|
||||
"GPT2ForSequenceClassification": ("gpt2", "GPT2ForSequenceClassification"),
|
||||
"GteNewForSequenceClassification": (
|
||||
"bert_with_rope",
|
||||
"GteNewForSequenceClassification",
|
||||
),
|
||||
"JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"),
|
||||
"LlamaBidirectionalForSequenceClassification": (
|
||||
"llama",
|
||||
"LlamaBidirectionalForSequenceClassification",
|
||||
),
|
||||
"ModernBertForSequenceClassification": (
|
||||
"modernbert",
|
||||
"ModernBertForSequenceClassification",
|
||||
),
|
||||
"RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
|
||||
"XLMRobertaForSequenceClassification": (
|
||||
"roberta",
|
||||
"RobertaForSequenceClassification",
|
||||
),
|
||||
# [Multimodal]
|
||||
"D3ForSequenceClassification": ("d3", "D3ForSequenceClassification"),
|
||||
"JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
|
||||
"LlamaNemotronVLForSequenceClassification": (
|
||||
"nemotron_vl",
|
||||
"LlamaNemotronVLForSequenceClassification",
|
||||
),
|
||||
}
|
||||
_SEQUENCE_CLASSIFICATION_MODELS = {}
|
||||
|
||||
_MULTIMODAL_MODELS = {
|
||||
# [Decoder-only]
|
||||
"AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
|
||||
"AudioFlamingo3ForConditionalGeneration": (
|
||||
"audioflamingo3",
|
||||
"AudioFlamingo3ForConditionalGeneration",
|
||||
),
|
||||
"BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
|
||||
"BailingMoeV3VLForConditionalGeneration": (
|
||||
"bailing_moe_v3_vl",
|
||||
"BailingMoeV3VLForConditionalGeneration",
|
||||
),
|
||||
"BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
|
||||
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
|
||||
"Cohere2VisionForConditionalGeneration": (
|
||||
"cohere2_vision",
|
||||
"Cohere2VisionForConditionalGeneration",
|
||||
),
|
||||
"CohereCompassForConditionalGeneration": (
|
||||
"cohere_compass",
|
||||
"CohereCompassForConditionalGeneration",
|
||||
),
|
||||
"Cosmos3ForConditionalGeneration": ("cosmos3", "Cosmos3ForConditionalGeneration"),
|
||||
"Cosmos3EdgeForConditionalGeneration": (
|
||||
"cosmos3_edge",
|
||||
"Cosmos3EdgeForConditionalGeneration",
|
||||
),
|
||||
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
|
||||
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
|
||||
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
|
||||
"DeepseekV4ForConditionalGeneration": (
|
||||
"vllm.models.deepseek_v4",
|
||||
"DeepseekV4ForConditionalGeneration",
|
||||
),
|
||||
"DeepseekV41ForCausalLM": (
|
||||
"vllm.models.deepseek_v41",
|
||||
"DeepseekV41ForCausalLM",
|
||||
),
|
||||
"Dots3NoteForCausalLM": (
|
||||
"vllm.models.dots3_note",
|
||||
"Dots3NoteForCausalLM",
|
||||
),
|
||||
"UnlimitedOCRForCausalLM": ("unlimited_ocr", "UnlimitedOCRForCausalLM"),
|
||||
"DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
|
||||
"Eagle2_5_VLForConditionalGeneration": (
|
||||
"eagle2_5_vl",
|
||||
"Eagle2_5_VLForConditionalGeneration",
|
||||
),
|
||||
"Ernie4_5_VLMoeForConditionalGeneration": (
|
||||
"ernie45_vl",
|
||||
"Ernie4_5_VLMoeForConditionalGeneration",
|
||||
),
|
||||
"Exaone4_5_ForConditionalGeneration": (
|
||||
"exaone4_5",
|
||||
"Exaone4_5_ForConditionalGeneration",
|
||||
), # noqa: E501
|
||||
"FireRedASR2ForConditionalGeneration": (
|
||||
"fireredasr2",
|
||||
"FireRedASR2ForConditionalGeneration",
|
||||
),
|
||||
"FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),
|
||||
"FunAudioChatForConditionalGeneration": (
|
||||
"funaudiochat",
|
||||
"FunAudioChatForConditionalGeneration",
|
||||
),
|
||||
"Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"),
|
||||
"Gemma3nForConditionalGeneration": (
|
||||
"gemma3n_mm",
|
||||
"Gemma3nForConditionalGeneration",
|
||||
),
|
||||
"DiffusionGemmaForBlockDiffusion": (
|
||||
"diffusion_gemma",
|
||||
"DiffusionGemmaForConditionalGeneration",
|
||||
),
|
||||
"Gemma4ForConditionalGeneration": ("gemma4_mm", "Gemma4ForConditionalGeneration"),
|
||||
"Gemma4UnifiedForConditionalGeneration": (
|
||||
"gemma4_unified",
|
||||
"Gemma4UnifiedForConditionalGeneration",
|
||||
),
|
||||
"GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
|
||||
"GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
|
||||
"Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
|
||||
"Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
|
||||
"GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"),
|
||||
"Glm5NextForConditionalGeneration": (
|
||||
"vllm.models.glm5next",
|
||||
"Glm5NextForConditionalGeneration",
|
||||
),
|
||||
"GraniteSpeechForConditionalGeneration": (
|
||||
"granite_speech",
|
||||
"GraniteSpeechForConditionalGeneration",
|
||||
),
|
||||
"GraniteSpeechPlusForConditionalGeneration": (
|
||||
"granite_speech_plus",
|
||||
"GraniteSpeechPlusForConditionalGeneration",
|
||||
),
|
||||
"Granite4VisionForConditionalGeneration": (
|
||||
"granite4_vision",
|
||||
"Granite4VisionForConditionalGeneration",
|
||||
),
|
||||
"H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
|
||||
"InternVLChatModel": ("internvl", "InternVLChatModel"),
|
||||
"InternS1ForConditionalGeneration": (
|
||||
"interns1",
|
||||
"InternS1ForConditionalGeneration",
|
||||
),
|
||||
"InternVLForConditionalGeneration": (
|
||||
"interns1",
|
||||
"InternS1ForConditionalGeneration",
|
||||
),
|
||||
"InternS1ProForConditionalGeneration": (
|
||||
"interns1_pro",
|
||||
"InternS1ProForConditionalGeneration",
|
||||
),
|
||||
"InternS2MobiusForConditionalGeneration": (
|
||||
"interns2_mobius",
|
||||
"InternS2MobiusForConditionalGeneration",
|
||||
),
|
||||
"InternS2PreviewForConditionalGeneration": (
|
||||
"interns2_preview",
|
||||
"InternS2PreviewForConditionalGeneration",
|
||||
),
|
||||
"Idefics3ForConditionalGeneration": (
|
||||
"idefics3",
|
||||
"Idefics3ForConditionalGeneration",
|
||||
),
|
||||
"IsaacForConditionalGeneration": ("isaac", "IsaacForConditionalGeneration"),
|
||||
"KananaVForConditionalGeneration": ("kanana_v", "KananaVForConditionalGeneration"),
|
||||
"KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
|
||||
"KeyeVL1_5ForConditionalGeneration": (
|
||||
"keye_vl1_5",
|
||||
"KeyeVL1_5ForConditionalGeneration",
|
||||
),
|
||||
"KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"),
|
||||
"KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"),
|
||||
"KimiK3ForConditionalGeneration": (
|
||||
"vllm.models.kimi_k3",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
),
|
||||
"MoonshotKimiaForCausalLM": ("kimi_audio", "KimiAudioForConditionalGeneration"),
|
||||
"MossTranscribeDiarizeForConditionalGeneration": (
|
||||
"moss_transcribe_diarize",
|
||||
"MossTranscribeDiarizeForConditionalGeneration",
|
||||
),
|
||||
"LightOnOCRForConditionalGeneration": (
|
||||
"lightonocr",
|
||||
"LightOnOCRForConditionalGeneration",
|
||||
),
|
||||
"Lfm2VlForConditionalGeneration": ("lfm2_vl", "Lfm2VLForConditionalGeneration"),
|
||||
"Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"),
|
||||
"Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
|
||||
"LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
|
||||
"LlavaNextForConditionalGeneration": (
|
||||
"llava_next",
|
||||
"LlavaNextForConditionalGeneration",
|
||||
),
|
||||
"LlavaNextVideoForConditionalGeneration": (
|
||||
"llava_next_video",
|
||||
"LlavaNextVideoForConditionalGeneration",
|
||||
),
|
||||
"LlavaOnevisionForConditionalGeneration": (
|
||||
"llava_onevision",
|
||||
"LlavaOnevisionForConditionalGeneration",
|
||||
),
|
||||
"LlavaOnevision2ForConditionalGeneration": (
|
||||
"llava_onevision2",
|
||||
"LlavaOnevision2ForConditionalGeneration",
|
||||
),
|
||||
"MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
|
||||
"MiMoV2OmniForCausalLM": ("mimo_v2_omni", "MiMoV2OmniForCausalLM"),
|
||||
"MiniMaxM3SparseForConditionalGeneration": (
|
||||
"vllm.models.minimax_m3",
|
||||
"MiniMaxM3SparseForConditionalGeneration",
|
||||
),
|
||||
"MiniCPMO": ("minicpmo", "MiniCPMO"),
|
||||
"MiniCPMV": ("minicpmv", "MiniCPMV"),
|
||||
"MiniCPMV4_6ForConditionalGeneration": (
|
||||
"minicpmv4_6",
|
||||
"MiniCPMV4_6ForConditionalGeneration",
|
||||
),
|
||||
"MiniCPMV4_7ForConditionalGeneration": (
|
||||
"minicpmv4_7",
|
||||
"MiniCPMV4_7ForConditionalGeneration",
|
||||
),
|
||||
"Mistral3ForConditionalGeneration": (
|
||||
"mistral3",
|
||||
"Mistral3ForConditionalGeneration",
|
||||
),
|
||||
"MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
|
||||
"Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
|
||||
"Moondream3ForCausalLM": ("moondream3", "Moondream3ForCausalLM"),
|
||||
"MossAudioModel": ("moss_audio", "MossAudioModel"),
|
||||
"HfMoondream": ("moondream3", "Moondream3ForCausalLM"),
|
||||
"NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
|
||||
"NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
|
||||
"NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
|
||||
"NemotronH_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
|
||||
"NVLM_D": ("nvlm_d", "NVLM_D_Model"),
|
||||
"MuseGlimmerForConditionalGeneration": ("muse_glimmer", "MuseGlimmerForCausalLM"),
|
||||
"OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),
|
||||
"OpenPanguVLForConditionalGeneration": (
|
||||
"openpangu_vl",
|
||||
"OpenPanguVLForConditionalGeneration",
|
||||
),
|
||||
"OpenVLAForActionPrediction": ("openvla", "OpenVLAForActionPrediction"),
|
||||
"Ovis": ("ovis", "Ovis"),
|
||||
"Ovis2_5": ("ovis2_5", "Ovis2_5"),
|
||||
"Ovis2_6ForCausalLM": ("ovis2_5", "Ovis2_5"),
|
||||
"Ovis2_6_MoeForCausalLM": ("ovis2_5", "Ovis2_5"),
|
||||
"PaddleOCRVLForConditionalGeneration": (
|
||||
"paddleocr_vl",
|
||||
"PaddleOCRVLForConditionalGeneration",
|
||||
),
|
||||
"PaliGemmaForConditionalGeneration": (
|
||||
"paligemma",
|
||||
"PaliGemmaForConditionalGeneration",
|
||||
),
|
||||
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
|
||||
"Phi4ForCausalLMV": ("phi4siglip", "Phi4ForCausalLMV"),
|
||||
"Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
|
||||
"PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),
|
||||
"QianfanOCRForConditionalGeneration": (
|
||||
"qianfan_ocr",
|
||||
"QianfanOCRForConditionalGeneration",
|
||||
),
|
||||
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
|
||||
"Qwen2_5_VLForConditionalGeneration": (
|
||||
"qwen2_5_vl",
|
||||
"Qwen2_5_VLForConditionalGeneration",
|
||||
),
|
||||
"Qwen2AudioForConditionalGeneration": (
|
||||
"qwen2_audio",
|
||||
"Qwen2AudioForConditionalGeneration",
|
||||
),
|
||||
"Qwen2_5OmniModel": (
|
||||
"qwen2_5_omni_thinker",
|
||||
"Qwen2_5OmniThinkerForConditionalGeneration",
|
||||
),
|
||||
"Qwen2_5OmniForConditionalGeneration": (
|
||||
"qwen2_5_omni_thinker",
|
||||
"Qwen2_5OmniThinkerForConditionalGeneration",
|
||||
),
|
||||
"Qwen3OmniMoeForConditionalGeneration": (
|
||||
"qwen3_omni_moe_thinker",
|
||||
"Qwen3OmniMoeThinkerForConditionalGeneration",
|
||||
),
|
||||
"Qwen3ASRForConditionalGeneration": (
|
||||
"qwen3_asr",
|
||||
"Qwen3ASRForConditionalGeneration",
|
||||
),
|
||||
"Qwen3ASRRealtimeGeneration": ("qwen3_asr_realtime", "Qwen3ASRRealtimeGeneration"),
|
||||
"Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"),
|
||||
"Qwen3VLMoeForConditionalGeneration": (
|
||||
"qwen3_vl_moe",
|
||||
"Qwen3VLMoeForConditionalGeneration",
|
||||
),
|
||||
"Qwen3_5ForConditionalGeneration": ("qwen3_5", "Qwen3_5ForConditionalGeneration"),
|
||||
"Qwen3_5MoeForConditionalGeneration": (
|
||||
"qwen3_5",
|
||||
"Qwen3_5MoeForConditionalGeneration",
|
||||
),
|
||||
"Qwen4ExpForConditionalGeneration": (
|
||||
"vllm.models.qwen4_exp",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
),
|
||||
"RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
|
||||
"SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
|
||||
"SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
|
||||
"StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
|
||||
"Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),
|
||||
"Step3p7ForConditionalGeneration": ("step3p7", "Step3p7ForConditionalGeneration"),
|
||||
"UltravoxModel": ("ultravox", "UltravoxModel"),
|
||||
"VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),
|
||||
"VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),
|
||||
# [Encoder-decoder]
|
||||
"CohereAsrForConditionalGeneration": (
|
||||
"cohere_asr",
|
||||
"CohereAsrForConditionalGeneration",
|
||||
),
|
||||
"NemotronParseForConditionalGeneration": (
|
||||
"nemotron_parse",
|
||||
"NemotronParseForConditionalGeneration",
|
||||
),
|
||||
"WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"),
|
||||
}
|
||||
|
||||
_SPECULATIVE_DECODING_MODELS = {
|
||||
"ExtractHiddenStatesModel": ("extract_hidden_states", "ExtractHiddenStatesModel"),
|
||||
"MiMoMTPModel": ("mimo_mtp", "MiMoMTP"),
|
||||
"MiMoV2MTPModel": ("mimo_v2_mtp", "MiMoV2MTP"),
|
||||
"MiMoV2OmniMTPModel": ("mimo_v2_mtp", "MiMoV2OmniMTP"),
|
||||
"EagleCohereForCausalLM": ("cohere_eagle", "EagleCohereForCausalLM"),
|
||||
"EagleLlamaForCausalLM": ("llama_eagle", "EagleLlamaForCausalLM"),
|
||||
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
|
||||
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
|
||||
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"LiLiCorrDraftModel": ("lilicorr", "LiLiCorrForCausalLM"),
|
||||
"DFlash2DraftModel": ("qwen3_dflash2", "DFlash2Qwen3ForCausalLM"),
|
||||
# Muse Glimmer's DFlash draft head, reusing the generic qwen3_dflash
|
||||
# implementation. EAGLEConfig rewrites a dflash draft's architecture to
|
||||
# DFlash{arch} unless it already starts or ends with "DFlash" (see
|
||||
# transformers_utils/configs/eagle.py), so the name the registry is asked
|
||||
# for is DFlashMuseGlimmerAssistantModel -- same convention as
|
||||
# DFlashLagunaForCausalLM. The bare name is kept as a defensive alias.
|
||||
"MuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DFlashMuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
|
||||
"DSparkV41DraftModel": (
|
||||
"vllm.models.deepseek_v41",
|
||||
"DSparkDeepseekV4ForCausalLM",
|
||||
),
|
||||
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
|
||||
"Qwen3OmniDSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
|
||||
"K3DSparkModel": (
|
||||
"vllm.models.kimi_k3.nvidia.dspark_mla",
|
||||
"K3DSparkForCausalLM",
|
||||
),
|
||||
"DFlashLagunaForCausalLM": ("laguna_dflash", "DFlashLagunaForCausalLM"),
|
||||
"Gemma4DSparkModel": ("gemma4_dspark", "Gemma4DSparkForCausalLM"),
|
||||
"PEagleDraftModel": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"PeagleLlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"Eagle3MiniMaxM2ForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"Eagle3Qwen2_5vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"Eagle3Qwen3vlForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
|
||||
"Eagle3Qwen3ForCausalLM": ("qwen3_eagle3", "Eagle3Qwen3ForCausalLM"),
|
||||
"PeagleQwen3ForCausalLM": ("qwen3_eagle3", "Eagle3Qwen3ForCausalLM"),
|
||||
"EagleMistralForCausalLM": ("mistral_eagle", "EagleMistralForCausalLM"),
|
||||
"EagleMistralLarge3ForCausalLM": (
|
||||
"mistral_large_3_eagle",
|
||||
"EagleMistralLarge3ForCausalLM",
|
||||
),
|
||||
"Eagle3DeepseekV2ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
|
||||
"Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
|
||||
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
|
||||
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
|
||||
"DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"),
|
||||
"Dots3NoteMTPModel": ("vllm.models.dots3_note", "Dots3NoteMTP"),
|
||||
"DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"),
|
||||
"BailingMoeV3MTPModel": ("bailing_moe_v3_mtp", "BailingMoeV3MTPModel"),
|
||||
"MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"),
|
||||
"BailingMoeV25MTPModel": ("bailing_moe_mtp", "BailingMoeV25MTPModel"),
|
||||
"InklingMTPModel": ("vllm.models.inkling", "InklingMTP"),
|
||||
"Gemma4MTPModel": ("gemma4_mtp", "Gemma4MTP"),
|
||||
"ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
|
||||
"ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
|
||||
"Exaone4_5_MTP": ("exaone4_5_mtp", "Exaone4_5_MTP"),
|
||||
"NemotronHMTPModel": ("nemotron_h_mtp", "NemotronHMTP"),
|
||||
"LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
|
||||
"Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
|
||||
"Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
|
||||
"GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
|
||||
"Glm5NextMTPModel": ("vllm.models.glm5next", "Glm5NextMTP"),
|
||||
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
|
||||
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
|
||||
"Qwen4ExpMTP": ("vllm.models.qwen4_exp", "Qwen4ExpMTP"),
|
||||
"Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
|
||||
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
"InternS2MobiusMTP": ("interns2_mobius", "InternS2MobiusMTP"),
|
||||
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
|
||||
"HYV4MTPModel": ("vllm.models.hy_v4", "HYV4MTP"),
|
||||
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
|
||||
}
|
||||
|
||||
_TRANSFORMERS_SUPPORTED_MODELS = {
|
||||
|
||||
Reference in New Issue
Block a user