diff --git a/docs/contributing/model/multimodal.md b/docs/contributing/model/multimodal.md
index c59c1a80bf..951d25587a 100644
--- a/docs/contributing/model/multimodal.md
+++ b/docs/contributing/model/multimodal.md
@@ -719,7 +719,6 @@ Examples:
Examples:
-- Chameleon (appends `sep_token`): [vllm/model_executor/models/chameleon.py](../../../vllm/model_executor/models/chameleon.py)
- Molmo2 (prepends `bos_token`): [vllm/model_executor/models/molmo2.py](../../../vllm/model_executor/models/molmo2.py)
- Molmo (applies chat template which is not defined elsewhere): [vllm/model_executor/models/molmo.py](../../../vllm/model_executor/models/molmo.py)
diff --git a/docs/models/pooling_models/embed.md b/docs/models/pooling_models/embed.md
index c548ed80d2..a60bd4307a 100644
--- a/docs/models/pooling_models/embed.md
+++ b/docs/models/pooling_models/embed.md
@@ -42,7 +42,6 @@ You can compute pairwise similarity scores to build a similarity matrix using th
| `BgeM3EmbeddingModel` | BGE-M3 | `BAAI/bge-m3` | | |
| `Gemma2Model`C | Gemma 2-based | `BAAI/bge-multilingual-gemma2`, etc. | ✅︎ | ✅︎ |
| `Gemma3TextModel`C | Gemma 3-based | `google/embeddinggemma-300m`, etc. | ✅︎ | ✅︎ |
-| `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
| `GteModel` | Arctic-Embed-2.0-M | `Snowflake/snowflake-arctic-embed-m-v2.0`. | | |
| `GteNewModel` | mGTE-TRM (see note) | `Alibaba-NLP/gte-multilingual-base`, etc. | | |
| `JinaEmbeddingsV5Model`C | Qwen3-decoder or EuroBERT-encoder backbone with task-specific LoRA adapters | `jinaai/jina-embeddings-v5-text-small`, `jinaai/jina-embeddings-v5-text-nano` (see note) | ✅︎ | ✅︎ |
diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index f9c70adec7..38eebf0e01 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -358,7 +358,6 @@ th {
| `AfmoeForCausalLM` | Afmoe | TBA | ✅︎ | ✅︎ |
| `ApertusForCausalLM` | Apertus | `swiss-ai/Apertus-8B-2509`, `swiss-ai/Apertus-70B-Instruct-2509`, etc. | ✅︎ | ✅︎ |
| `ArceeForCausalLM` | Arcee (AFM) | `arcee-ai/AFM-4.5B-Base`, etc. | ✅︎ | ✅︎ |
-| `ArcticForCausalLM` | Arctic | `Snowflake/snowflake-arctic-base`, `Snowflake/snowflake-arctic-instruct`, etc. | | ✅︎ |
| `AXK1ForCausalLM` | A.X-K1 | `skt/A.X-K1`, etc. | | ✅︎ |
| `BailingMoeForCausalLM` | Ling | `inclusionAI/Ling-lite-1.5`, `inclusionAI/Ling-plus`, etc. | ✅︎ | ✅︎ |
| `BailingMoeV2ForCausalLM` | Ling | `inclusionAI/Ling-mini-2.0`, etc. | ✅︎ | ✅︎ |
@@ -382,7 +381,6 @@ th {
| `ExaoneForCausalLM` | EXAONE-3 | `LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct`, etc. | ✅︎ | ✅︎ |
| `ExaoneMoeForCausalLM` | K-EXAONE, K-EXAONE-2 | `LGAI-EXAONE/K-EXAONE-236B-A23B`, `LGAI-EXAONE/K-EXAONE-2.0-750B-A37B`, etc. | | |
| `Exaone4ForCausalLM` | EXAONE-4 | `LGAI-EXAONE/EXAONE-4.0-32B`, etc. | ✅︎ | ✅︎ |
-| `Fairseq2LlamaForCausalLM` | Llama (fairseq2 format) | `mgleize/fairseq2-dummy-Llama-3.2-1B`, etc. | ✅︎ | ✅︎ |
| `FalconForCausalLM` | Falcon | `tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc. | | ✅︎ |
| `FalconMambaForCausalLM` | FalconMamba | `tiiuae/falcon-mamba-7b`, `tiiuae/falcon-mamba-7b-instruct`, etc. | | ✅︎ |
| `FalconH1ForCausalLM` | Falcon-H1 | `tiiuae/Falcon-H1-34B-Base`, `tiiuae/Falcon-H1-34B-Instruct`, etc. | ✅︎ | ✅︎ |
@@ -407,7 +405,6 @@ th {
| `GraniteMoeSharedForCausalLM` | Granite MoE Shared | `ibm-research/moe-7b-1b-active-shared-experts` (test model) | ✅︎ | ✅︎ |
| `GraniteMoeSWAForCausalLM` | Granite MoE SWA | `ibm-granite/granite-swash-3b-a600m` | ✅︎ | ✅︎ |
| `GraniteSWAForCausalLM` | Granite SWA | `ibm-granite/granite-swash-2b` | ✅︎ | ✅︎ |
-| `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
| `HrmTextForCausalLM` | HRM-Text | `sapientinc/HRM-Text-1B`, etc. | | |
| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ |
| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ |
@@ -437,7 +434,6 @@ th {
| `MistralForCausalLM` | Ministral-3, Mistral, Mistral-Instruct | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc. | ✅︎ | ✅︎ |
| `MistralLarge3ForCausalLM` | Mistral-Large-3-675B-Base-2512, Mistral-Large-3-675B-Instruct-2512 | `mistralai/Mistral-Large-3-675B-Base-2512`, `mistralai/Mistral-Large-3-675B-Instruct-2512`, etc. | ✅︎ | ✅︎ |
| `MixtralForCausalLM` | Mixtral-8x7B, Mixtral-8x7B-Instruct | `mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, `mistral-community/Mixtral-8x22B-v0.1`, etc. | ✅︎ | ✅︎ |
-| `MPTForCausalLM` | MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter | `mosaicml/mpt-7b`, `mosaicml/mpt-7b-storywriter`, `mosaicml/mpt-30b`, etc. | | ✅︎ |
| `NemotronForCausalLM` | Nemotron-3, Nemotron-4, Minitron | `nvidia/Minitron-8B-Base`, `mgoin/Nemotron-4-340B-Base-hf-FP8`, etc. | ✅︎ | ✅︎ |
| `NemotronHForCausalLM` | Nemotron-H | `nvidia/Nemotron-H-8B-Base-8K`, `nvidia/Nemotron-H-47B-Base-8K`, `nvidia/Nemotron-H-56B-Base-8K`, etc. | ✅︎ | ✅︎ |
| `Olmo3ForCausalLM` | OLMo3 | `allenai/Olmo-3-7B-Instruct`, `allenai/Olmo-3-32B-Think`, etc. | ✅︎ | ✅︎ |
@@ -458,14 +454,12 @@ th {
| `Qwen3ForCausalLM` | Qwen3 | `Qwen/Qwen3-8B`, etc. | ✅︎ | ✅︎ |
| `Qwen3MoeForCausalLM` | Qwen3MoE | `Qwen/Qwen3-30B-A3B`, etc. | ✅︎ | ✅︎ |
| `Qwen3NextForCausalLM` | Qwen3NextMoE | `Qwen/Qwen3-Next-80B-A3B-Instruct`, etc. | ✅︎ | ✅︎ |
-| `RWForCausalLM` | Falcon RW | `tiiuae/falcon-40b`, etc. | | ✅︎ |
| `Rnj1ForCausalLM` | Rnj1 | `EssentialAI/rnj-1-instruct`, etc. | | |
| `SarvamMoEForCausalLM` | Sarvam 2 | `sarvamai/sarvam2-30b-a3b`, etc. | ✅︎ | ✅︎ |
| `SarvamMLAForCausalLM` | Sarvam 2 | `sarvamai/sarvam2-105b-a9b`, etc. | | ✅︎ |
| `SeedOssForCausalLM` | SeedOss | `ByteDance-Seed/Seed-OSS-36B-Instruct`, etc. | ✅︎ | ✅︎ |
| `SolarForCausalLM` | Solar Pro | `upstage/solar-pro-preview-instruct`, etc. | ✅︎ | ✅︎ |
| `StableLmForCausalLM` | StableLM | `stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc. | | |
-| `StableLMEpochForCausalLM` | StableLM Epoch | `stabilityai/stablelm-zephyr-3b`, etc. | | ✅︎ |
| `Step1ForCausalLM` | Step-Audio | `stepfun-ai/Step-Audio-EditX`, etc. | ✅︎ | ✅︎ |
| `Step3p5ForCausalLM` | Step-3.5-flash | `stepfun-ai/Step-3.5-Flash`, etc. | | ✅︎ |
| `TeleChat2ForCausalLM` | TeleChat2 | `Tele-AI/TeleChat2-3B`, `Tele-AI/TeleChat2-7B`, `Tele-AI/TeleChat2-35B`, etc. | ✅︎ | ✅︎ |
@@ -527,8 +521,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `BagelForConditionalGeneration` | BAGEL | T + I+ | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ |
| `BeeForConditionalGeneration` | Bee-8B | T + IE+ | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |
| `Blip2ForConditionalGeneration` | BLIP-2 | T + IE | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | ✅︎ | ✅︎ |
-| `ChameleonForConditionalGeneration` | Chameleon | T + I | `facebook/chameleon-7b`, etc. | | ✅︎ |
-| `CheersForConditionalGeneration` | Cheers | T + I | `ai9stars/Cheers` | | ✅︎ |
| `Cohere2VisionForConditionalGeneration` | Command A Vision, Command-A+ | T + I+ | `CohereLabs/command-a-vision-07-2025`, `CohereLabs/command-a-plus-05-2026`, etc. | | ✅︎ |
| `Cosmos3ForConditionalGeneration` | Cosmos3 (understanding tower) | T + IE+ + VE+ | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | | ✅︎ |
| `Cosmos3EdgeForConditionalGeneration` | Cosmos3-Edge (understanding tower) | T + IE+ + VE+ | `nvidia/Cosmos3-Edge` | | ✅︎ |
@@ -549,7 +541,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `Granite4VisionForConditionalGeneration` | Granite 4 Vision | T + IE+ | `ibm-granite/granite-4.1-3b-vision`, etc. | ✅︎ | ✅︎ |
| `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ |
| `GraniteSpeechPlusForConditionalGeneration` | Granite Speech Plus | T + A | `ibm-granite/granite-speech-4.1-2b-plus` | ✅︎ | ✅︎ |
-| `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | T + I+ + V+ | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | |
| `HCXVisionV2ForCausalLM` | HyperCLOVAX-SEED-Think-32B | T + I+ + V+ | `naver-hyperclovax/HyperCLOVAX-SEED-Think-32B` | | |
| `H2OVLChatModel` | H2OVL | T + IE+ | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | ✅︎ | ✅︎ |
| `HunYuanVLForConditionalGeneration` | HunyuanOCR | T + IE+ | `tencent/HunyuanOCR`, etc. | ✅︎ | ✅︎ |
@@ -700,7 +691,6 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
| ------------ | ------ | ----------------- | -------------------- | ------------------------- |
| `CohereAsrForConditionalGeneration` | Cohere-Transcribe | `CohereLabs/cohere-transcribe-03-2026` | | |
| `FireRedASR2ForConditionalGeneration` | FireRedASR2 | `allendou/FireRedASR2-LLM-vllm`, etc. | | |
-| `FireRedLIDForConditionalGeneration` | FireRedLID | `PatchyTisa/FireRedLID-vllm`, etc. | | |
| `FunASRForConditionalGeneration` | FunASR | `allendou/Fun-ASR-Nano-2512-vllm`, etc. | | |
| `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
| `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
diff --git a/examples/generate/multimodal/audio_language_offline.py b/examples/generate/multimodal/audio_language_offline.py
index fc20e8fed1..a2e9b19756 100644
--- a/examples/generate/multimodal/audio_language_offline.py
+++ b/examples/generate/multimodal/audio_language_offline.py
@@ -496,30 +496,9 @@ def run_whisper(question: str, audio_count: int) -> ModelRequestData:
)
-# FireRedLID
-def run_fireredlid(question: str, audio_count: int) -> ModelRequestData:
- assert audio_count == 1, "FireRedLID only supports single audio input per prompt"
- model_name = "PatchyTisa/FireRedLID-vllm"
-
- prompt = ""
-
- engine_args = EngineArgs(
- model=model_name,
- max_model_len=8,
- max_num_seqs=5,
- limit_mm_per_prompt={"audio": audio_count},
- )
-
- return ModelRequestData(
- engine_args=engine_args,
- prompt=prompt,
- )
-
-
model_example_map = {
"audioflamingo3": run_audioflamingo3,
"cohere_asr": run_cohere_asr,
- "fireredlid": run_fireredlid,
"funaudiochat": run_funaudiochat,
"gemma3n": run_gemma3n,
"glmasr": run_glmasr,
diff --git a/examples/generate/multimodal/encoder_decoder_multimodal_offline.py b/examples/generate/multimodal/encoder_decoder_multimodal_offline.py
index 4fc74e9555..1ab087de3f 100644
--- a/examples/generate/multimodal/encoder_decoder_multimodal_offline.py
+++ b/examples/generate/multimodal/encoder_decoder_multimodal_offline.py
@@ -95,51 +95,8 @@ def run_fireredasr2():
)
-def run_fireredlid():
- """
- FireRedLID – Language Identification model.
-
- This encoder-decoder model identifies the spoken language of an audio
- clip. It outputs at most 2 tokens representing the detected language
- (e.g. "en", "zh mandarin").
- """
- engine_args = EngineArgs(
- model="PatchyTisa/FireRedLID-vllm",
- max_model_len=8,
- max_num_seqs=16,
- limit_mm_per_prompt={"audio": 1},
- )
-
- prompts = [
- { # Test explicit encoder/decoder prompt
- "encoder_prompt": {
- "prompt": "",
- "multi_modal_data": {
- "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
- },
- },
- "decoder_prompt": "",
- },
- { # Another audio sample
- "encoder_prompt": {
- "prompt": "",
- "multi_modal_data": {
- "audio": AudioAsset("winning_call").audio_and_sample_rate,
- },
- },
- "decoder_prompt": "",
- },
- ]
-
- return ModelRequestData(
- engine_args=engine_args,
- prompts=prompts,
- )
-
-
model_example_map = {
"fireredasr2": run_fireredasr2,
- "fireredlid": run_fireredlid,
"whisper": run_whisper,
}
diff --git a/examples/generate/multimodal/vision_language_multi_image_offline.py b/examples/generate/multimodal/vision_language_multi_image_offline.py
index 1b9fddc99e..8b12df3991 100644
--- a/examples/generate/multimodal/vision_language_multi_image_offline.py
+++ b/examples/generate/multimodal/vision_language_multi_image_offline.py
@@ -365,53 +365,6 @@ def load_hunyuan_vl(question: str, image_urls: list[str]) -> ModelRequestData:
)
-def load_hyperclovax_seed_vision(
- question: str, image_urls: list[str]
-) -> ModelRequestData:
- model_name = "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B"
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-
- engine_args = EngineArgs(
- model=model_name,
- trust_remote_code=True,
- max_model_len=16384,
- limit_mm_per_prompt={"image": len(image_urls)},
- )
-
- message = {"role": "user", "content": list()}
- for _image_url in image_urls:
- message["content"].append(
- {
- "type": "image",
- "image": _image_url,
- "ocr": "",
- "lens_keywords": "",
- "lens_local_keywords": "",
- }
- )
- message["content"].append(
- {
- "type": "text",
- "text": question,
- }
- )
-
- prompt = tokenizer.apply_chat_template(
- [
- message,
- ],
- tokenize=False,
- add_generation_prompt=True,
- )
-
- return ModelRequestData(
- engine_args=engine_args,
- prompt=prompt,
- stop_token_ids=None,
- image_data=[fetch_image(url) for url in image_urls],
- )
-
-
def load_idefics3(question: str, image_urls: list[str]) -> ModelRequestData:
model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
@@ -1397,7 +1350,6 @@ model_example_map = {
"granite4_vision": load_granite4_vision,
"h2ovl_chat": load_h2ovl,
"hunyuan_vl": load_hunyuan_vl,
- "hyperclovax_seed_vision": load_hyperclovax_seed_vision,
"idefics3": load_idefics3,
"interns1": load_interns1,
"internvl_chat": load_internvl,
diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py
index d17ead4d7a..97053ba0c2 100644
--- a/examples/generate/multimodal/vision_language_offline.py
+++ b/examples/generate/multimodal/vision_language_offline.py
@@ -140,51 +140,6 @@ def run_blip2(questions: list[str], modality: str) -> ModelRequestData:
)
-# Chameleon
-def run_chameleon(questions: list[str], modality: str) -> ModelRequestData:
- assert modality == "image"
-
- prompts = [f"{question}" for question in questions]
- engine_args = EngineArgs(
- model="facebook/chameleon-7b",
- max_model_len=4096,
- max_num_seqs=2,
- limit_mm_per_prompt={modality: 1},
- )
-
- return ModelRequestData(
- engine_args=engine_args,
- prompts=prompts,
- )
-
-
-# Cheers
-def run_cheers(questions: list[str], modality: str) -> ModelRequestData:
- assert modality == "image"
- model_name = "ai9stars/Cheers"
-
- engine_args = EngineArgs(
- model=model_name,
- trust_remote_code=True,
- max_model_len=4096,
- limit_mm_per_prompt={modality: 1},
- )
-
- prompts = [
- (
- f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
- f"<|im_start|>user\n<|image_pad|>{question}<|im_end|>\n"
- f"<|im_start|>assistant\n"
- )
- for question in questions
- ]
-
- return ModelRequestData(
- engine_args=engine_args,
- prompts=prompts,
- )
-
-
def run_command_a_vision(questions: list[str], modality: str) -> ModelRequestData:
assert modality == "image"
@@ -801,109 +756,6 @@ def run_hunyuan_vl(questions: list[str], modality: str) -> ModelRequestData:
)
-# naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B
-def run_hyperclovax_seed_vision(
- questions: list[str], modality: str
-) -> ModelRequestData:
- model_name = "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B"
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-
- mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
- engine_args = EngineArgs(
- model=model_name,
- trust_remote_code=True,
- max_model_len=16384 if modality in ("video", "image+video") else 8192,
- limit_mm_per_prompt=mm_limit,
- )
-
- messages = list()
- for question in questions:
- if modality == "image":
- """
- ocr: List the words in the image in raster order.
- Even if the word order feels unnatural for reading,
- the model will handle it as long as it follows raster order.
- e.g. "Naver, CLOVA, bigshane"
- lens_keywords: List the entity names in the image.
- e.g. "iPhone"
- lens_local_keywords: List the entity names with quads in the image.
- e.g. "[0.07, 0.21, 0.92, 0.90] iPhone"
- """
- messages.append(
- [
- {
- "role": "user",
- "content": [
- {
- "type": "image",
- "ocr": "",
- "lens_keywords": "",
- "lens_local_keywords": "",
- },
- {
- "type": "text",
- "text": question,
- },
- ],
- }
- ]
- )
- elif modality == "video":
- messages.append(
- [
- {
- "role": "user",
- "content": [
- {
- "type": "video",
- },
- {
- "type": "text",
- "text": question,
- },
- ],
- }
- ]
- )
- elif modality == "image+video":
- messages.append(
- [
- {
- "role": "user",
- "content": [
- {
- "type": "image",
- "ocr": "",
- "lens_keywords": "",
- "lens_local_keywords": "",
- },
- {
- "type": "video",
- },
- {
- "type": "text",
- "text": question,
- },
- ],
- }
- ]
- )
- else:
- raise ValueError(f"Unsupported modality: {modality}")
-
- prompts = tokenizer.apply_chat_template(
- messages,
- tokenize=False,
- add_generation_prompt=True,
- )
-
- return ModelRequestData(
- engine_args=engine_args,
- prompts=prompts,
- stop_token_ids=None,
- )
-
-
# Idefics3-8B-Llama3
def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:
assert modality == "image"
@@ -2328,10 +2180,8 @@ def run_step_vl(questions: list[str], modality: str) -> ModelRequestData:
model_example_map = {
"aria": run_aria,
"bagel": run_bagel,
- "cheers": run_cheers,
"bee": run_bee,
"blip-2": run_blip2,
- "chameleon": run_chameleon,
"command_a_vision": run_command_a_vision,
"deepseek_vl_v2": run_deepseek_vl2,
"deepseek_ocr": run_deepseek_ocr,
@@ -2350,7 +2200,6 @@ model_example_map = {
"glm_ocr": run_glm_ocr,
"h2ovl_chat": run_h2ovl,
"hunyuan_vl": run_hunyuan_vl,
- "hyperclovax_seed_vision": run_hyperclovax_seed_vision,
"idefics3": run_idefics3,
"interns1": run_interns1,
"interns1_pro": run_interns1_pro,
diff --git a/examples/speech_to_text/lid/openai_lid_client.py b/examples/speech_to_text/lid/openai_lid_client.py
deleted file mode 100644
index d91df3298c..0000000000
--- a/examples/speech_to_text/lid/openai_lid_client.py
+++ /dev/null
@@ -1,193 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-Language Identification (LID) demo using the FireRedLID model on vLLM.
-
-FireRedLID is an audio encoder-decoder model that identifies the spoken
-language of an audio clip. Unlike ASR models that output full transcriptions,
-FireRedLID outputs at most 2 tokens representing the detected language
-(e.g. "en", "zh mandarin").
-
-Start the vLLM server:
-
- vllm serve PatchyTisa/FireRedLID-vllm
-
-Then run this script:
-
- # Use the built-in sample audio
- python examples/speech_to_text/lid/openai_lid_client.py
-
- # Use your own audio file(s)
- python examples/speech_to_text/lid/openai_lid_client.py \
- --audio_paths audio_en.wav audio_zh.wav audio_fr.wav
-
- # Batch-identify multiple files in one run
- python examples/speech_to_text/lid/openai_lid_client.py \
- --audio_paths /path/to/dir/*.wav
-
-Requirements:
-- vLLM with audio support
-- openai Python SDK
-- kaldi_native_fbank (pulled in by the model)
-"""
-
-import argparse
-import json
-import os
-
-from openai import OpenAI
-
-from vllm.assets.audio import AudioAsset
-
-# ──────────────────────────────────────────────────────────────────────
-# Helpers
-# ──────────────────────────────────────────────────────────────────────
-
-
-def identify_language(
- audio_path: str,
- client: OpenAI,
- model: str,
-) -> str:
- """
- Send a single audio file to the vLLM transcription endpoint and return
- the detected language tag.
-
- FireRedLID re-uses the OpenAI-compatible ``/v1/audio/transcriptions``
- endpoint. The "transcription" it returns is actually the language label
- (e.g. ``"en"`` or ``"zh mandarin"``).
- """
- with open(audio_path, "rb") as f:
- result = client.audio.transcriptions.create(
- file=f,
- model=model,
- response_format="json",
- temperature=0.0,
- )
- return result.text.strip()
-
-
-def identify_language_raw(
- audio_path: str,
- model: str,
- api_base: str,
-) -> str:
- """
- Same as :func:`identify_language` but uses raw HTTP so that the demo
- works without the ``openai`` SDK (useful for quick debugging).
- """
- import requests
-
- url = f"{api_base}/audio/transcriptions"
- with open(audio_path, "rb") as f:
- files = {"file": (os.path.basename(audio_path), f)}
- data = {
- "model": model,
- "response_format": "json",
- }
- resp = requests.post(url, files=files, data=data)
- resp.raise_for_status()
- return resp.json()["text"].strip()
-
-
-def identify_language_streaming(
- audio_path: str,
- model: str,
- api_base: str,
-) -> str:
- """
- Streaming variant – demonstrates the streaming transcription endpoint.
- For a 1-2 token output the stream finishes almost instantly, but this
- shows that the API path works end-to-end.
- """
- import requests
-
- url = f"{api_base}/audio/transcriptions"
- with open(audio_path, "rb") as f:
- files = {"file": (os.path.basename(audio_path), f)}
- data = {
- "stream": "true",
- "model": model,
- "response_format": "json",
- }
- response = requests.post(url, files=files, data=data, stream=True)
- response.raise_for_status()
-
- tokens: list[str] = []
- for chunk in response.iter_lines(
- chunk_size=8192, decode_unicode=False, delimiter=b"\n"
- ):
- if not chunk:
- continue
- payload = json.loads(chunk[len("data: ") :].decode("utf-8"))
- choice = payload["choices"][0]
- delta = choice.get("delta", {}).get("content", "")
- if delta:
- tokens.append(delta)
- if choice.get("finish_reason") is not None:
- break
-
- return "".join(tokens).strip()
-
-
-# ──────────────────────────────────────────────────────────────────────
-# Main
-# ──────────────────────────────────────────────────────────────────────
-
-
-def main(args: argparse.Namespace) -> None:
- api_base = args.api_base.rstrip("/")
- client = OpenAI(api_key="EMPTY", base_url=api_base)
- model = client.models.list().data[0].id
- print(f"Model : {model}")
- print(f"Server: {api_base}\n")
-
- # Resolve audio paths ------------------------------------------------
- if args.audio_paths:
- audio_paths = args.audio_paths
- else:
- # Fall back to the built-in vLLM sample audios (both are English).
- audio_paths = [
- str(AudioAsset("mary_had_lamb").get_local_path()),
- str(AudioAsset("winning_call").get_local_path()),
- ]
-
- # Run LID for each file ----------------------------------------------
- print(f"{'Audio File':<50} {'Language (sync)':<20} {'Language (stream)'}")
- print("-" * 90)
-
- for path in audio_paths:
- basename = os.path.basename(path)
-
- # 1) Synchronous via OpenAI SDK
- lang_sync = identify_language(path, client, model)
-
- # 2) Streaming via raw HTTP
- lang_stream = identify_language_streaming(path, model, api_base)
-
- print(f"{basename:<50} {lang_sync:<20} {lang_stream}")
-
- print()
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(
- description="FireRedLID – Language Identification demo via vLLM",
- )
- parser.add_argument(
- "--audio_paths",
- nargs="+",
- default=None,
- help=(
- "One or more audio files to identify. "
- "If omitted, uses vLLM's built-in sample audios."
- ),
- )
- parser.add_argument(
- "--api_base",
- type=str,
- default="http://localhost:8000/v1",
- help="vLLM API base URL (default: http://localhost:8000/v1)",
- )
- args = parser.parse_args()
- main(args)
diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt
index c59d40484b..0925d06fa5 100644
--- a/requirements/test/nightly-torch.txt
+++ b/requirements/test/nightly-torch.txt
@@ -10,7 +10,7 @@ pytest-timeout
# testing utils
backoff # required for phi4mm test
blobfile # required for kimi-vl test
-einops # required for MPT, qwen-vl and Mamba
+einops # required for qwen-vl and Mamba
httpx
librosa # required for audio tests
vocos # required for minicpmo_26 test
diff --git a/tests/config/base_model_arch_groundtruth.json b/tests/config/base_model_arch_groundtruth.json
index 14e3f4f49d..de06739832 100644
--- a/tests/config/base_model_arch_groundtruth.json
+++ b/tests/config/base_model_arch_groundtruth.json
@@ -84,23 +84,6 @@
"is_multimodal_model": false,
"dtype": "torch.bfloat16"
},
- "mosaicml/mpt-7b": {
- "architectures": [
- "MPTForCausalLM"
- ],
- "model_type": "mpt",
- "text_model_type": "mpt",
- "hidden_size": 4096,
- "total_num_hidden_layers": 32,
- "total_num_attention_heads": 32,
- "head_size": 128,
- "vocab_size": 50432,
- "total_num_kv_heads": 32,
- "num_experts": 0,
- "is_deepseek_mla": false,
- "is_multimodal_model": false,
- "dtype": "torch.bfloat16"
- },
"databricks/dbrx-instruct": {
"architectures": [
"DbrxForCausalLM"
diff --git a/tests/config/test_model_arch_config.py b/tests/config/test_model_arch_config.py
index 0688be4fd7..fb2db3f2c7 100644
--- a/tests/config/test_model_arch_config.py
+++ b/tests/config/test_model_arch_config.py
@@ -38,8 +38,6 @@ BASE_MODELS_TO_TEST = [
# (NonGeoDataset import error). Tested in model initialization tests.
# "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
"Zyphra/Zamba2-7B-instruct",
- # FIXME: mosaicml/mpt-7b has been deleted
- # "mosaicml/mpt-7b",
# FIXME: databricks/dbrx-instruct has been deleted
# "databricks/dbrx-instruct",
"tiiuae/falcon-7b",
diff --git a/tests/distributed/test_pipeline_parallel.py b/tests/distributed/test_pipeline_parallel.py
index 22e0321da6..a464a61683 100644
--- a/tests/distributed/test_pipeline_parallel.py
+++ b/tests/distributed/test_pipeline_parallel.py
@@ -102,7 +102,6 @@ class PPTestSettings:
TEXT_GENERATION_MODELS = {
# [Decoder-only]
- "Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(load_format="dummy"),
"bigscience/bloomz-1b1": PPTestSettings.fast(),
"zai-org/chatglm3-6b": PPTestSettings.fast(),
"CohereLabs/c4ai-command-r-v01": PPTestSettings.fast(load_format="dummy"),
@@ -131,7 +130,6 @@ TEXT_GENERATION_MODELS = {
# "mistralai/Mistral-7B-Instruct-v0.1": PPTestSettings.fast(),
"state-spaces/mamba-130m-hf": PPTestSettings.fast(),
"mistralai/Mixtral-8x7B-Instruct-v0.1": PPTestSettings.fast(load_format="dummy"),
- "mosaicml/mpt-7b": PPTestSettings.fast(),
"nvidia/Minitron-8B-Base": PPTestSettings.fast(),
"allenai/OLMo-1B-hf": PPTestSettings.fast(),
"allenai/OLMo-2-0425-1B": PPTestSettings.fast(),
@@ -165,7 +163,6 @@ EMBEDDING_MODELS = { # type: ignore[var-annotated]
MULTIMODAL_MODELS = {
# [Decoder-only]
"Salesforce/blip2-opt-6.7b": PPTestSettings.fast(),
- "facebook/chameleon-7b": PPTestSettings.fast(),
"zai-org/glm-4v-9b": PPTestSettings.fast(),
"OpenGVLab/InternVL3-1B": PPTestSettings.fast(),
"llava-hf/llava-1.5-7b-hf": PPTestSettings.fast(),
diff --git a/tests/models/language/pooling/test_gritlm.py b/tests/models/language/pooling/test_gritlm.py
deleted file mode 100644
index 64265d6a0c..0000000000
--- a/tests/models/language/pooling/test_gritlm.py
+++ /dev/null
@@ -1,188 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import numpy as np
-import pytest
-from scipy.spatial.distance import cosine
-
-from vllm import LLM, SamplingParams
-from vllm.config import ModelConfig
-
-from ....utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
-from .embed_utils import run_client_embeddings
-
-MODEL_NAME = "parasail-ai/GritLM-7B-vllm"
-MAX_MODEL_LEN = 4000
-ATOL = 2.3e-3
-
-
-def _arr(arr):
- """
- Convert a list of integers to an array of integers.
- """
- return np.array(arr)
-
-
-def test_find_array():
- from vllm.model_executor.models.gritlm import GritLMMeanPool
-
- model_config = ModelConfig(
- MODEL_NAME,
- runner="pooling",
- dtype="bfloat16",
- seed=0,
- )
- pooling = GritLMMeanPool(model_config=model_config)
-
- arr = _arr([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
-
- assert pooling._find_array(arr, _arr([3, 4, 5]), start_idx=0) == 3
- assert pooling._find_array(arr, _arr([3, 4, 5]), start_idx=1) == 3
- assert pooling._find_array(arr, _arr([3, 4, 5]), start_idx=5) == -1
- assert pooling._find_array(arr, _arr([3, 4, 5]), end_idx=3) == -1
- assert pooling._find_array(arr, _arr([3, 4, 5]), end_idx=4) == 3
- assert pooling._find_array(arr, _arr([3, 5]), start_idx=0) == -1
-
- with pytest.raises(ValueError):
- pooling._find_array(arr, _arr([3, 4, 5]), start_idx=-1)
-
-
-def run_llm_encode(
- llm: LLM,
- queries: list[str],
- instruction: str,
-) -> list[list[float]]:
- outputs = llm.embed([instruction + q for q in queries])
- return [output.outputs.embedding for output in outputs]
-
-
-def gritlm_instruction(instruction):
- return (
- "<|user|>\n" + instruction + "\n<|embed|>\n" if instruction else "<|embed|>\n"
- )
-
-
-def get_test_data():
- """
- Grabbed this test data and the expected values from
- README.md in https://github.com/ContextualAI/gritlm
- """
- q_instruction = gritlm_instruction(
- "Given a scientific paper title, retrieve the paper's abstract",
- )
- queries = [
- "Bitcoin: A Peer-to-Peer Electronic Cash System",
- "Generative Representational Instruction Tuning",
- ]
-
- d_instruction = gritlm_instruction("")
- documents = [
- # ruff: noqa: E501
- "A purely peer-to-peer version of electronic cash would allow online payments to be sent directly from one party to another without going through a financial institution. Digital signatures provide part of the solution, but the main benefits are lost if a trusted third party is still required to prevent double-spending. We propose a solution to the double-spending problem using a peer-to-peer network. The network timestamps transactions by hashing them into an ongoing chain of hash-based proof-of-work, forming a record that cannot be changed without redoing the proof-of-work. The longest chain not only serves as proof of the sequence of events witnessed, but proof that it came from the largest pool of CPU power. As long as a majority of CPU power is controlled by nodes that are not cooperating to attack the network, they'll generate the longest chain and outpace attackers. The network itself requires minimal structure. Messages are broadcast on a best effort basis, and nodes can leave and rejoin the network at will, accepting the longest proof-of-work chain as proof of what happened while they were gone.",
- "All text-based language problems can be reduced to either generation or embedding. Current models only perform well at one or the other. We introduce generative representational instruction tuning (GRIT) whereby a large language model is trained to handle both generative and embedding tasks by distinguishing between them through instructions. Compared to other open models, our resulting GritLM 7B sets a new state of the art on the Massive Text Embedding Benchmark (MTEB) and outperforms all models up to its size on a range of generative tasks. By scaling up further, GritLM 8X7B outperforms all open generative language models that we tried while still being among the best embedding models. Notably, we find that GRIT matches training on only generative or embedding data, thus we can unify both at no performance loss. Among other benefits, the unification via GRIT speeds up Retrieval-Augmented Generation (RAG) by > 60% for long documents, by no longer requiring separate retrieval and generation models. Models, code, etc. are freely available at https://github.com/ContextualAI/gritlm.",
- ]
-
- return queries, q_instruction, documents, d_instruction
-
-
-def validate_embed_output(q_rep: list[list[float]], d_rep: list[list[float]]):
- cosine_sim_q0_d0 = 1 - cosine(q_rep[0], d_rep[0])
- assert cosine_sim_q0_d0 == pytest.approx(0.609, abs=ATOL)
-
- cosine_sim_q0_d1 = 1 - cosine(q_rep[0], d_rep[1])
- assert cosine_sim_q0_d1 == pytest.approx(0.101, abs=ATOL)
-
- cosine_sim_q1_d0 = 1 - cosine(q_rep[1], d_rep[0])
- assert cosine_sim_q1_d0 == pytest.approx(0.120, abs=ATOL)
-
- cosine_sim_q1_d1 = 1 - cosine(q_rep[1], d_rep[1])
- assert cosine_sim_q1_d1 == pytest.approx(0.534, abs=ATOL)
-
-
-def test_gritlm_offline_embedding(vllm_runner):
- queries, q_instruction, documents, d_instruction = get_test_data()
-
- with vllm_runner(
- MODEL_NAME,
- runner="pooling",
- max_model_len=MAX_MODEL_LEN,
- ) as vllm_model:
- llm = vllm_model.llm
-
- d_rep = run_llm_encode(
- llm,
- documents,
- d_instruction,
- )
- q_rep = run_llm_encode(
- llm,
- queries,
- q_instruction,
- )
-
- validate_embed_output(q_rep, d_rep)
-
-
-@pytest.mark.asyncio
-async def test_gritlm_api_server_embedding():
- queries, q_instruction, documents, d_instruction = get_test_data()
-
- args = [
- "--runner",
- "pooling",
- "--max_model_len",
- str(MAX_MODEL_LEN),
- *ROCM_EXTRA_ARGS,
- ]
-
- with RemoteOpenAIServer(MODEL_NAME, args) as server:
- client_embedding = server.get_async_client()
-
- d_rep = await run_client_embeddings(
- client_embedding,
- MODEL_NAME,
- documents,
- d_instruction,
- )
- q_rep = await run_client_embeddings(
- client_embedding,
- MODEL_NAME,
- queries,
- q_instruction,
- )
-
- validate_embed_output(q_rep, d_rep)
-
-
-def test_gritlm_offline_generate(monkeypatch: pytest.MonkeyPatch, vllm_runner):
- input = "<|user|>\nWhat is the capital of France?\n<|assistant|>\n"
-
- with vllm_runner(
- MODEL_NAME,
- runner="generate",
- max_model_len=MAX_MODEL_LEN,
- ) as vllm_model:
- llm = vllm_model.llm
-
- sampling_params = SamplingParams(temperature=0.0, max_tokens=256)
- outputs = llm.generate(input, sampling_params=sampling_params)
-
- assert outputs[0].outputs[0].text == "The capital of France is Paris."
-
-
-@pytest.mark.asyncio
-async def test_gritlm_api_server_generate():
- input = "<|user|>\nWhat is the capital of France?\n<|assistant|>\n"
-
- args = ["--runner", "generate", "--max_model_len", str(MAX_MODEL_LEN)]
-
- with RemoteOpenAIServer(MODEL_NAME, args) as server:
- client_generate = server.get_async_client()
-
- outputs = await client_generate.completions.create(
- model=MODEL_NAME,
- prompt=input,
- max_tokens=256,
- temperature=0.0,
- )
-
- assert outputs.choices[0].text == "The capital of France is Paris."
diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py
index b367767c6b..45ad479d83 100644
--- a/tests/models/multimodal/generation/test_common.py
+++ b/tests/models/multimodal/generation/test_common.py
@@ -30,7 +30,6 @@ from ....conftest import (
VllmRunner,
)
from ....utils import create_new_process_for_each_test, large_gpu_mark, multi_gpu_marks
-from ...utils import check_outputs_equal
from .vlm_utils import custom_inputs, model_utils, runners
from .vlm_utils.case_filtering import get_parametrized_options
from .vlm_utils.types import (
@@ -321,20 +320,6 @@ VLM_TEST_SETTINGS = {
# FIXME: https://github.com/huggingface/transformers/pull/38510
marks=[pytest.mark.skip("Model is broken")],
),
- "chameleon": VLMTestInfo(
- models=["facebook/chameleon-7b"],
- test_type=VLMTestType.IMAGE,
- prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
- max_model_len=4096,
- max_num_seqs=2,
- auto_cls=AutoModelForImageTextToText,
- # For chameleon, we only compare the sequences
- vllm_output_post_proc=lambda vllm_output, model: vllm_output[:2],
- hf_output_post_proc=lambda hf_output, model: hf_output[:2],
- comparator=check_outputs_equal,
- max_tokens=8,
- dtype="bfloat16",
- ),
"cosmos3": VLMTestInfo(
models=["nvidia/Cosmos3-Nano"],
test_type=(
@@ -935,17 +920,6 @@ VLM_TEST_SETTINGS = {
num_logprobs=10,
),
### Tensor parallel / multi-gpu broadcast tests
- "chameleon-broadcast": VLMTestInfo(
- models=["facebook/chameleon-7b"],
- prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
- max_model_len=4096,
- auto_cls=AutoModelForImageTextToText,
- vllm_output_post_proc=lambda vllm_output, model: vllm_output[:2],
- hf_output_post_proc=lambda hf_output, model: hf_output[:2],
- comparator=check_outputs_equal,
- marks=multi_gpu_marks(num_gpus=2),
- **COMMON_BROADCAST_SETTINGS, # type: ignore
- ),
"llava-broadcast": VLMTestInfo(
models=["llava-hf/llava-1.5-7b-hf"],
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
diff --git a/tests/models/registry.py b/tests/models/registry.py
index 712f160e3d..c46772db9d 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -202,7 +202,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"),
"ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),
"ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
- "ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct"),
"AXK1ForCausalLM": _HfExamplesInfo("skt/A.X-K1", trust_remote_code=True),
"BailingMoeForCausalLM": _HfExamplesInfo(
"inclusionAI/Ling-lite-1.5", trust_remote_code=True
@@ -273,7 +272,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"ExaoneMoeForCausalLM": _HfExamplesInfo(
"LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0"
),
- "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"),
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
@@ -448,9 +446,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"mistralai/Mixtral-8x7B-Instruct-v0.1",
{"tiny": "TitanML/tiny-mixtral"},
),
- "MptForCausalLM": _HfExamplesInfo("mpt", is_available_online=False),
- # FIXME: mosaicml/mpt-7b has been deleted
- "MPTForCausalLM": _HfExamplesInfo("mosaicml/mpt-7b", is_available_online=False),
"NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"),
"NemotronHForCausalLM": _HfExamplesInfo(
"nvidia/Nemotron-H-8B-Base-8K", trust_remote_code=True
@@ -523,7 +518,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"EssentialAI/rnj-1-instruct",
is_available_online=False,
),
- "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b"),
"SarvamMoEForCausalLM": _HfExamplesInfo(
"sarvamai/sarvam-30b",
trust_remote_code=True,
@@ -548,7 +542,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
trust_remote_code=True,
),
"SmolLM3ForCausalLM": _HfExamplesInfo("HuggingFaceTB/SmolLM3-3B"),
- "StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b"),
"StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t"),
"Starcoder2ForCausalLM": _HfExamplesInfo("bigcode/starcoder2-3b"),
"Step1ForCausalLM": _HfExamplesInfo(
@@ -600,7 +593,6 @@ _EMBEDDING_EXAMPLE_MODELS = {
),
"Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
"Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"),
- "GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"),
"GteModel": _HfExamplesInfo(
"Snowflake/snowflake-arctic-embed-m-v2.0", trust_remote_code=True
),
@@ -642,14 +634,6 @@ _EMBEDDING_EXAMPLE_MODELS = {
),
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"),
"SiglipModel": _HfExamplesInfo("google/siglip-base-patch16-224"),
- "PrithviGeoSpatialMAE": _HfExamplesInfo(
- "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
- dtype="float16",
- enforce_eager=True,
- require_embed_inputs=True,
- # This is to avoid the model going OOM in CI
- max_num_seqs=32,
- ),
"Terratorch": _HfExamplesInfo(
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
dtype="float16",
@@ -813,15 +797,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Salesforce/blip2-opt-2.7b",
extras={"6b": "Salesforce/blip2-opt-6.7b"},
),
- "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"),
- "Cheers": _HfExamplesInfo(
- "ai9stars/Cheers",
- trust_remote_code=True,
- ),
- "CheersForConditionalGeneration": _HfExamplesInfo(
- "ai9stars/Cheers",
- trust_remote_code=True,
- ),
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
"CohereLabs/command-a-vision-07-2025",
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},
@@ -890,15 +865,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"(dict object has no attribute '__name__').",
},
),
- "FireRedLIDForConditionalGeneration": _HfExamplesInfo(
- "PatchyTisa/FireRedLID-vllm",
- trust_remote_code=True,
- max_transformers_version="5.1",
- transformers_version_reason={
- "vllm": "Incompatible with transformers v5.2+ "
- "(dict object has no attribute '__name__').",
- },
- ),
"FunASRForConditionalGeneration": _HfExamplesInfo(
"allendou/Fun-ASR-Nano-2512-vllm",
trust_remote_code=True,
@@ -966,10 +932,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
max_transformers_version="4.48",
transformers_version_reason={"hf": "HF model is not compatible."},
),
- "HCXVisionForCausalLM": _HfExamplesInfo(
- "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
- trust_remote_code=True,
- ),
"HCXVisionV2ForCausalLM": _HfExamplesInfo(
"naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
trust_remote_code=True,
diff --git a/tests/models/test_initialization.py b/tests/models/test_initialization.py
index 914d8d95ea..395d7e3706 100644
--- a/tests/models/test_initialization.py
+++ b/tests/models/test_initialization.py
@@ -39,7 +39,7 @@ MINIMAL_MODEL_ARCH_LIST = [
"InternVLChatModel",
"InternLM2ForRewardModel",
"TransformersMultiModalForCausalLM",
- "PrithviGeoSpatialMAE",
+ "Terratorch",
"UltravoxModel",
"DeepSeekMTPModel",
"XLMRobertaModel",
@@ -111,7 +111,7 @@ def can_initialize(
"which is not configured in test environment"
)
- if model_arch in ("PrithviGeoSpatialMAE", "Terratorch"):
+ if model_arch == "Terratorch":
import importlib.util
if importlib.util.find_spec("terratorch") is None:
diff --git a/tests/models/test_registry.py b/tests/models/test_registry.py
index 81d6b31215..81ccbf63a4 100644
--- a/tests/models/test_registry.py
+++ b/tests/models/test_registry.py
@@ -38,7 +38,7 @@ def test_registry_imports(model_arch):
check_version_reason="vllm",
)
- if model_arch in ("PrithviGeoSpatialMAE", "Terratorch"):
+ if model_arch == "Terratorch":
import importlib.util
if importlib.util.find_spec("terratorch") is None:
diff --git a/tests/renderers/test_hf.py b/tests/renderers/test_hf.py
index f611bac02b..24a9e26fe9 100644
--- a/tests/renderers/test_hf.py
+++ b/tests/renderers/test_hf.py
@@ -424,7 +424,6 @@ def test_resolve_content_format_hf_defined(model, expected_format):
("model", "expected_format"),
[
("Salesforce/blip2-opt-2.7b", "string"),
- ("facebook/chameleon-7b", "string"),
("deepseek-ai/deepseek-vl2-tiny", "string"),
("google/paligemma-3b-mix-224", "string"),
],
diff --git a/tests/tokenizers_/test_detokenize.py b/tests/tokenizers_/test_detokenize.py
index ad399ba36d..9db049090f 100644
--- a/tests/tokenizers_/test_detokenize.py
+++ b/tests/tokenizers_/test_detokenize.py
@@ -39,8 +39,6 @@ TOKENIZERS = [
"EleutherAI/gpt-j-6b",
"EleutherAI/pythia-70m",
"bigscience/bloom-560m",
- # FIXME: mosaicml/mpt-7b has been deleted
- # "mosaicml/mpt-7b",
"tiiuae/falcon-7b",
"meta-llama/Llama-3.2-1B-Instruct",
"codellama/CodeLlama-7b-hf",
diff --git a/tests/utils_/test_tensor_schema.py b/tests/utils_/test_tensor_schema.py
index c86bed7547..8b340ccec0 100644
--- a/tests/utils_/test_tensor_schema.py
+++ b/tests/utils_/test_tensor_schema.py
@@ -1,13 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from typing import Annotated, Literal
+
import pytest
import torch
from vllm.model_executor.models.glm4_1v import Glm4vImageEmbeddingInputs
from vllm.model_executor.models.granite_speech import GraniteSpeechAudioInputs
-from vllm.model_executor.models.hyperclovax_vision import HCXVisionVideoPixelInputs
from vllm.model_executor.models.phi3v import Phi3VImagePixelInputs
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
+
+
+class _DoubleNestedVideoPixelInputs(TensorSchema):
+ """Minimal schema exercising doubly-nested list[list[torch.Tensor]] inputs.
+
+ Dimensions:
+ - n: Number of videos
+ - f: Number of frames
+ - g: Number of grids
+ - c: Number of channels (3)
+ - h: Height
+ - w: Width
+ """
+
+ type: Literal["pixel_values_videos"] = "pixel_values_videos"
+ pixel_values_videos: Annotated[
+ list[list[torch.Tensor]],
+ TensorShape("n", "f", "g", 3, "h", "w", dynamic_dims={"f", "g"}),
+ ]
def test_tensor_schema_valid_tensor():
@@ -100,7 +121,7 @@ def test_tensor_schema_double_nested_tensors():
x = torch.rand(4, 3, 32, 32)
y = torch.rand(2, 3, 32, 32)
- HCXVisionVideoPixelInputs(pixel_values_videos=([x, y, x], [y], [x, y]))
+ _DoubleNestedVideoPixelInputs(pixel_values_videos=([x, y, x], [y], [x, y]))
def test_tensor_schema_inconsistent_shapes_in_list():
diff --git a/tests/weight_loading/models.txt b/tests/weight_loading/models.txt
index 398a12afd5..dd197a33b5 100644
--- a/tests/weight_loading/models.txt
+++ b/tests/weight_loading/models.txt
@@ -16,5 +16,4 @@ compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
#compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
awq, casperhansen/mixtral-instruct-awq, main
awq_marlin, casperhansen/mixtral-instruct-awq, main
-fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
-None, mgleize/fairseq2-dummy-Llama-3.2-1B, main
\ No newline at end of file
+fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
\ No newline at end of file
diff --git a/vllm/model_executor/models/apertus.py b/vllm/model_executor/models/apertus.py
index b904802a3a..45b67036b8 100644
--- a/vllm/model_executor/models/apertus.py
+++ b/vllm/model_executor/models/apertus.py
@@ -255,8 +255,7 @@ class ApertusDecoderLayer(nn.Module):
# Apertus defaults to causal attention as it is a decoder-only model.
# You can override the HF config with `is_causal=False` to enable
- # bidirectional attention, which is used in some embedding models
- # (e.g. parasail-ai/GritLM-7B-vllm)
+ # bidirectional attention, which is used in some embedding models.
if getattr(config, "is_causal", True):
attn_type = AttentionType.DECODER
else:
diff --git a/vllm/model_executor/models/arctic.py b/vllm/model_executor/models/arctic.py
deleted file mode 100644
index 555e58aa13..0000000000
--- a/vllm/model_executor/models/arctic.py
+++ /dev/null
@@ -1,593 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Inference-only Snowflake Arctic model."""
-
-from collections.abc import Iterable
-from itertools import islice
-
-import torch
-from torch import nn
-
-from vllm.compilation.decorators import support_torch_compile
-from vllm.config import CacheConfig, VllmConfig
-from vllm.distributed import (
- get_pp_group,
- get_tensor_model_parallel_rank,
- get_tensor_model_parallel_world_size,
- tensor_model_parallel_all_reduce,
-)
-from vllm.model_executor.layers.activation import SiluAndMul
-from vllm.model_executor.layers.attention import Attention
-from vllm.model_executor.layers.fused_moe import (
- fused_experts,
- fused_topk,
-)
-from vllm.model_executor.layers.layernorm import RMSNorm
-from vllm.model_executor.layers.linear import (
- MergedColumnParallelLinear,
- QKVParallelLinear,
- ReplicatedLinear,
- RowParallelLinear,
-)
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
-from vllm.model_executor.layers.quantization import QuantizationConfig
-from vllm.model_executor.layers.rotary_embedding import get_rope
-from vllm.model_executor.layers.vocab_parallel_embedding import (
- ParallelLMHead,
- VocabParallelEmbedding,
-)
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
-from vllm.model_executor.utils import set_weight_attrs
-from vllm.platforms import current_platform
-from vllm.sequence import IntermediateTensors
-from vllm.transformers_utils.configs.arctic import ArcticConfig
-
-from .interfaces import SupportsPP, SupportsQuant
-from .utils import (
- AutoWeightsLoader,
- extract_layer_index,
- is_pp_missing_parameter,
- make_empty_intermediate_tensors_factory,
- make_layers,
- maybe_prefix,
-)
-
-
-class ArcticMLP(nn.Module):
- def __init__(
- self,
- config: ArcticConfig,
- expert_id: int = -1,
- is_residual_mlp: bool = False,
- quant_config: QuantizationConfig | None = None,
- reduce_results: bool = True,
- prefix: str = "",
- ):
- super().__init__()
- self.hidden_size = config.hidden_size
- self.expert_id = expert_id
-
- self.ffn_dim = (
- config.intermediate_size if not is_residual_mlp else self.hidden_size
- )
-
- self.w13 = MergedColumnParallelLinear(
- self.hidden_size,
- [self.ffn_dim] * 2,
- bias=False,
- quant_config=quant_config,
- prefix=f"{prefix}.w13",
- )
- self.w2 = RowParallelLinear(
- self.ffn_dim,
- self.hidden_size,
- bias=False,
- reduce_results=reduce_results,
- quant_config=quant_config,
- prefix=f"{prefix}.w2",
- )
- if config.hidden_act != "silu":
- raise ValueError(
- f"Unsupported activation: {config.hidden_act}. "
- "Only silu is supported for now."
- )
- self.act_fn = SiluAndMul()
-
- def forward(self, hidden_states):
- gate_up, _ = self.w13(hidden_states)
- hidden_states = self.act_fn(gate_up)
- hidden_states, _ = self.w2(hidden_states)
- return hidden_states
-
-
-class ArcticMoE(nn.Module):
- """
- Model-parallel implementation of Arctic MoE Layer.
- """
-
- def __init__(
- self,
- config: ArcticConfig,
- tp_size: int | None = None,
- params_dtype: torch.dtype | None = None,
- quant_config: QuantizationConfig | None = None,
- reduce_results: bool = True,
- prefix: str = "",
- ):
- super().__init__()
-
- layer_id = extract_layer_index(prefix)
- self.tp_size = tp_size or get_tensor_model_parallel_world_size()
- self.hidden_size = config.hidden_size
- self.num_experts = config.num_local_experts
- self.layer_id = layer_id
- self.top_k = config.num_experts_per_tok
- self.intermediate_size = config.intermediate_size // self.tp_size
-
- self.is_moe_layer = (layer_id + 1) % config.moe_layer_frequency == 0
- self.reduce_results = reduce_results
- # Some other parameters
- if params_dtype is None:
- params_dtype = torch.get_default_dtype()
- self.params_dtype = params_dtype
-
- if not self.is_moe_layer:
- self.mlp = ArcticMLP(
- config,
- quant_config=quant_config,
- reduce_results=reduce_results,
- prefix=f"{prefix}.mlp",
- )
- else:
- self.gate = ReplicatedLinear(
- self.hidden_size,
- self.num_experts,
- bias=False,
- params_dtype=self.params_dtype,
- quant_config=quant_config,
- prefix=f"{prefix}.gate",
- )
- self.ws = nn.Parameter(
- torch.empty(
- self.num_experts,
- 2 * self.intermediate_size,
- self.hidden_size,
- device=current_platform.device_type,
- dtype=self.params_dtype,
- )
- )
- self.w2s = nn.Parameter(
- torch.empty(
- self.num_experts,
- self.hidden_size,
- self.intermediate_size,
- device=current_platform.device_type,
- dtype=self.params_dtype,
- )
- )
- set_weight_attrs(
- self.ws,
- {
- "weight_loader": self.weight_loader,
- },
- )
- set_weight_attrs(
- self.w2s,
- {
- "weight_loader": self.weight_loader,
- },
- )
-
- def weight_loader(
- self,
- param: nn.Parameter,
- loaded_weight: torch.Tensor,
- weight_name: str,
- expert_id: int,
- ):
- tp_rank = get_tensor_model_parallel_rank()
- param_data = param.data
- shard_size = self.intermediate_size
- shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
- if weight_name.endswith("w1.weight"):
- param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
- if weight_name.endswith("w3.weight"):
- param_data[expert_id, shard_size : 2 * shard_size, :] = loaded_weight[
- shard, :
- ]
- if weight_name.endswith("w2.weight"):
- param_data[expert_id, :, :] = loaded_weight[:, shard]
-
- def local_moe_fused(self, hidden_states: torch.Tensor) -> torch.Tensor:
- num_tokens, hidden_size = hidden_states.shape
- hidden_states = hidden_states.view(-1, self.hidden_size)
- # router_logits: (num_tokens, n_experts)
- router_logits, _ = self.gate(hidden_states)
- do_normalize = self.top_k > 1
- topk_weights, topk_ids, token_expert_indices = fused_topk(
- hidden_states, router_logits, self.top_k, renormalize=do_normalize
- )
- final_hidden_states = fused_experts(
- hidden_states,
- self.ws,
- self.w2s,
- topk_weights,
- topk_ids,
- )
- if self.reduce_results and self.tp_size > 1:
- final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
- return final_hidden_states.view(num_tokens, hidden_size)
-
- def forward(self, hidden_states: torch.Tensor):
- if self.is_moe_layer:
- final_hidden_states = self.local_moe_fused(hidden_states)
- else:
- final_hidden_states = self.mlp(hidden_states)
- return final_hidden_states
-
-
-class ArcticAttention(nn.Module):
- def __init__(
- self,
- config: ArcticConfig,
- cache_config: CacheConfig | None = None,
- quant_config: QuantizationConfig | None = None,
- prefix: str = "",
- ):
- super().__init__()
- self.config = config
- self.hidden_size = config.hidden_size
-
- tp_size = get_tensor_model_parallel_world_size()
- self.total_num_heads = config.num_attention_heads
- assert self.total_num_heads % tp_size == 0
- self.num_heads = self.total_num_heads // tp_size
- self.total_num_kv_heads = config.num_key_value_heads
- if self.total_num_kv_heads >= tp_size:
- assert self.total_num_kv_heads % tp_size == 0
- else:
- assert tp_size % self.total_num_kv_heads == 0
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
- self.head_dim = self.hidden_size // self.total_num_heads
- self.q_size = self.num_heads * self.head_dim
- self.kv_size = self.num_kv_heads * self.head_dim
-
- self.max_position_embeddings = config.max_position_embeddings
- self.scaling = self.head_dim**-0.5
-
- self.qkv_proj = QKVParallelLinear(
- self.hidden_size,
- self.head_dim,
- self.total_num_heads,
- self.total_num_kv_heads,
- bias=False,
- quant_config=quant_config,
- prefix=f"{prefix}.qkv_proj",
- )
- self.o_proj = RowParallelLinear(
- self.total_num_heads * self.head_dim,
- self.hidden_size,
- bias=False,
- reduce_results=True,
- quant_config=quant_config,
- prefix=f"{prefix}.o_proj",
- )
-
- self.rotary_emb = get_rope(
- self.head_dim,
- max_position=self.max_position_embeddings,
- rope_parameters=config.rope_parameters,
- is_neox_style=True,
- )
-
- self.attn = Attention(
- self.num_heads,
- self.head_dim,
- self.scaling,
- num_kv_heads=self.num_kv_heads,
- cache_config=cache_config,
- quant_config=quant_config,
- prefix=f"{prefix}.attn",
- )
-
- def forward(
- self,
- positions: torch.Tensor,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor:
- qkv, _ = self.qkv_proj(hidden_states)
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
- q, k = self.rotary_emb(positions, q, k)
- attn_output = self.attn(q, k, v)
- output, _ = self.o_proj(attn_output)
- return output
-
-
-class ArcticDecoderLayer(nn.Module):
- def __init__(
- self,
- config: ArcticConfig,
- cache_config: CacheConfig | None = None,
- quant_config: QuantizationConfig | None = None,
- prefix: str = "",
- ) -> None:
- super().__init__()
- self.hidden_size = config.hidden_size
- layer_idx = extract_layer_index(prefix)
- is_moe_layer = (layer_idx + 1) % config.moe_layer_frequency == 0
- self.use_residual = config.use_residual and is_moe_layer
- self.self_attn = ArcticAttention(
- config,
- cache_config,
- quant_config=quant_config,
- prefix=f"{prefix}.self_attn",
- )
- self.block_sparse_moe = ArcticMoE(
- config,
- quant_config=quant_config,
- reduce_results=(not self.use_residual),
- prefix=f"{prefix}.block_sparse_moe",
- )
-
- self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
- self.post_attention_layernorm = RMSNorm(
- config.hidden_size, eps=config.rms_norm_eps
- )
-
- if self.use_residual:
- self.residual_layernorm = RMSNorm(
- config.hidden_size, eps=config.rms_norm_eps
- )
- self.residual_mlp = ArcticMLP(
- config,
- is_residual_mlp=True,
- reduce_results=False,
- prefix=f"{prefix}.residual_mlp",
- )
-
- def forward(
- self,
- positions: torch.Tensor,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor:
- residual_input = hidden_states
- hidden_states = self.input_layernorm(hidden_states)
- hidden_states = self.self_attn(
- positions=positions,
- hidden_states=hidden_states,
- )
- hidden_states = residual_input + hidden_states
-
- residual_attn = hidden_states
- if self.use_residual:
- hidden_states = self.residual_layernorm(hidden_states)
- hidden_states = self.residual_mlp(hidden_states)
- residual_mlp = hidden_states
- hidden_states = self.post_attention_layernorm(residual_input)
- hidden_states = self.block_sparse_moe(hidden_states)
- hidden_states = residual_mlp + hidden_states
- hidden_states = tensor_model_parallel_all_reduce(hidden_states)
- hidden_states = residual_attn + hidden_states
- else:
- hidden_states = self.post_attention_layernorm(hidden_states)
- hidden_states = self.block_sparse_moe(hidden_states)
- hidden_states = residual_attn + hidden_states
- return hidden_states
-
-
-@support_torch_compile
-class ArcticModel(nn.Module):
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
-
- config = vllm_config.model_config.hf_config
- cache_config = vllm_config.cache_config
- quant_config = vllm_config.quant_config
-
- self.config = config
- self.vocab_size = config.vocab_size
- self.embed_tokens = VocabParallelEmbedding(
- self.vocab_size, config.hidden_size, org_num_embeddings=self.vocab_size
- )
- self.start_layer, self.end_layer, self.layers = make_layers(
- config.num_hidden_layers,
- lambda prefix: ArcticDecoderLayer(
- config, cache_config, quant_config, prefix=prefix
- ),
- prefix=f"{prefix}.layers",
- )
- self._attn_implementation = config._attn_implementation
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
- self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
- ["hidden_states"], config.hidden_size
- )
-
- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
- return self.embed_tokens(input_ids)
-
- def forward(
- self,
- input_ids: torch.Tensor | None,
- positions: torch.Tensor,
- intermediate_tensors: IntermediateTensors | None,
- inputs_embeds: torch.Tensor | None = None,
- ) -> torch.Tensor | IntermediateTensors:
- if get_pp_group().is_first_rank:
- if inputs_embeds is not None:
- hidden_states = inputs_embeds
- else:
- hidden_states = self.embed_input_ids(input_ids)
- else:
- assert intermediate_tensors is not None
- hidden_states = intermediate_tensors["hidden_states"]
- for layer in islice(self.layers, self.start_layer, self.end_layer):
- hidden_states = layer(positions, hidden_states)
- if not get_pp_group().is_last_rank:
- return IntermediateTensors({"hidden_states": hidden_states})
- hidden_states = self.norm(hidden_states)
- return hidden_states
-
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
- stacked_params_mapping = [
- # (param_name, shard_name, shard_id)
- ("qkv_proj", "q_proj", "q"),
- ("qkv_proj", "k_proj", "k"),
- ("qkv_proj", "v_proj", "v"),
- ]
-
- mlp_params_mapping: list[tuple[str, str, int]] = []
- expert_params_mapping: list[tuple[str, str, int]] = []
-
- for layer in range(self.config.num_hidden_layers):
- is_moe_layer = (layer + 1) % self.config.moe_layer_frequency == 0
- if is_moe_layer and self.config.use_residual:
- mlp_params_mapping.append(
- (
- f"layers.{layer}.residual_mlp.w13.weight",
- f"layers.{layer}.residual_mlp.w1.weight",
- 0,
- )
- )
- mlp_params_mapping.append(
- (
- f"layers.{layer}.residual_mlp.w13.weight",
- f"layers.{layer}.residual_mlp.w3.weight",
- 1,
- )
- )
-
- if is_moe_layer:
- for expert_id in range(self.config.num_local_experts):
- expert_params_mapping.append(
- ("ws", f"experts.{expert_id}.w1.weight", expert_id)
- )
- expert_params_mapping.append(
- ("w2s", f"experts.{expert_id}.w2.weight", expert_id)
- )
- expert_params_mapping.append(
- ("ws", f"experts.{expert_id}.w3.weight", expert_id)
- )
- else:
- mlp_params_mapping.append(
- (
- f"layers.{layer}.block_sparse_moe.mlp.w13.weight",
- f"layers.{layer}.block_sparse_moe.mlp.w1.weight",
- 0,
- )
- )
- mlp_params_mapping.append(
- (
- f"layers.{layer}.block_sparse_moe.mlp.w13.weight",
- f"layers.{layer}.block_sparse_moe.mlp.w3.weight",
- 1,
- )
- )
-
- params_dict = dict(self.named_parameters())
- loaded_params: set[str] = set()
-
- for name, loaded_weight in weights:
- for param_name, weight_name, shard_id in stacked_params_mapping:
- if weight_name not in name:
- continue
- name = name.replace(weight_name, param_name)
- # Skip loading extra bias for GPTQ models.
- if name.endswith(".bias") and name not in params_dict:
- continue
- if is_pp_missing_parameter(name, self):
- continue
- param = params_dict[name]
- weight_loader = param.weight_loader
- weight_loader(param, loaded_weight, shard_id)
- break
- else:
- for param_name, weight_name, mlp_shard_id in mlp_params_mapping:
- if weight_name not in name:
- continue
- name = name.replace(weight_name, param_name)
- if is_pp_missing_parameter(name, self):
- continue
- param = params_dict[name]
- weight_loader = param.weight_loader
- weight_loader(param, loaded_weight, mlp_shard_id)
- break
- else:
- for param_name, weight_name, expert_id in expert_params_mapping:
- if weight_name not in name:
- continue
- name = name.replace(weight_name, param_name)
- if is_pp_missing_parameter(name, self):
- continue
- param = params_dict[name]
- weight_loader = param.weight_loader
- weight_loader(
- param, loaded_weight, weight_name, expert_id=expert_id
- )
- break
- else:
- if name.endswith(".bias") and name not in params_dict:
- continue
- if is_pp_missing_parameter(name, self):
- continue
- param = params_dict[name]
- weight_loader = getattr(
- param, "weight_loader", default_weight_loader
- )
- weight_loader(param, loaded_weight)
- loaded_params.add(name)
- return loaded_params
-
-
-class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant):
- packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
-
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
- config = vllm_config.model_config.hf_config
- quant_config = vllm_config.quant_config
- self.config = config
- self.model = ArcticModel(
- vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
- )
- self.vocab_size = config.vocab_size
- self.lm_head = ParallelLMHead(
- self.vocab_size,
- config.hidden_size,
- quant_config=quant_config,
- prefix=maybe_prefix(prefix, "lm_head"),
- )
- if self.config.tie_word_embeddings:
- self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
- self.num_experts = config.num_local_experts
- self.num_experts_per_tok = config.num_experts_per_tok
-
- self.logits_processor = LogitsProcessor(config.vocab_size)
- self.make_empty_intermediate_tensors = (
- self.model.make_empty_intermediate_tensors
- )
-
- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
- return self.model.embed_input_ids(input_ids)
-
- def forward(
- self,
- input_ids: torch.Tensor | None,
- positions: torch.Tensor,
- intermediate_tensors: IntermediateTensors | None = None,
- inputs_embeds: torch.Tensor | None = None,
- ) -> torch.Tensor | IntermediateTensors:
- hidden_states = self.model(
- input_ids, positions, intermediate_tensors, inputs_embeds
- )
- return hidden_states
-
- def compute_logits(
- self,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor | None:
- logits = self.logits_processor(self.lm_head, hidden_states)
- return logits
-
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
- loader = AutoWeightsLoader(self)
- return loader.load_weights(weights)
diff --git a/vllm/model_executor/models/chameleon.py b/vllm/model_executor/models/chameleon.py
deleted file mode 100644
index fa9cc13f0b..0000000000
--- a/vllm/model_executor/models/chameleon.py
+++ /dev/null
@@ -1,1043 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-from collections.abc import Iterable, Mapping, Sequence
-from functools import cached_property
-from itertools import islice
-from typing import Annotated, Any, Literal
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from transformers import (
- BatchFeature,
- ChameleonConfig,
- ChameleonProcessor,
- ChameleonVQVAEConfig,
-)
-
-from vllm.config import CacheConfig, VllmConfig
-from vllm.config.multimodal import BaseDummyOptions
-from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
-from vllm.inputs import MultiModalDataDict
-from vllm.logger import init_logger
-from vllm.model_executor.layers.activation import SiluAndMul
-from vllm.model_executor.layers.attention import Attention
-from vllm.model_executor.layers.conv import Conv2dLayer
-from vllm.model_executor.layers.layernorm import RMSNorm
-from vllm.model_executor.layers.linear import (
- MergedColumnParallelLinear,
- QKVParallelLinear,
- RowParallelLinear,
-)
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
-from vllm.model_executor.layers.quantization import QuantizationConfig
-from vllm.model_executor.layers.rotary_embedding import get_rope
-from vllm.model_executor.layers.vocab_parallel_embedding import (
- ParallelLMHead,
- VocabParallelEmbedding,
-)
-from vllm.model_executor.model_loader.weight_utils import row_parallel_weight_loader
-from vllm.model_executor.utils import set_weight_attrs
-from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.inputs import (
- MultiModalFieldConfig,
- MultiModalKwargsItems,
-)
-from vllm.multimodal.parse import MultiModalDataItems
-from vllm.multimodal.processing import (
- BaseDummyInputsBuilder,
- BaseMultiModalProcessor,
- BaseProcessingInfo,
- PromptReplacement,
- PromptUpdate,
- PromptUpdateDetails,
-)
-from vllm.sequence import IntermediateTensors
-from vllm.utils.tensor_schema import TensorSchema, TensorShape
-from vllm.utils.torch_utils import async_tensor_h2d
-
-from .interfaces import (
- MultiModalEmbeddings,
- SupportsMultiModal,
- SupportsPP,
- SupportsQuant,
-)
-from .utils import (
- AutoWeightsLoader,
- WeightsMapper,
- make_empty_intermediate_tensors_factory,
- make_layers,
- maybe_prefix,
-)
-
-logger = init_logger(__name__)
-
-
-class ChameleonImagePixelInputs(TensorSchema):
- """
- Dimensions:
- - bn: Batch size * number of images
- - c: Number of channels (3)
- - h: Height of each image
- - w: Width of each image
- """
-
- type: Literal["pixel_values"]
- data: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")]
-
-
-class ChameleonProcessingInfo(BaseProcessingInfo):
- def get_hf_config(self):
- return self.ctx.get_hf_config(ChameleonConfig)
-
- def get_hf_processor(self, **kwargs: object):
- return self.ctx.get_hf_processor(ChameleonProcessor, **kwargs)
-
- def get_supported_mm_limits(self) -> Mapping[str, int | None]:
- return {"image": 1}
-
- def get_num_image_tokens(self) -> int:
- processor = self.get_hf_processor()
- return processor.image_seq_length
-
-
-class ChameleonDummyInputsBuilder(BaseDummyInputsBuilder[ChameleonProcessingInfo]):
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
- num_images = mm_counts.get("image", 0)
-
- processor = self.info.get_hf_processor()
- image_token = processor.image_token
-
- return image_token * num_images
-
- def get_dummy_mm_data(
- self,
- seq_len: int,
- mm_counts: Mapping[str, int],
- mm_options: Mapping[str, BaseDummyOptions],
- ) -> MultiModalDataDict:
- config = self.info.get_hf_config()
-
- width = height = config.vq_config.resolution
- num_images = mm_counts.get("image", 0)
-
- image_overrides = mm_options.get("image")
-
- return {
- "image": self._get_dummy_images(
- width=width,
- height=height,
- num_images=num_images,
- overrides=image_overrides,
- )
- }
-
-
-class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingInfo]):
- def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
- # ChameleonProcessor requires text corresponding to the images
- return self.dummy_inputs.get_dummy_text(mm_counts)
-
- def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
- # HF processor adds sep token for chat mode
- tokenizer = self.info.get_tokenizer()
- sep_token_id = tokenizer.get_vocab()[tokenizer.sep_token] # type: ignore
-
- return [*prompt, sep_token_id]
-
- def _get_mm_fields_config(
- self,
- hf_inputs: BatchFeature,
- hf_processor_mm_kwargs: Mapping[str, object],
- ) -> Mapping[str, MultiModalFieldConfig]:
- return dict(pixel_values=MultiModalFieldConfig.batched("image"))
-
- def _get_prompt_updates(
- self,
- mm_items: MultiModalDataItems,
- hf_processor_mm_kwargs: Mapping[str, object],
- out_mm_kwargs: MultiModalKwargsItems,
- ) -> Sequence[PromptUpdate]:
- processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
- tokenizer = self.info.get_tokenizer()
- vocab = tokenizer.get_vocab()
-
- image_start_id = vocab[processor.image_start_token]
- image_token_id = vocab[processor.image_token]
- image_end_id = vocab[processor.image_end_token]
-
- num_image_tokens = self.info.get_num_image_tokens()
- image_tokens = [image_token_id] * num_image_tokens
-
- return [
- PromptReplacement(
- modality="image",
- target=[image_token_id],
- replacement=PromptUpdateDetails.select_token_id(
- [image_start_id] + image_tokens + [image_end_id],
- embed_token_id=image_token_id,
- ),
- )
- ]
-
-
-class ChameleonLayerNorm(nn.LayerNorm):
- def __init__(self, hidden_size, *args, **kwargs):
- super().__init__(hidden_size, *args, **kwargs)
- self.normalized_shape = (hidden_size[-1],)
-
- set_weight_attrs(self.weight, {"weight_loader": row_parallel_weight_loader})
- set_weight_attrs(self.bias, {"weight_loader": row_parallel_weight_loader})
-
- def forward(self, hidden_states):
- hidden_states = F.layer_norm(
- hidden_states, self.normalized_shape, None, None, eps=1e-5
- )
- hidden_states = hidden_states * self.weight + self.bias
- return hidden_states
-
-
-# Copied from vllm.model_executor.models.llama.LlamaMLP -> ChameleonMLP
-class ChameleonMLP(nn.Module):
- def __init__(
- self,
- hidden_size: int,
- intermediate_size: int,
- hidden_act: str,
- quant_config: QuantizationConfig | None = None,
- bias: bool = False,
- prefix: str = "",
- ) -> None:
- super().__init__()
- self.gate_up_proj = MergedColumnParallelLinear(
- input_size=hidden_size,
- output_sizes=[intermediate_size] * 2,
- bias=bias,
- quant_config=quant_config,
- prefix=f"{prefix}.gate_up_proj",
- )
- self.down_proj = RowParallelLinear(
- input_size=intermediate_size,
- output_size=hidden_size,
- bias=bias,
- quant_config=quant_config,
- prefix=f"{prefix}.down_proj",
- )
- if hidden_act != "silu":
- raise ValueError(
- f"Unsupported activation: {hidden_act}. Only silu is supported for now."
- )
- self.act_fn = SiluAndMul()
-
- def forward(self, x):
- gate_up, _ = self.gate_up_proj(x)
- x = self.act_fn(gate_up)
- x, _ = self.down_proj(x)
- return x
-
-
-# Modified from vllm.model_executor.models.llama.LlamaAttention -> ChameleonAttention #noqa
-class ChameleonAttention(nn.Module):
- def __init__(
- self,
- hidden_size: int,
- num_heads: int,
- num_kv_heads: int,
- rope_parameters: dict[str, Any],
- max_position_embeddings: int = 4096,
- quant_config: QuantizationConfig | None = None,
- bias: bool = False,
- cache_config: CacheConfig | None = None,
- prefix: str = "",
- ) -> None:
- super().__init__()
- self.hidden_size = hidden_size
- tp_size = get_tensor_model_parallel_world_size()
- self.total_num_heads = num_heads
- assert self.total_num_heads % tp_size == 0
- self.num_heads = self.total_num_heads // tp_size
- self.total_num_kv_heads = num_kv_heads
- if self.total_num_kv_heads >= tp_size:
- # Number of KV heads is greater than TP size, so we partition
- # the KV heads across multiple tensor parallel GPUs.
- assert self.total_num_kv_heads % tp_size == 0
- else:
- # Number of KV heads is less than TP size, so we replicate
- # the KV heads across multiple tensor parallel GPUs.
- assert tp_size % self.total_num_kv_heads == 0
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
- self.head_dim = hidden_size // self.total_num_heads
- self.q_size = self.num_heads * self.head_dim
- self.kv_size = self.num_kv_heads * self.head_dim
- self.scaling = self.head_dim**-0.5
- self.max_position_embeddings = max_position_embeddings
-
- self.qkv_proj = QKVParallelLinear(
- hidden_size=hidden_size,
- head_size=self.head_dim,
- total_num_heads=self.total_num_heads,
- total_num_kv_heads=self.total_num_kv_heads,
- bias=bias,
- quant_config=quant_config,
- prefix=f"{prefix}.qkv_proj",
- )
- self.o_proj = RowParallelLinear(
- input_size=self.total_num_heads * self.head_dim,
- output_size=hidden_size,
- bias=bias,
- quant_config=quant_config,
- prefix=f"{prefix}.o_proj",
- )
- self.q_norm = ChameleonLayerNorm((self.num_heads, self.head_dim))
- self.k_norm = ChameleonLayerNorm((self.num_kv_heads, self.head_dim))
- self.rotary_emb = get_rope(
- self.head_dim,
- max_position=max_position_embeddings,
- rope_parameters=rope_parameters,
- )
-
- self.attn = Attention(
- self.num_heads,
- self.head_dim,
- self.scaling,
- num_kv_heads=self.num_kv_heads,
- cache_config=cache_config,
- quant_config=quant_config,
- prefix=f"{prefix}.attn",
- )
-
- def _apply_qk_norm(
- self, q: torch.Tensor, k: torch.Tensor
- ) -> tuple[torch.Tensor, torch.Tensor]:
- # reshape for layernorm
- q = q.reshape(-1, self.num_heads, self.head_dim)
- k = k.reshape(-1, self.num_kv_heads, self.head_dim)
- q = self.q_norm(q)
- k = self.k_norm(k)
- q = q.view(*q.shape[:-2], -1)
- k = k.view(*k.shape[:-2], -1)
- return q, k
-
- def forward(
- self,
- positions: torch.Tensor,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor:
- qkv, _ = self.qkv_proj(hidden_states)
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
- q, k = self._apply_qk_norm(q, k)
-
- q, k = self.rotary_emb(positions, q, k)
- attn_output = self.attn(q, k, v)
- output, _ = self.o_proj(attn_output)
- return output
-
-
-class ChameleonDecoderLayer(nn.Module):
- def __init__(
- self,
- config: ChameleonConfig,
- cache_config: CacheConfig | None = None,
- quant_config: QuantizationConfig | None = None,
- prefix: str = "",
- ) -> None:
- super().__init__()
- self.hidden_size = config.hidden_size
- max_position_embeddings = getattr(config, "max_position_embeddings", 4096)
-
- self.self_attn = ChameleonAttention(
- hidden_size=self.hidden_size,
- num_heads=config.num_attention_heads,
- num_kv_heads=getattr(
- config, "num_key_value_heads", config.num_attention_heads
- ),
- rope_parameters=config.rope_parameters,
- max_position_embeddings=max_position_embeddings,
- quant_config=quant_config,
- bias=False,
- cache_config=cache_config,
- prefix=f"{prefix}.self_attn",
- )
- self.mlp = ChameleonMLP(
- hidden_size=self.hidden_size,
- intermediate_size=config.intermediate_size,
- hidden_act=config.hidden_act,
- quant_config=quant_config,
- bias=getattr(config, "mlp_bias", False),
- prefix=f"{prefix}.mlp",
- )
- self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
- self.post_attention_layernorm = RMSNorm(
- config.hidden_size, eps=config.rms_norm_eps
- )
-
- def forward(
- self,
- positions: torch.Tensor,
- hidden_states: torch.Tensor,
- residual: torch.Tensor | None,
- ) -> tuple[torch.Tensor, torch.Tensor | None]:
- if residual is None:
- residual = hidden_states
- hidden_states = self.input_layernorm(hidden_states)
- else:
- hidden_states, residual = self.input_layernorm(hidden_states, residual)
- hidden_states = self.self_attn(
- positions=positions,
- hidden_states=hidden_states,
- )
-
- # Fully Connected
- hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
- hidden_states = self.mlp(hidden_states)
-
- return hidden_states, residual
-
-
-class ChameleonSwinDecoderLayer(nn.Module):
- def __init__(
- self,
- config: ChameleonConfig,
- cache_config: CacheConfig | None = None,
- quant_config: QuantizationConfig | None = None,
- prefix: str = "",
- ) -> None:
- super().__init__()
- self.hidden_size = config.hidden_size
- max_position_embeddings = getattr(config, "max_position_embeddings", 4096)
-
- self.self_attn = ChameleonAttention(
- hidden_size=self.hidden_size,
- num_heads=config.num_attention_heads,
- num_kv_heads=getattr(
- config, "num_key_value_heads", config.num_attention_heads
- ),
- rope_parameters=config.rope_parameters,
- max_position_embeddings=max_position_embeddings,
- quant_config=quant_config,
- bias=False,
- cache_config=cache_config,
- prefix=f"{prefix}.self_attn",
- )
- self.mlp = ChameleonMLP(
- hidden_size=self.hidden_size,
- intermediate_size=config.intermediate_size,
- hidden_act=config.hidden_act,
- quant_config=quant_config,
- bias=getattr(config, "mlp_bias", False),
- prefix=f"{prefix}.mlp",
- )
- self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
- self.post_attention_layernorm = RMSNorm(
- config.hidden_size, eps=config.rms_norm_eps
- )
-
- def forward(
- self,
- positions: torch.Tensor,
- hidden_states: torch.Tensor,
- residual: torch.Tensor | None,
- ) -> tuple[torch.Tensor, torch.Tensor]:
- residual = hidden_states
- hidden_states = self.self_attn(
- positions=positions,
- hidden_states=hidden_states,
- )
-
- hidden_states = self.input_layernorm(hidden_states)
- hidden_states = hidden_states + residual
-
- # Fully Connected
- residual = hidden_states
- hidden_states = self.mlp(hidden_states)
- hidden_states = self.post_attention_layernorm(hidden_states)
- hidden_states = residual + hidden_states
-
- return hidden_states, residual
-
-
-# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEVectorQuantizer #noqa
-class ChameleonVQVAEVectorQuantizer(nn.Module):
- def __init__(self, config: ChameleonVQVAEConfig):
- super().__init__()
- self.num_embeddings = config.num_embeddings
- self.embedding_dim = config.embed_dim
- self.beta = getattr(config, "beta", 0.25)
-
- self.embedding = nn.Embedding(self.num_embeddings, self.embedding_dim)
- self.re_embed = self.num_embeddings
-
- def forward(self, hidden_state: torch.Tensor):
- hidden_state = hidden_state.permute(0, 2, 3, 1).contiguous()
- hidden_state_flattened = hidden_state.view(-1, self.embedding_dim)
-
- # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z
- distances = (
- torch.sum(hidden_state_flattened**2, dim=1, keepdim=True)
- + torch.sum(self.embedding.weight**2, dim=1)
- - 2
- * torch.einsum(
- "bd,dn->bn",
- hidden_state_flattened,
- self.embedding.weight.transpose(0, 1),
- )
- )
-
- min_encoding_indices = torch.argmin(distances, dim=1)
- hidden_state_quant = self.embedding(min_encoding_indices).view(
- hidden_state.shape
- )
-
- # compute loss for embedding
- loss = torch.mean(
- (hidden_state_quant.detach() - hidden_state) ** 2
- ) + self.beta * torch.mean((hidden_state_quant - hidden_state.detach()) ** 2)
-
- # preserve gradients
- hidden_state_quant = hidden_state + (hidden_state_quant - hidden_state).detach()
-
- # reshape back to match original input shape
- hidden_state_quant = hidden_state_quant.permute(0, 3, 1, 2).contiguous()
-
- return hidden_state_quant, loss, min_encoding_indices
-
-
-# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderConvDownsample #noqa
-class ChameleonVQVAEEncoderConvDownsample(nn.Module):
- def __init__(self, in_channels: int):
- super().__init__()
- self.conv = Conv2dLayer(
- in_channels, in_channels, kernel_size=3, stride=2, padding=0
- )
-
- def forward(self, hidden_states: torch.Tensor):
- # no asymmetric padding in torch conv, must do it ourselves
- hidden_states = F.pad(hidden_states, pad=(0, 1, 0, 1), mode="constant", value=0)
- hidden_states = self.conv(hidden_states)
- return hidden_states
-
-
-# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderResnetBlock #noqa
-class ChameleonVQVAEEncoderResnetBlock(nn.Module):
- def __init__(
- self,
- config: ChameleonVQVAEConfig,
- in_channels: int,
- out_channels=None,
- conv_shortcut=False,
- ):
- super().__init__()
- self.in_channels = in_channels
- self.out_channels = in_channels if out_channels is None else out_channels
- self.use_conv_shortcut = conv_shortcut
-
- self.norm1 = torch.nn.GroupNorm(
- num_groups=32, num_channels=in_channels, eps=1e-6, affine=True
- )
- self.conv1 = Conv2dLayer(
- in_channels, out_channels, kernel_size=3, stride=1, padding=1
- )
- self.norm2 = torch.nn.GroupNorm(
- num_groups=32, num_channels=out_channels, eps=1e-6, affine=True
- )
- self.dropout = torch.nn.Dropout(config.dropout)
- self.conv2 = Conv2dLayer(
- out_channels, out_channels, kernel_size=3, stride=1, padding=1
- )
- if self.in_channels != self.out_channels:
- if self.use_conv_shortcut:
- self.conv_shortcut = Conv2dLayer(
- in_channels, out_channels, kernel_size=3, stride=1, padding=1
- )
- else:
- self.nin_shortcut = Conv2dLayer(
- in_channels, out_channels, kernel_size=1, stride=1, padding=0
- )
-
- def forward(self, hidden_states: torch.Tensor):
- residual = hidden_states
- hidden_states = self.norm1(hidden_states)
- hidden_states *= torch.sigmoid(hidden_states)
- hidden_states = self.conv1(hidden_states)
-
- hidden_states = self.norm2(hidden_states)
- hidden_states *= torch.sigmoid(hidden_states)
- hidden_states = self.dropout(hidden_states)
- hidden_states = self.conv2(hidden_states)
-
- if self.in_channels != self.out_channels:
- if self.use_conv_shortcut:
- residual = self.conv_shortcut(residual)
- else:
- residual = self.nin_shortcut(residual)
-
- return residual + hidden_states
-
-
-# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoderAttnBlock #noqa
-class ChameleonVQVAEEncoderAttnBlock(nn.Module):
- def __init__(self, in_channels: int):
- super().__init__()
- self.in_channels = in_channels
-
- self.norm = torch.nn.GroupNorm(
- num_groups=32, num_channels=in_channels, eps=1e-6, affine=True
- )
- self.q = Conv2dLayer(
- in_channels, in_channels, kernel_size=1, stride=1, padding=0
- )
- self.k = Conv2dLayer(
- in_channels, in_channels, kernel_size=1, stride=1, padding=0
- )
- self.v = Conv2dLayer(
- in_channels, in_channels, kernel_size=1, stride=1, padding=0
- )
- self.proj_out = Conv2dLayer(
- in_channels, in_channels, kernel_size=1, stride=1, padding=0
- )
-
- def forward(self, hidden_states: torch.Tensor):
- residual = hidden_states
- hidden_states = self.norm(hidden_states)
- query_states = self.q(hidden_states)
- key_states = self.k(hidden_states)
- value_states = self.v(hidden_states)
-
- # compute attention
- batch_size, channels, height, width = query_states.shape
- query_states = query_states.reshape(
- batch_size, channels, height * width
- ).permute(0, 2, 1)
- key_states = key_states.reshape(batch_size, channels, height * width)
- attn_weights = torch.bmm(query_states, key_states)
- attn_weights = attn_weights * (int(channels) ** (-0.5))
- attn_weights = F.softmax(attn_weights, dim=2)
-
- # attend to values
- value_states = value_states.reshape(batch_size, channels, height * width)
- attn_weights = attn_weights.permute(0, 2, 1)
- attn_output = torch.bmm(value_states, attn_weights).reshape(
- batch_size, channels, height, width
- )
-
- attn_output = self.proj_out(attn_output)
- return residual + attn_output
-
-
-# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAEEncoder #noqa
-class ChameleonVQVAEEncoder(nn.Module):
- def __init__(self, config: ChameleonVQVAEConfig):
- super().__init__()
-
- self.num_resolutions = len(config.channel_multiplier)
- self.num_res_blocks = config.num_res_blocks
- base_channels = config.base_channels
- resolution = config.resolution
- in_channels = config.in_channels
- double_latent = config.double_latent
- latent_channels = config.latent_channels
- channel_multiplier = config.channel_multiplier
-
- self.conv_in = Conv2dLayer(
- in_channels, base_channels, kernel_size=3, stride=1, padding=1
- )
-
- curr_res = resolution
- in_channel_multiplier = (1,) + tuple(channel_multiplier)
- self.in_channel_multiplier = in_channel_multiplier
- self.down = nn.ModuleList()
- for i_level in range(self.num_resolutions):
- block = nn.ModuleList()
- attn = nn.ModuleList()
- block_in = base_channels * in_channel_multiplier[i_level]
- block_out = base_channels * channel_multiplier[i_level]
- for i_block in range(self.num_res_blocks):
- block.append(
- ChameleonVQVAEEncoderResnetBlock(
- config=config,
- in_channels=block_in,
- out_channels=block_out,
- )
- )
- block_in = block_out
- if (
- config.attn_resolutions is not None
- and curr_res in config.attn_resolutions
- and config.attn_type == "vanilla"
- ):
- attn.append(ChameleonVQVAEEncoderAttnBlock(block_in))
-
- down = nn.Module()
- down.block = block
- down.attn = attn
- if i_level != self.num_resolutions - 1:
- down.downsample = ChameleonVQVAEEncoderConvDownsample(block_in)
- curr_res = curr_res // 2
- self.down.append(down)
-
- self.mid = nn.Module()
- self.mid.block_1 = ChameleonVQVAEEncoderResnetBlock(
- config=config,
- in_channels=block_in,
- out_channels=block_in,
- )
- self.mid.attn_1 = (
- ChameleonVQVAEEncoderAttnBlock(block_in)
- if config.attn_type == "vanilla"
- else nn.Identity()
- )
- self.mid.block_2 = ChameleonVQVAEEncoderResnetBlock(
- config=config,
- in_channels=block_in,
- out_channels=block_in,
- )
-
- self.norm_out = torch.nn.GroupNorm(
- num_groups=32, num_channels=block_in, eps=1e-6, affine=True
- )
- self.conv_out = Conv2dLayer(
- block_in,
- 2 * latent_channels if double_latent else latent_channels,
- kernel_size=3,
- stride=1,
- padding=1,
- )
-
- def forward(self, pixel_values: torch.Tensor):
- pixel_values = pixel_values.to(self.conv_in.weight.dtype)
-
- # downsampling
- hidden_states = [self.conv_in(pixel_values)]
- for i_level in range(self.num_resolutions):
- for i_block in range(self.num_res_blocks):
- hidden_state = self.down[i_level].block[i_block](hidden_states[-1])
- if len(self.down[i_level].attn) > 0:
- hidden_state = self.down[i_level].attn[i_block](hidden_state)
- hidden_states.append(hidden_state)
- if i_level != self.num_resolutions - 1:
- hidden_states.append(self.down[i_level].downsample(hidden_states[-1]))
-
- # middle
- last_hidden_state = hidden_states[-1]
- last_hidden_state = self.mid.block_1(last_hidden_state)
- last_hidden_state = self.mid.attn_1(last_hidden_state)
- last_hidden_state = self.mid.block_2(last_hidden_state)
-
- # end
- last_hidden_state = self.norm_out(last_hidden_state)
- last_hidden_state *= torch.sigmoid(last_hidden_state)
- last_hidden_state = self.conv_out(last_hidden_state)
- return last_hidden_state
-
-
-# Adapted from transformers.models.chameleon.modeling_chameleon.ChameleonVQVAE #noqa
-class ChameleonVQVAE(nn.Module):
- def __init__(self, config: ChameleonVQVAEConfig):
- super().__init__()
- self.encoder = ChameleonVQVAEEncoder(config)
- self.quantize = ChameleonVQVAEVectorQuantizer(config)
- self.quant_conv = Conv2dLayer(config.latent_channels, config.embed_dim, 1)
- self.post_quant_conv = Conv2dLayer(config.embed_dim, config.latent_channels, 1)
- self.eval() # Chameleon's VQ model is frozen
-
- def encode(
- self, pixel_values: torch.Tensor
- ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
- hidden_states = self.encoder(pixel_values)
- hidden_states = self.quant_conv(hidden_states)
- quant, emb_loss, indices = self.quantize(hidden_states)
- return quant, emb_loss, indices
-
-
-# Copied from transformers.models.chameleon.modeling_chameleon.ChameleonImageVocabularyMapping #noqa
-class ChameleonImageVocabularyMapping:
- """
- A class for mapping discrete image tokens from VQGAN to BPE tokens.
- """
-
- def __init__(self, vocab_map: dict[str, int]):
- self.vocab_map = vocab_map
- self.image_token_id = vocab_map.get("")
-
- @cached_property
- def val2name(self):
- return {v: k for k, v in self.vocab_map.items()}
-
- @cached_property
- def image_tokens(self):
- return sorted(
- [val for name, val in self.vocab_map.items() if name.startswith("IMGIMG")]
- )
-
- @cached_property
- def bpe2img(self):
- img_tkn_chr_mapping = {chr(ord("A") + i): str(i) for i in range(10)}
-
- def remap(old_name: str) -> str:
- return "".join(
- img_tkn_chr_mapping.get(c, c) for c in old_name[len("IMGIMG") : -1]
- )
-
- return {tok: int(remap(self.val2name[tok])) for tok in self.image_tokens}
-
- @cached_property
- def img2bpe(self):
- return {v: k for k, v in self.bpe2img.items()}
-
- @cached_property
- def bpe2img_search_tensors(self):
- return torch.tensor(sorted(self.bpe2img.keys())), torch.tensor(
- sorted(self.bpe2img.values())
- )
-
- @cached_property
- def img2bpe_mapping_tensor(self):
- mapping = torch.zeros(max(self.img2bpe.keys()) + 1, dtype=torch.int)
- for k, v in self.img2bpe.items():
- mapping[k] = v
- return mapping
-
- def convert_img2bpe(self, img_batch: torch.Tensor) -> torch.Tensor:
- device = img_batch.device
- # Cache a per-device copy of the (small, static) mapping tensor so we
- # can index entirely on `device` instead of forcing a D2H on
- # `img_batch` and an H2D on the result.
- cache = getattr(self, "_img2bpe_mapping_cache", None)
- if cache is None:
- cache = {}
- self._img2bpe_mapping_cache = cache
- mapping_on_device = cache.get(device)
- if mapping_on_device is None:
- mapping_on_device = async_tensor_h2d(
- self.img2bpe_mapping_tensor, device=device
- )
- cache[device] = mapping_on_device
- return mapping_on_device[img_batch]
-
-
-class ChameleonModel(nn.Module):
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
-
- config = vllm_config.model_config.hf_config
- cache_config = vllm_config.cache_config
- quant_config = vllm_config.quant_config
-
- self.config = config
- self.vocab_size = config.vocab_size
- self.embed_tokens = VocabParallelEmbedding(
- self.vocab_size,
- config.hidden_size,
- )
- self.vocabulary_mapping = ChameleonImageVocabularyMapping(config.vocabulary_map)
- decoder_layer = (
- ChameleonDecoderLayer
- if not self.config.swin_norm
- else ChameleonSwinDecoderLayer
- )
-
- self.start_layer, self.end_layer, self.layers = make_layers(
- config.num_hidden_layers,
- lambda prefix: decoder_layer(
- config=config,
- cache_config=cache_config,
- quant_config=quant_config,
- prefix=prefix,
- ),
- prefix=f"{prefix}.layers",
- )
-
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
- self.vqmodel = ChameleonVQVAE(config.vq_config)
- self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
- ["hidden_states", "residual"], config.hidden_size
- )
-
- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
- return self.embed_tokens(input_ids)
-
- def get_image_tokens(self, pixel_values: torch.Tensor) -> torch.Tensor:
- """
- Tokenizes images into discrete tokens with VQGAN module. Converts
- obtained image tokens into BPE tokens and wraps with "boi" and "eoi"
- special tokens.
- """
- batch_size = pixel_values.shape[0]
- _, _, image_toks = self.vqmodel.encode(pixel_values)
- bpe_toks = self.vocabulary_mapping.convert_img2bpe(image_toks)
- bpe_toks = bpe_toks.view(batch_size, -1)
- return bpe_toks
-
- def forward(
- self,
- input_ids: torch.Tensor | None,
- positions: torch.Tensor,
- intermediate_tensors: IntermediateTensors | None,
- inputs_embeds: torch.Tensor | None = None,
- ) -> torch.Tensor | IntermediateTensors:
- if get_pp_group().is_first_rank:
- if inputs_embeds is not None:
- hidden_states = inputs_embeds
- else:
- hidden_states = self.embed_input_ids(input_ids)
- residual = None
- else:
- assert intermediate_tensors is not None
- hidden_states = intermediate_tensors["hidden_states"]
- residual = intermediate_tensors["residual"]
- for layer in islice(self.layers, self.start_layer, self.end_layer):
- hidden_states, residual = layer(
- positions,
- hidden_states,
- residual,
- )
- if not get_pp_group().is_last_rank:
- return IntermediateTensors(
- {"hidden_states": hidden_states, "residual": residual}
- )
- hidden_states, _ = self.norm(hidden_states, residual)
- return hidden_states
-
-
-@MULTIMODAL_REGISTRY.register_processor(
- ChameleonMultiModalProcessor,
- info=ChameleonProcessingInfo,
- dummy_inputs=ChameleonDummyInputsBuilder,
-)
-class ChameleonForConditionalGeneration(
- nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant
-):
- packed_modules_mapping = {
- "qkv_proj": ["q_proj", "k_proj", "v_proj"],
- "gate_up_proj": ["gate_proj", "up_proj"],
- }
-
- hf_to_vllm_mapper = WeightsMapper(
- orig_to_new_stacked={
- ".q_proj": (".qkv_proj", "q"),
- ".k_proj": (".qkv_proj", "k"),
- ".v_proj": (".qkv_proj", "v"),
- ".gate_proj": (".gate_up_proj", 0),
- ".up_proj": (".gate_up_proj", 1),
- },
- )
-
- @classmethod
- def get_placeholder_str(cls, modality: str, i: int) -> str | None:
- if modality.startswith("image"):
- return ""
-
- raise ValueError("Only image modality is supported")
-
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
- config = vllm_config.model_config.hf_config
- multimodal_config = vllm_config.model_config.multimodal_config
- self.config = config
- self.multimodal_config = multimodal_config
-
- with self._mark_composite_model(
- vllm_config,
- language_targets=(
- ChameleonDecoderLayer
- if not self.config.swin_norm
- else ChameleonSwinDecoderLayer
- ),
- tower_targets={"image": ChameleonVQVAE},
- ):
- self.model = ChameleonModel(
- vllm_config=vllm_config,
- prefix=maybe_prefix(prefix, "model"),
- )
-
- self.lm_head = ParallelLMHead(
- config.vocab_size,
- config.hidden_size,
- prefix=maybe_prefix(prefix, "lm_head"),
- )
- if config.tie_word_embeddings:
- self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
-
- logit_scale = getattr(config, "logit_scale", 1.0)
- self.logits_processor = LogitsProcessor(config.vocab_size, scale=logit_scale)
- self.make_empty_intermediate_tensors = (
- self.model.make_empty_intermediate_tensors
- )
-
- def _parse_and_validate_image_input(
- self, **kwargs: object
- ) -> ChameleonImagePixelInputs | None:
- pixel_values = kwargs.pop("pixel_values", None)
-
- if pixel_values is None:
- return None
-
- vq_config: ChameleonVQVAEConfig = self.config.vq_config
- expected_h = expected_w = vq_config.resolution
-
- return ChameleonImagePixelInputs(
- type="pixel_values",
- data=pixel_values,
- resolve_bindings={"h": expected_h, "w": expected_w},
- )
-
- def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
- image_input = self._parse_and_validate_image_input(**kwargs)
- if image_input is None:
- return []
- assert self.model.vqmodel is not None
- image_tokens = self.model.get_image_tokens(
- image_input["data"].to(self.config.dtype)
- )
- vision_embeddings = self.model.embed_input_ids(image_tokens)
- return vision_embeddings
-
- def forward(
- self,
- input_ids: torch.Tensor | None,
- positions: torch.Tensor,
- intermediate_tensors: IntermediateTensors | None = None,
- inputs_embeds: torch.Tensor | None = None,
- **kwargs,
- ) -> torch.Tensor | IntermediateTensors:
- if intermediate_tensors is not None:
- inputs_embeds = None
-
- hidden_states = self.model(
- input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds
- )
- return hidden_states
-
- def compute_logits(
- self,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor | None:
- logits = self.logits_processor(self.lm_head, hidden_states)
-
- # Disallow image tokens which does not include special
- # begin-image and end-image tokens
- if logits is not None:
- # Cache a per-device index tensor for the (static) image-token
- # set, and use `index_fill_` instead of advanced-index assign
- # so the scatter runs entirely on device (no host roundtrip
- # for the scalar fill value or for the Python-list indices).
- cache = getattr(self, "_image_tokens_index_cache", None)
- if cache is None:
- cache = {}
- self._image_tokens_index_cache = cache
- image_tokens_idx = cache.get(logits.device)
- if image_tokens_idx is None:
- image_tokens_idx = async_tensor_h2d(
- self.model.vocabulary_mapping.image_tokens,
- dtype=torch.long,
- device=logits.device,
- )
- cache[logits.device] = image_tokens_idx
- logits.index_fill_(1, image_tokens_idx, torch.finfo(logits.dtype).min)
-
- return logits
-
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
- loader = AutoWeightsLoader(self)
- return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
diff --git a/vllm/model_executor/models/cheers.py b/vllm/model_executor/models/cheers.py
deleted file mode 100644
index 2b934f3867..0000000000
--- a/vllm/model_executor/models/cheers.py
+++ /dev/null
@@ -1,734 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Inference-only Cheers (UMM) model compatible with HuggingFace weights.
-
-Cheers is a unified multimodal model for image understanding and generation.
-For vLLM, we focus on the image understanding (vision-to-text) capabilities.
-The image generation part (gen_projector, hi_gate, etc.) is not supported,
-but the VAE encoder + decoder projector are required for image understanding.
-"""
-
-import math
-from collections.abc import Iterable, Mapping, Sequence
-from typing import Any, Literal, TypeAlias
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from einops import rearrange
-
-from vllm.config import VllmConfig
-from vllm.config.multimodal import BaseDummyOptions
-from vllm.inputs import MultiModalDataDict
-from vllm.logger import init_logger
-from vllm.model_executor.layers.quantization import QuantizationConfig
-from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.inputs import (
- MultiModalFieldConfig,
- MultiModalKwargsItems,
-)
-from vllm.multimodal.parse import MultiModalDataItems
-from vllm.multimodal.processing import (
- BaseDummyInputsBuilder,
- BaseMultiModalProcessor,
- BaseProcessingInfo,
- PromptReplacement,
-)
-from vllm.sequence import IntermediateTensors
-from vllm.transformers_utils.processors.cheers import CheersProcessor
-from vllm.utils.tensor_schema import TensorSchema
-
-from .interfaces import (
- MultiModalEmbeddings,
- SupportsLoRA,
- SupportsMultiModal,
- SupportsPP,
-)
-from .siglip import SiglipVisionModel
-from .utils import (
- AutoWeightsLoader,
- WeightsMapper,
- init_vllm_registered_model,
- maybe_prefix,
-)
-
-logger = init_logger(__name__)
-
-
-# ── VAE components (needed for image understanding pipeline) ────────
-
-
-def _swish(x: torch.Tensor) -> torch.Tensor:
- return x * torch.sigmoid(x)
-
-
-class _AttnBlock(nn.Module):
- def __init__(self, in_channels: int):
- super().__init__()
- self.norm = nn.GroupNorm(32, in_channels, eps=1e-6, affine=True)
- self.q = nn.Conv2d(in_channels, in_channels, 1)
- self.k = nn.Conv2d(in_channels, in_channels, 1)
- self.v = nn.Conv2d(in_channels, in_channels, 1)
- self.proj_out = nn.Conv2d(in_channels, in_channels, 1)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- h_ = self.norm(x)
- q = self.q(h_)
- k = self.k(h_)
- v = self.v(h_)
- b, c, h, w = q.shape
- q = rearrange(q, "b c h w -> b 1 (h w) c").contiguous()
- k = rearrange(k, "b c h w -> b 1 (h w) c").contiguous()
- v = rearrange(v, "b c h w -> b 1 (h w) c").contiguous()
- h_ = F.scaled_dot_product_attention(q, k, v)
- h_ = rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
- return x + self.proj_out(h_)
-
-
-class _ResnetBlock(nn.Module):
- def __init__(self, in_channels: int, out_channels: int):
- super().__init__()
- self.in_channels = in_channels
- self.out_channels = out_channels
- self.norm1 = nn.GroupNorm(32, in_channels, eps=1e-6, affine=True)
- self.conv1 = nn.Conv2d(in_channels, out_channels, 3, 1, 1)
- self.norm2 = nn.GroupNorm(32, out_channels, eps=1e-6, affine=True)
- self.conv2 = nn.Conv2d(out_channels, out_channels, 3, 1, 1)
- if in_channels != out_channels:
- self.nin_shortcut = nn.Conv2d(in_channels, out_channels, 1)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- h = _swish(self.norm1(x))
- h = self.conv1(h)
- h = _swish(self.norm2(h))
- h = self.conv2(h)
- if self.in_channels != self.out_channels:
- x = self.nin_shortcut(x)
- return x + h
-
-
-class _Downsample(nn.Module):
- def __init__(self, in_channels: int):
- super().__init__()
- self.conv = nn.Conv2d(in_channels, in_channels, 3, stride=2, padding=0)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- x = F.pad(x, (0, 1, 0, 1), mode="constant", value=0)
- return self.conv(x)
-
-
-class _Upsample(nn.Module):
- def __init__(self, in_channels: int):
- super().__init__()
- self.conv = nn.Conv2d(in_channels, in_channels, 3, 1, 1)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- x = F.interpolate(x, scale_factor=2.0, mode="nearest")
- return self.conv(x)
-
-
-_VAE_ENCODER_DEFAULTS = {
- "in_channels": 3,
- "ch": 128,
- "ch_mult": [1, 2, 4, 4],
- "num_res_blocks": 2,
- "z_channels": 32,
-}
-_VAE_DECODER_DEFAULTS = {
- "in_channels": 3,
- "out_ch": 3,
- "ch": 128,
- "ch_mult": [1, 2, 4, 4],
- "num_res_blocks": 2,
- "z_channels": 32,
-}
-
-
-def _cfg(config, key, defaults=None):
- """Access config attribute whether it's a dict or namespace object."""
- if isinstance(config, dict):
- if key in config:
- return config[key]
- if defaults and key in defaults:
- return defaults[key]
- raise KeyError(f"Key '{key}' not found in config dict: {list(config.keys())}")
- return getattr(config, key)
-
-
-class CheersVAEEncoder(nn.Module):
- """VAE encoder from the Cheers/UMM model."""
-
- def __init__(self, config):
- super().__init__()
- d = _VAE_ENCODER_DEFAULTS
- ch = _cfg(config, "ch", d)
- ch_mult = _cfg(config, "ch_mult", d)
- num_res_blocks = _cfg(config, "num_res_blocks", d)
- z_channels = _cfg(config, "z_channels", d)
- in_channels = _cfg(config, "in_channels", d)
- num_resolutions = len(ch_mult)
-
- self.quant_conv = nn.Conv2d(2 * z_channels, 2 * z_channels, 1)
- self.conv_in = nn.Conv2d(in_channels, ch, 3, 1, 1)
-
- in_ch_mult = (1,) + tuple(ch_mult)
- self.down = nn.ModuleList()
- block_in = ch
- for i_level in range(num_resolutions):
- block = nn.ModuleList()
- attn = nn.ModuleList()
- block_in = ch * in_ch_mult[i_level]
- block_out = ch * ch_mult[i_level]
- for _ in range(num_res_blocks):
- block.append(_ResnetBlock(block_in, block_out))
- block_in = block_out
- down = nn.Module()
- down.block = block
- down.attn = attn
- if i_level != num_resolutions - 1:
- down.downsample = _Downsample(block_in)
- self.down.append(down)
-
- self.mid = nn.Module()
- self.mid.block_1 = _ResnetBlock(block_in, block_in)
- self.mid.attn_1 = _AttnBlock(block_in)
- self.mid.block_2 = _ResnetBlock(block_in, block_in)
-
- self.norm_out = nn.GroupNorm(32, block_in, eps=1e-6, affine=True)
- self.conv_out = nn.Conv2d(block_in, 2 * z_channels, 3, 1, 1)
- self._num_resolutions = num_resolutions
- self._num_res_blocks = num_res_blocks
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- hs = [self.conv_in(x)]
- for i_level in range(self._num_resolutions):
- for i_block in range(self._num_res_blocks):
- h = self.down[i_level].block[i_block](hs[-1])
- if len(self.down[i_level].attn) > 0:
- h = self.down[i_level].attn[i_block](h)
- hs.append(h)
- if hasattr(self.down[i_level], "downsample"):
- hs.append(self.down[i_level].downsample(hs[-1]))
- h = hs[-1]
- h = self.mid.block_1(h)
- h = self.mid.attn_1(h)
- h = self.mid.block_2(h)
- h = _swish(self.norm_out(h))
- h = self.conv_out(h)
- h = self.quant_conv(h)
- return h
-
-
-class CheersVAEDecoder(nn.Module):
- """VAE decoder (used inside VAEDecoderProjector)."""
-
- def __init__(self, config):
- super().__init__()
- d = _VAE_DECODER_DEFAULTS
- ch = _cfg(config, "ch", d)
- ch_mult = _cfg(config, "ch_mult", d)
- num_res_blocks = _cfg(config, "num_res_blocks", d)
- z_channels = _cfg(config, "z_channels", d)
- out_ch = _cfg(config, "out_ch", d)
- num_resolutions = len(ch_mult)
-
- self.post_quant_conv = nn.Conv2d(z_channels, z_channels, 1)
- block_in = ch * ch_mult[num_resolutions - 1]
- self.conv_in = nn.Conv2d(z_channels, block_in, 3, 1, 1)
-
- self.mid = nn.Module()
- self.mid.block_1 = _ResnetBlock(block_in, block_in)
- self.mid.attn_1 = _AttnBlock(block_in)
- self.mid.block_2 = _ResnetBlock(block_in, block_in)
-
- self.up = nn.ModuleList()
- for i_level in reversed(range(num_resolutions)):
- block = nn.ModuleList()
- attn = nn.ModuleList()
- block_out = ch * ch_mult[i_level]
- for _ in range(num_res_blocks + 1):
- block.append(_ResnetBlock(block_in, block_out))
- block_in = block_out
- up = nn.Module()
- up.block = block
- up.attn = attn
- if i_level != 0:
- up.upsample = _Upsample(block_in)
- self.up.insert(0, up)
-
- self.norm_out = nn.GroupNorm(32, block_in, eps=1e-6, affine=True)
- self.conv_out = nn.Conv2d(block_in, out_ch, 3, 1, 1)
- self._num_resolutions = num_resolutions
- self._num_res_blocks = num_res_blocks
-
- def forward(self, z: torch.Tensor) -> torch.Tensor:
- z = self.post_quant_conv(z)
- upscale_dtype = next(self.up.parameters()).dtype
- h = self.conv_in(z)
- h = self.mid.block_1(h)
- h = self.mid.attn_1(h)
- h = self.mid.block_2(h)
- h = h.to(upscale_dtype)
- for i_level in reversed(range(self._num_resolutions)):
- for i_block in range(self._num_res_blocks + 1):
- h = self.up[i_level].block[i_block](h)
- if len(self.up[i_level].attn) > 0:
- h = self.up[i_level].attn[i_block](h)
- if i_level != 0:
- h = self.up[i_level].upsample(h)
- h = _swish(self.norm_out(h))
- return self.conv_out(h)
-
-
-class CheersVAEModel(nn.Module):
- """VAE model with encoder only (for image understanding)."""
-
- def __init__(self, config):
- super().__init__()
- enc_cfg = _cfg(config, "vae_encoder_config")
- self.encoder = CheersVAEEncoder(enc_cfg)
- self.ps = [2, 2]
- z_ch = _cfg(enc_cfg, "z_channels", _VAE_ENCODER_DEFAULTS)
- self.bn = nn.BatchNorm2d(
- math.prod(self.ps) * z_ch,
- eps=1e-4,
- momentum=0.1,
- affine=False,
- track_running_stats=True,
- )
-
- def encode(self, x: torch.Tensor) -> torch.Tensor:
- self.bn.eval()
- moments = self.encoder(x)
- mean = torch.chunk(moments, 2, dim=1)[0]
- z = rearrange(
- mean,
- "... c (i pi) (j pj) -> ... (c pi pj) i j",
- pi=self.ps[0],
- pj=self.ps[1],
- )
- return self.bn(z)
-
-
-class CheersVAEDecoderProjector(nn.Module):
- """VAE decoder projector that converts latent back to pixel-like space."""
-
- def __init__(self, config):
- super().__init__()
- dec_cfg = _cfg(config, "vae_decoder_config")
- enc_cfg = _cfg(config, "vae_encoder_config")
- self.decoder = CheersVAEDecoder(dec_cfg)
- self.ps = [2, 2]
- z_ch = _cfg(enc_cfg, "z_channels", _VAE_ENCODER_DEFAULTS)
- self.bn = nn.BatchNorm2d(
- math.prod(self.ps) * z_ch,
- eps=1e-4,
- momentum=0.1,
- affine=False,
- track_running_stats=True,
- )
-
- def forward(self, z: torch.Tensor) -> torch.Tensor:
- self.bn.eval()
- s = torch.sqrt(self.bn.running_var.view(1, -1, 1, 1) + 1e-4)
- m = self.bn.running_mean.view(1, -1, 1, 1)
- z = z * s + m
- z = rearrange(
- z,
- "... (c pi pj) i j -> ... c (i pi) (j pj)",
- pi=self.ps[0],
- pj=self.ps[1],
- )
- return self.decoder(z)
-
-
-class CheersImagePixelInputs(TensorSchema):
- """
- Dimensions:
- - bn: Batch size * number of images
- - c: Number of channels (3)
- - h: Height of each image
- - w: Width of each image
- """
-
- type: Literal["pixel_values"]
- pixel_values: torch.Tensor # Shape: (bn, 3, h, w)
-
-
-CheersImageInputs: TypeAlias = CheersImagePixelInputs
-
-
-class CheersUndProjector(nn.Module):
- """Understanding projector that maps vision features to LLM dimension
- with 2x2 spatial compression (4x token reduction)."""
-
- def __init__(
- self,
- image_embed_dim: int,
- text_embed_dim: int,
- compression_factor: tuple[int, int] = (2, 2),
- quant_config: QuantizationConfig | None = None,
- prefix: str = "",
- ):
- super().__init__()
- self.image_embed_dim = image_embed_dim
- self.text_embed_dim = text_embed_dim
- self.compression_factor = compression_factor
- self.layernorm = nn.LayerNorm(image_embed_dim)
- hidden_size = image_embed_dim * (compression_factor[0] * compression_factor[1])
- self.mlp = nn.Sequential(
- nn.Linear(hidden_size, hidden_size),
- nn.GELU(),
- nn.Linear(hidden_size, text_embed_dim),
- )
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- x = self.layernorm(x)
- height = width = int(x.size(1) ** 0.5)
- x = x.permute(0, 2, 1).unflatten(-1, (height, width))
- batch_size, dim, height, width = x.shape
- unfolded = x.unfold(
- 2, self.compression_factor[0], self.compression_factor[0]
- ).unfold(3, self.compression_factor[1], self.compression_factor[1])
- unfolded = unfolded.contiguous().view(
- batch_size,
- dim,
- -1,
- self.compression_factor[0] * self.compression_factor[1],
- )
- unfolded = (
- unfolded.permute(0, 2, 3, 1)
- .contiguous()
- .view(
- batch_size,
- -1,
- dim * self.compression_factor[0] * self.compression_factor[1],
- )
- )
- return self.mlp(unfolded)
-
-
-class CheersProcessingInfo(BaseProcessingInfo):
- """Processing information for Cheers model."""
-
- def get_hf_processor(self, **kwargs: object) -> CheersProcessor:
- from vllm.transformers_utils.processor import cached_get_image_processor
-
- image_processor = cached_get_image_processor(
- self.ctx.model_config.model,
- revision=self.ctx.model_config.revision,
- trust_remote_code=self.ctx.model_config.trust_remote_code,
- )
-
- tokenizer = self.get_tokenizer()
-
- return CheersProcessor(
- image_processor=image_processor,
- tokenizer=tokenizer,
- **kwargs,
- )
-
- def get_supported_mm_limits(self) -> Mapping[str, int | None]:
- return {"image": None}
-
- def get_mm_max_tokens_per_item(
- self,
- seq_len: int,
- mm_counts: Mapping[str, int],
- ) -> Mapping[str, int]:
- hf_config = self.get_hf_config()
- vit_config = hf_config.vision_representation_config
- patch_size = vit_config.patch_size
- image_size = vit_config.image_size
- num_patches = (image_size // patch_size) ** 2
- # After 2x2 compression, tokens reduce by 4x
- num_tokens = num_patches // 4
- return {"image": num_tokens}
-
- def get_num_image_tokens(
- self,
- *,
- image_width: int,
- image_height: int,
- ) -> int:
- hf_config = self.get_hf_config()
- vit_config = hf_config.vision_representation_config
- patch_size = vit_config.patch_size
- image_size = vit_config.image_size
- num_patches = (image_size // patch_size) ** 2
- return num_patches // 4
-
-
-class CheersDummyInputsBuilder(BaseDummyInputsBuilder[CheersProcessingInfo]):
- """Build dummy inputs for Cheers model profiling."""
-
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
- num_images = mm_counts.get("image", 0)
- return "<|image_pad|>" * num_images
-
- def get_dummy_mm_data(
- self,
- seq_len: int,
- mm_counts: Mapping[str, int],
- mm_options: Mapping[str, BaseDummyOptions] | None = None,
- ) -> MultiModalDataDict:
- num_images = mm_counts.get("image", 0)
- hf_config = self.info.get_hf_config()
- vit_config = hf_config.vision_representation_config
- image_size = vit_config.image_size
- image_overrides = mm_options.get("image") if mm_options else None
-
- return {
- "image": self._get_dummy_images(
- width=image_size,
- height=image_size,
- num_images=num_images,
- overrides=image_overrides,
- ),
- }
-
-
-class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
- """Multimodal processor for Cheers model."""
-
- def _get_prompt_updates(
- self,
- mm_items: MultiModalDataItems,
- hf_processor_mm_kwargs: Mapping[str, Any],
- out_mm_kwargs: MultiModalKwargsItems,
- ) -> Sequence[PromptReplacement]:
- hf_config = self.info.get_hf_config()
- vit_config = hf_config.vision_representation_config
- patch_size = vit_config.patch_size
- image_size = vit_config.image_size
-
- tokenizer = self.info.get_tokenizer()
- image_token_id = tokenizer.get_vocab().get("<|image_pad|>")
- if image_token_id is None:
- raise ValueError(
- "Image token '<|image_pad|>' not found in tokenizer vocabulary"
- )
-
- def get_replacement_cheers(item_idx: int):
- num_patches = (image_size // patch_size) ** 2
- num_tokens = num_patches // 4
- return [image_token_id] * num_tokens
-
- return [
- PromptReplacement(
- modality="image",
- target=[image_token_id],
- replacement=get_replacement_cheers,
- )
- ]
-
- def _get_mm_fields_config(
- self,
- hf_inputs: Any,
- hf_processor_mm_kwargs: Mapping[str, object],
- ) -> Mapping[str, MultiModalFieldConfig]:
- return {
- "pixel_values": MultiModalFieldConfig.batched("image"),
- }
-
-
-@MULTIMODAL_REGISTRY.register_processor(
- CheersMultiModalProcessor,
- info=CheersProcessingInfo,
- dummy_inputs=CheersDummyInputsBuilder,
-)
-class CheersForConditionalGeneration(
- nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP
-):
- """
- Cheers: A unified multimodal model for image understanding and generation.
-
- For vLLM, we focus on the image understanding (vision-to-text) capabilities.
- The image generation part is not supported in vLLM.
- """
-
- requires_raw_input_tokens = True
-
- hf_to_vllm_mapper = WeightsMapper(
- orig_to_new_prefix={
- "model.language_model.": "language_model.model.",
- "model.vision_representation.": "vision_representation.vision_model.",
- "model.und_projector.": "und_projector.",
- "model.vae_model.": "vae_model.",
- "model.vae_decoder_projector.": "vae_decoder_projector.",
- "lm_head.": "language_model.lm_head.",
- }
- )
-
- @classmethod
- def get_placeholder_str(cls, modality: str, i: int) -> str | None:
- if modality.startswith("image"):
- return "<|image_pad|>"
- raise ValueError("Only image modality is supported")
-
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
-
- config = vllm_config.model_config.hf_config
- quant_config = vllm_config.quant_config
- multimodal_config = vllm_config.model_config.multimodal_config
-
- if type(config).__name__ not in ("CheersConfig", "UMMConfig"):
- raise ValueError(
- f"Expected CheersConfig or UMMConfig, got {type(config).__name__}."
- )
-
- self.config = config
- self.multimodal_config = multimodal_config
-
- # The Cheers model's custom Qwen2Config defaults rope_theta to
- # 1_000_000, but this isn't stored in the JSON. vLLM's standard
- # Qwen2Config defaults to 10_000, causing a 100× mismatch.
- # We must patch BOTH the attribute AND rope_parameters (which
- # patch_rope_parameters may have already populated from the wrong
- # default before __init__ runs).
- _CHEERS_ROPE_THETA = 1_000_000.0
- tc = config.text_config
- old_theta = getattr(tc, "rope_theta", None)
- if old_theta != _CHEERS_ROPE_THETA:
- logger.info(
- "Overriding text_config.rope_theta from %s to %s",
- old_theta,
- _CHEERS_ROPE_THETA,
- )
- tc.rope_theta = _CHEERS_ROPE_THETA
- rp = getattr(tc, "rope_parameters", None)
- if rp is not None and rp.get("rope_theta") != _CHEERS_ROPE_THETA:
- logger.info(
- "Overriding rope_parameters.rope_theta from %s to %s",
- rp.get("rope_theta"),
- _CHEERS_ROPE_THETA,
- )
- rp["rope_theta"] = _CHEERS_ROPE_THETA
-
- with self._mark_language_model(vllm_config):
- self.language_model = init_vllm_registered_model(
- vllm_config=vllm_config,
- hf_config=config.text_config,
- prefix=maybe_prefix(prefix, "language_model"),
- architectures=["Qwen2ForCausalLM"],
- )
-
- vit_config = config.vision_representation_config
-
- with self._mark_tower_model(vllm_config, "image"):
- self.vae_model = CheersVAEModel(config)
- self.vae_decoder_projector = CheersVAEDecoderProjector(config)
-
- self.vision_representation = SiglipVisionModel(
- config=vit_config,
- quant_config=quant_config,
- prefix=maybe_prefix(prefix, "vision_representation"),
- )
-
- vit_hidden_size = vit_config.hidden_size
- llm_hidden_size = config.text_config.hidden_size
-
- self.und_projector = CheersUndProjector(
- image_embed_dim=vit_hidden_size,
- text_embed_dim=llm_hidden_size,
- compression_factor=(2, 2),
- quant_config=quant_config,
- prefix=maybe_prefix(prefix, "und_projector"),
- )
-
- self.make_empty_intermediate_tensors = (
- self.language_model.make_empty_intermediate_tensors
- )
-
- def _parse_and_validate_image_input(
- self, **kwargs: object
- ) -> CheersImageInputs | None:
- pixel_values = kwargs.pop("pixel_values", None)
- if pixel_values is None:
- return None
- return CheersImagePixelInputs(
- type="pixel_values",
- pixel_values=pixel_values,
- )
-
- def _process_image_input(
- self, image_input: CheersImageInputs
- ) -> tuple[torch.Tensor, ...]:
- """Process image inputs through VAE → SigLIP → projector pipeline.
-
- HF native path: pixel_values → VAE.encode(t=1.0) → vae_decoder_projector
- → SigLIP → und_projector → text-space embeddings
- """
- pixel_values = image_input["pixel_values"]
-
- if pixel_values.ndim == 5:
- batch_size, num_images, channels, height, width = pixel_values.shape
- pixel_values = pixel_values.reshape(
- batch_size * num_images, channels, height, width
- )
-
- with torch.no_grad():
- vae_dtype = next(self.vae_model.parameters()).dtype
- image_latent = self.vae_model.encode(pixel_values.to(dtype=vae_dtype))
- image_pixel_hat = self.vae_decoder_projector(image_latent)
-
- vision_features = self.vision_representation(image_pixel_hat)
- vision_embeds = self.und_projector(vision_features)
-
- return tuple(vision_embeds)
-
- def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
- image_input = self._parse_and_validate_image_input(**kwargs)
- if image_input is None:
- return []
- return self._process_image_input(image_input)
-
- def forward(
- self,
- input_ids: torch.Tensor | None,
- positions: torch.Tensor,
- intermediate_tensors: IntermediateTensors | None = None,
- inputs_embeds: torch.Tensor | None = None,
- **kwargs: object,
- ) -> torch.Tensor | IntermediateTensors:
- if intermediate_tensors is not None:
- inputs_embeds = None
-
- hidden_states = self.language_model.model(
- input_ids=input_ids,
- positions=positions,
- intermediate_tensors=intermediate_tensors,
- inputs_embeds=inputs_embeds,
- )
- return hidden_states
-
- def compute_logits(
- self,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor | None:
- return self.language_model.compute_logits(hidden_states)
-
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
- """Load weights, keeping VAE encoder/decoder projector for understanding."""
- skip_prefixes = [
- "model.time_embed.",
- "model.gen_projector.",
- "model.hi_gate.",
- "model.hi_projector.",
- "model.vae_model.decoder.",
- ]
- skip_keywords = [
- "text_loss_fc",
- ]
-
- filtered_weights = []
- for name, tensor in weights:
- if any(name.startswith(p) for p in skip_prefixes):
- continue
- if any(kw in name for kw in skip_keywords):
- continue
- filtered_weights.append((name, tensor))
-
- loader = AutoWeightsLoader(self)
- return loader.load_weights(filtered_weights, mapper=self.hf_to_vllm_mapper)
diff --git a/vllm/model_executor/models/conformer_encoder.py b/vllm/model_executor/models/conformer_encoder.py
index 0d2e312701..f64da4c472 100644
--- a/vllm/model_executor/models/conformer_encoder.py
+++ b/vllm/model_executor/models/conformer_encoder.py
@@ -1,11 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
-Shared Conformer encoder components for FireRedASR2 and FireRedLID.
+Conformer encoder components for FireRedASR2.
-Both models use the same Conformer-based audio encoder architecture
+The audio encoder architecture is
(Conv2dSubsampling → RelPositionalEncoding → N × RelPosEmbConformerBlock).
-This module factors out the common building blocks to avoid duplication.
"""
import torch
@@ -288,7 +287,7 @@ class RelPosEmbConformerBlock(nn.Module):
class ConformerEncoder(nn.Module):
"""
- Conformer encoder shared by FireRedASR2 and FireRedLID.
+ Conformer encoder used by FireRedASR2.
"""
def __init__(
diff --git a/vllm/model_executor/models/fairseq2_llama.py b/vllm/model_executor/models/fairseq2_llama.py
deleted file mode 100644
index 97afb760cf..0000000000
--- a/vllm/model_executor/models/fairseq2_llama.py
+++ /dev/null
@@ -1,149 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-# Copyright 2024 The vLLM team.
-# Copyright 2024 Meta Platforms, Inc. and affiliates. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Llama model for fairseq2 weights."""
-
-from collections.abc import Iterable
-
-import torch
-from torch.nn import Parameter
-
-from vllm.config import VllmConfig
-from vllm.distributed import (
- get_tensor_model_parallel_rank,
- get_tensor_model_parallel_world_size,
-)
-from vllm.model_executor.layers.linear import set_weight_attrs
-from vllm.model_executor.models.llama import LlamaForCausalLM
-
-from .utils import AutoWeightsLoader, WeightsMapper
-
-
-class Fairseq2LlamaForCausalLM(LlamaForCausalLM):
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__(vllm_config=vllm_config, prefix=prefix)
- self.tp_rank = get_tensor_model_parallel_rank()
- self.tp_size = get_tensor_model_parallel_world_size()
- # For the model loader to read only the relevant checkpoint files
- self.allow_patterns_overrides = [
- # either the full checkpoint
- "model.pt",
- # or the tp-sharded checkpoint of the current rank
- f"model.{self.tp_rank}.pt",
- ]
-
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
- # fairseq2's serialization adds a wrapper to usual .pt state_dict's:
- # { "model_key": my_model_name, "my_model_name": state_dict }
- # which we first need to unpack
- weights_wrapped = dict(weights)
- weights = weights_wrapped[weights_wrapped["model_key"]].items() # type: ignore
-
- # remap keys
- fs2_to_vllm_mapper = WeightsMapper(
- orig_to_new_prefix={
- "decoder_frontend.embed.": "model.embed_tokens.",
- "decoder.": "model.",
- "final_proj.": "lm_head.",
- },
- orig_to_new_substr={
- ".self_attn_layer_norm.": ".input_layernorm.",
- ".ffn_layer_norm.": ".post_attention_layernorm.",
- ".self_attn.output_proj.": ".self_attn.o_proj.",
- ".ffn.gate_proj.": ".mlp.gate_proj.",
- ".ffn.inner_proj.": ".mlp.up_proj.",
- ".ffn.output_proj.": ".mlp.down_proj.",
- ".layer_norm.": ".norm.",
- },
- )
- weights = fs2_to_vllm_mapper.apply(weights)
-
- params = dict(self.named_parameters())
-
- loader = AutoWeightsLoader(self)
- return loader.load_weights(
- self.reshape_fairseq2_weights(name, loaded_weight, params)
- for name, loaded_weight in weights
- )
-
- def flag_sharded_weights(self, params: dict[str, Parameter]):
- """Sets the `is_sharded_weight` flag to True for all sharded weights"""
- for name, param in params.items():
- modules = name.split(".")
- if "norm" in name and len(param.size()) < 2:
- # layer norms are not sharded
- continue
- elif any(emb in modules for emb in ["embed_tokens", "lm_head"]):
- # for now we repeat embedding layers for compatibility
- continue
- else:
- # all other layers are sharded
- set_weight_attrs(param, {"is_sharded_weight": True})
-
- def reshape_fairseq2_weights(
- self,
- name: str,
- loaded_weight: torch.Tensor,
- params: dict[str, Parameter],
- ) -> tuple[str, torch.Tensor]:
- """Reshape fairseq2's weights."""
-
- def permute(w: torch.Tensor, n_heads: int) -> torch.Tensor:
- attn_in = self.config.head_dim * n_heads
- # check for a sharded weight on dim 0
- if attn_in // self.tp_size == w.size()[0]:
- attn_in //= self.tp_size
- n_heads //= self.tp_size
- attn_out = self.config.hidden_size
- return (
- w.view(n_heads, attn_in // n_heads // 2, 2, attn_out)
- .transpose(1, 2)
- .reshape(attn_in, attn_out)
- )
-
- modules = name.split(".")
-
- # rotary embeds should be sliced
- if "k_proj" in modules:
- loaded_weight = permute(loaded_weight, self.config.num_key_value_heads)
-
- elif "q_proj" in modules:
- loaded_weight = permute(loaded_weight, self.config.num_attention_heads)
-
- # We make the loaded weights compatible with both
- # full checkpoints and tp sharded checkpoints.
- # Embeddings are repeated to fit the vocab size.
- # Other weights are flagged for the weight_loader calls.
- if any(emb in modules for emb in ["embed_tokens", "lm_head"]):
- # Embeddings are sharded on dim 0
- dim = 0
- # In fairseq2, vocab size has to be divisible by tp_size
- # so we don't worry about padding
- if self.tp_size > 1 and loaded_weight.shape[dim] < self.config.vocab_size:
- assert (
- loaded_weight.shape[dim] * self.tp_size == self.config.vocab_size
- ), "vocab_size should be divisible by tp_size."
- repeats = [1] * len(loaded_weight.size())
- repeats[dim] = self.tp_size
- # repeat to match vocab size and to be easily 'narrow'able
- loaded_weight = loaded_weight.repeat(repeats)
- set_weight_attrs(params[name], {"is_sharded_weight": False})
- # if embeddings are sharded, the rest is too
- if "embed_tokens" in modules:
- self.flag_sharded_weights(params)
-
- return name, loaded_weight
diff --git a/vllm/model_executor/models/fireredlid.py b/vllm/model_executor/models/fireredlid.py
deleted file mode 100644
index 7376556b3f..0000000000
--- a/vllm/model_executor/models/fireredlid.py
+++ /dev/null
@@ -1,794 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-FireRedLID – Language Identification model adapted for vLLM.
-
-Architecture: ConformerEncoder + TransformerDecoder (6-layer cross-attn)
-Vocabulary: 120 LID tokens (dict.txt)
-Output: Up to 2 tokens (e.g. "en", "zh mandarin")
-
-This implementation follows the Whisper-style encoder-decoder pattern:
- • Encoder processes audio features (Fbank + CMVN via FeatureExtractor)
- • Decoder performs single-step autoregressive forward
- • vLLM's generation loop handles beam search / sampling
-"""
-
-from __future__ import annotations
-
-from collections.abc import Iterable, Mapping, Sequence
-from typing import Annotated, Literal
-
-import torch
-from torch import nn
-from transformers import BatchFeature
-
-from vllm.config import ModelConfig, VllmConfig
-from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
-from vllm.config.speech_to_text import SpeechToTextConfig, SpeechToTextParams
-from vllm.distributed import get_tensor_model_parallel_world_size
-from vllm.inputs import MultiModalDataDict, PromptType
-from vllm.logger import init_logger
-from vllm.model_executor.layers.attention import Attention, CrossAttention
-from vllm.model_executor.layers.linear import (
- ColumnParallelLinear,
- ReplicatedLinear,
- RowParallelLinear,
-)
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
-from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
-from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.inputs import (
- MultiModalFieldConfig,
- MultiModalKwargsItems,
-)
-from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
-from vllm.multimodal.processing import (
- BaseDummyInputsBuilder,
- BaseProcessingInfo,
- EncDecMultiModalProcessor,
- PromptReplacement,
- PromptUpdate,
-)
-from vllm.transformers_utils.processor import cached_processor_from_config
-from vllm.utils.tensor_schema import TensorSchema, TensorShape
-
-from .conformer_encoder import ConformerEncoder
-from .interfaces import (
- MultiModalEmbeddings,
- SupportsMultiModal,
- SupportsTranscription,
-)
-from .utils import (
- AutoWeightsLoader,
- WeightsMapper,
- maybe_prefix,
-)
-from .whisper_utils import ISO639_1_SUPPORTED_LANGS
-
-logger = init_logger(__name__)
-
-
-class FireRedLIDAudioInputs(TensorSchema):
- """
- Dimensions:
- - b: Batch size
- - t: Time frames (variable across utterances)
- - nmb: Number of mel bins (80)
- """
-
- input_features: Annotated[
- list[torch.Tensor] | None,
- TensorShape("b", "t", "nmb", dynamic_dims={"t"}),
- ]
- speech_lengths: Annotated[
- list[torch.Tensor] | None,
- TensorShape("b"),
- ]
- fake_token_lengths: Annotated[
- list[torch.Tensor] | None,
- TensorShape("b"),
- ]
-
-
-FireRedLIDEncoder = ConformerEncoder
-
-
-class FireRedLIDPositionalEmbedding(nn.Module):
- """Absolute sinusoidal positional embedding indexed by `positions`."""
-
- def __init__(self, d_model: int, max_len: int = 5000):
- super().__init__()
- assert d_model % 2 == 0
- pe = torch.zeros(max_len, d_model, requires_grad=False)
- position = torch.arange(0, max_len).unsqueeze(1).float()
- div_term = torch.exp(
- torch.arange(0, d_model, 2).float()
- * -(torch.log(torch.tensor(10000.0)).item() / d_model)
- )
- pe[:, 0::2] = torch.sin(position * div_term)
- pe[:, 1::2] = torch.cos(position * div_term)
- self.register_buffer("pe", pe, persistent=False)
-
- def forward(self, position_ids: torch.Tensor) -> torch.Tensor:
- return self.pe[position_ids]
-
-
-class FireRedLIDAttention(nn.Module):
- """Base attention with shared QKV/FC projections for the LID decoder."""
-
- def __init__(
- self,
- d_model: int,
- n_head: int,
- *,
- vllm_config: VllmConfig,
- prefix: str = "",
- ):
- super().__init__()
- tp_size = get_tensor_model_parallel_world_size()
- assert n_head % tp_size == 0
- self.total_num_heads = n_head
- self.num_heads = n_head // tp_size
- self.num_kv_heads = max(1, n_head // tp_size)
- self.head_dim = d_model // n_head
- self.scaling = self.head_dim**-0.5
-
- cache_config = vllm_config.cache_config
- quant_config = vllm_config.quant_config
-
- self.w_qs = ColumnParallelLinear(
- d_model,
- d_model,
- bias=True,
- quant_config=quant_config,
- prefix=f"{prefix}.w_qs",
- )
- self.w_ks = ColumnParallelLinear(
- d_model,
- d_model,
- bias=False,
- quant_config=quant_config,
- prefix=f"{prefix}.w_ks",
- )
- self.w_vs = ColumnParallelLinear(
- d_model,
- d_model,
- bias=True,
- quant_config=quant_config,
- prefix=f"{prefix}.w_vs",
- )
- self.fc = RowParallelLinear(
- d_model,
- d_model,
- bias=True,
- quant_config=quant_config,
- prefix=f"{prefix}.fc",
- )
- self._init_attn(cache_config, quant_config, prefix)
-
- def _init_attn(self, cache_config, quant_config, prefix: str) -> None:
- raise NotImplementedError
-
-
-class FireRedLIDSelfAttention(FireRedLIDAttention):
- def _init_attn(self, cache_config, quant_config, prefix: str) -> None:
- self.attn = Attention(
- self.num_heads,
- self.head_dim,
- self.scaling,
- num_kv_heads=self.num_kv_heads,
- cache_config=cache_config,
- quant_config=quant_config,
- prefix=f"{prefix}.attn",
- )
-
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
- q, _ = self.w_qs(hidden_states)
- k, _ = self.w_ks(hidden_states)
- v, _ = self.w_vs(hidden_states)
- attn_output = self.attn(q, k, v)
- output, _ = self.fc(attn_output)
- return output
-
-
-class FireRedLIDCrossAttention(FireRedLIDAttention):
- def _init_attn(self, cache_config, quant_config, prefix: str) -> None:
- self.attn = CrossAttention(
- self.num_heads,
- self.head_dim,
- self.scaling,
- num_kv_heads=self.num_kv_heads,
- cache_config=cache_config,
- quant_config=quant_config,
- prefix=f"{prefix}.attn",
- )
-
- def forward(
- self,
- hidden_states: torch.Tensor,
- encoder_hidden_states: torch.Tensor | None,
- ) -> torch.Tensor:
- q, _ = self.w_qs(hidden_states)
- if encoder_hidden_states is not None:
- k, _ = self.w_ks(encoder_hidden_states)
- v, _ = self.w_vs(encoder_hidden_states)
- else:
- k = v = None
-
- attn_output = self.attn(q, k, v)
- output, _ = self.fc(attn_output)
- return output
-
-
-class FireRedLIDFFN(nn.Module):
- def __init__(self, d_model: int, d_ff: int):
- super().__init__()
- self.w_1 = ReplicatedLinear(d_model, d_ff, bias=True)
- self.act = nn.GELU()
- self.w_2 = ReplicatedLinear(d_ff, d_model, bias=True)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- x, _ = self.w_1(x)
- x = self.act(x)
- x, _ = self.w_2(x)
- return x
-
-
-class FireRedLIDDecoderLayer(nn.Module):
- """vLLM-native decoder layer while preserving FireRedLID parameter names."""
-
- def __init__(
- self,
- d_model: int,
- n_head: int,
- *,
- vllm_config: VllmConfig,
- prefix: str = "",
- ):
- super().__init__()
- self.self_attn_norm = nn.LayerNorm(d_model)
- self.self_attn = FireRedLIDSelfAttention(
- d_model,
- n_head,
- vllm_config=vllm_config,
- prefix=f"{prefix}.self_attn",
- )
-
- self.cross_attn_norm = nn.LayerNorm(d_model)
- self.cross_attn = FireRedLIDCrossAttention(
- d_model,
- n_head,
- vllm_config=vllm_config,
- prefix=f"{prefix}.cross_attn",
- )
-
- self.mlp_norm = nn.LayerNorm(d_model)
- self.mlp = FireRedLIDFFN(d_model, d_model * 4)
-
- def forward(
- self,
- hidden_states: torch.Tensor,
- encoder_hidden_states: torch.Tensor | None,
- ) -> torch.Tensor:
- residual = hidden_states
- hidden_states = self.self_attn_norm(hidden_states)
- hidden_states = self.self_attn(hidden_states)
- hidden_states = residual + hidden_states
-
- residual = hidden_states
- hidden_states = self.cross_attn_norm(hidden_states)
- hidden_states = self.cross_attn(hidden_states, encoder_hidden_states)
- hidden_states = residual + hidden_states
-
- residual = hidden_states
- hidden_states = self.mlp_norm(hidden_states)
- hidden_states = residual + self.mlp(hidden_states)
-
- return hidden_states
-
-
-class FireRedLIDDecoder(nn.Module):
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
- config = vllm_config.model_config.hf_config
- self.pad_id = getattr(config, "pad_token_id", 2)
- self.n_layers = getattr(config, "n_layers_lid_dec", 6)
- self.d_model = getattr(config, "d_model", 1280)
- self.scale = self.d_model**0.5
-
- self.tgt_word_emb = nn.Embedding(
- getattr(config, "vocab_size", 120),
- self.d_model,
- padding_idx=self.pad_id,
- )
- self.positional_encoding = FireRedLIDPositionalEmbedding(
- self.d_model,
- max_len=getattr(config, "pe_maxlen", 5000),
- )
-
- self.layer_stack = nn.ModuleList(
- [
- FireRedLIDDecoderLayer(
- self.d_model,
- getattr(config, "n_head", 20),
- vllm_config=vllm_config,
- prefix=f"{prefix}.layer_stack.{idx}",
- )
- for idx in range(self.n_layers)
- ]
- )
- self.layer_norm_out = nn.LayerNorm(self.d_model)
-
- def forward(
- self,
- input_ids: torch.Tensor,
- positions: torch.Tensor,
- encoder_hidden_states: torch.Tensor | None,
- ) -> torch.Tensor:
- hidden_states = self.tgt_word_emb(input_ids) * self.scale
- hidden_states = hidden_states + self.positional_encoding(positions)
-
- for layer in self.layer_stack:
- hidden_states = layer(hidden_states, encoder_hidden_states)
-
- hidden_states = self.layer_norm_out(hidden_states)
- return hidden_states
-
- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
- return self.tgt_word_emb(input_ids)
-
-
-class FireRedLIDModel(nn.Module):
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
- config = vllm_config.model_config.hf_config
-
- self.encoder = FireRedLIDEncoder(
- idim=getattr(config, "idim", 80),
- n_layers_enc=getattr(config, "n_layers_enc", 16),
- n_head=getattr(config, "n_head", 20),
- d_model=getattr(config, "d_model", 1280),
- kernel_size=getattr(config, "kernel_size", 33),
- pe_maxlen=getattr(config, "pe_maxlen", 5000),
- )
-
- self.decoder = FireRedLIDDecoder(
- vllm_config=vllm_config,
- prefix=maybe_prefix(prefix, "decoder"),
- )
-
- def forward(
- self,
- input_ids: torch.Tensor,
- positions: torch.Tensor,
- encoder_outputs: list[torch.Tensor] | None = None,
- ) -> torch.Tensor:
- enc_states = (
- torch.cat(encoder_outputs, dim=0)
- if encoder_outputs and len(encoder_outputs) > 0
- else None
- )
- decoder_outputs = self.decoder(
- input_ids=input_ids,
- positions=positions,
- encoder_hidden_states=enc_states,
- )
- return decoder_outputs
-
- def get_encoder_outputs(
- self,
- speech: torch.Tensor | list[torch.Tensor],
- speech_lengths: torch.Tensor | list[torch.Tensor],
- ) -> tuple[torch.Tensor, torch.Tensor]:
- """Run the encoder and return padded outputs plus true sequence lengths."""
- enc_output, enc_lengths, _ = self.encoder(speech, speech_lengths)
- return enc_output, enc_lengths
-
-
-class FireRedLIDProcessingInfo(BaseProcessingInfo):
- def get_hf_config(self):
- return self.ctx.get_hf_config()
-
- def get_supported_mm_limits(self) -> Mapping[str, int | None]:
- return {"audio": 1}
-
- def get_feature_extractor(self, **kwargs):
- hf_processor = self.get_hf_processor(**kwargs)
- feature_extractor = hf_processor.feature_extractor
- return feature_extractor
-
- def get_data_parser(self) -> MultiModalDataParser:
- feature_extractor = self.get_feature_extractor()
- return MultiModalDataParser(
- target_sr=feature_extractor.sampling_rate,
- target_channels=1,
- )
-
- @property
- def skip_prompt_length_check(self) -> bool:
- return True
-
- def get_num_audio_tokens(self) -> int:
- # For encoder profiling – return a reasonable dummy length.
- # This doesn't affect actual inference since encoder processes
- # variable-length features.
- return 1
-
-
-class FireRedLIDDummyInputsBuilder(BaseDummyInputsBuilder[FireRedLIDProcessingInfo]):
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
- return ""
-
- def get_dummy_mm_data(
- self,
- seq_len: int,
- mm_counts: Mapping[str, int],
- mm_options: Mapping[str, BaseDummyOptions],
- ) -> MultiModalDataDict:
- feature_extractor = self.info.get_feature_extractor()
- sampling_rate = feature_extractor.sampling_rate
- audio_len = feature_extractor.chunk_length * sampling_rate
- num_audios = mm_counts.get("audio", 0)
- audio_overrides = mm_options.get("audio")
- assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
- return {
- "audio": self._get_dummy_audios(
- length=audio_len,
- num_audios=num_audios,
- overrides=audio_overrides,
- )
- }
-
-
-class FireRedLIDMultiModalProcessor(
- EncDecMultiModalProcessor[FireRedLIDProcessingInfo]
-):
- def create_encoder_prompt(
- self,
- prompt: list[int],
- mm_items: MultiModalDataItems,
- ) -> list[int]:
- # Dummy encoder prompt for profiling (encoder only processes audio).
- return [0]
-
- def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
- return self.dummy_inputs.get_dummy_text(mm_counts)
-
- def _preprocess_hf_mm_data(
- self,
- mm_data: Mapping[str, object],
- hf_processor_mm_kwargs: Mapping[str, object],
- ) -> tuple[Mapping[str, object], Mapping[str, object]]:
- feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
-
- mm_data = dict(mm_data)
- mm_data["audio"] = mm_data.pop("audios")
-
- hf_processor_mm_kwargs = dict(
- **hf_processor_mm_kwargs,
- sampling_rate=feature_extractor.sampling_rate,
- )
-
- return mm_data, hf_processor_mm_kwargs
-
- def _postprocess_hf_mm_data(
- self,
- mm_data: Mapping[str, object],
- hf_processor_mm_kwargs: Mapping[str, object],
- processed_data: BatchFeature,
- ) -> BatchFeature:
- if "labels" in processed_data:
- processed_data["input_ids"] = processed_data.pop("labels")
-
- return processed_data
-
- def _get_mm_fields_config(
- self,
- hf_inputs: BatchFeature,
- hf_processor_mm_kwargs: Mapping[str, object],
- ) -> Mapping[str, MultiModalFieldConfig]:
- return dict(
- input_features=MultiModalFieldConfig.batched("audio"),
- speech_lengths=MultiModalFieldConfig.batched("audio"),
- fake_token_lengths=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
- )
-
- def _get_prompt_updates(
- self,
- mm_items: MultiModalDataItems,
- hf_processor_mm_kwargs: Mapping[str, object],
- out_mm_kwargs: MultiModalKwargsItems,
- ) -> Sequence[PromptUpdate]:
- out_mm_data = out_mm_kwargs.get_data()
- fake_token_lengths = out_mm_data.get("fake_token_lengths")
-
- if fake_token_lengths is None:
- # Fallback to max encoder output length if not available
- audio_output_lengths = []
- else:
- assert isinstance(fake_token_lengths, torch.Tensor)
- audio_output_lengths = fake_token_lengths.tolist()
-
- def get_replacement(item_idx: int):
- if audio_output_lengths:
- num_tokens = int(audio_output_lengths[item_idx])
- else:
- num_tokens = self.info.get_num_audio_tokens()
- return [0] * num_tokens
-
- return [
- PromptReplacement(
- modality="audio",
- target=[0],
- replacement=get_replacement,
- )
- ]
-
-
-# FireRedLID supports a wider set of languages than Whisper's shared list.
-# Only ISO 639-1 codes are listed; FireRedLID's dialect tokens (mandarin,
-# xinan, wu, …) are output tokens but not valid language *request* codes.
-_FIREREDLID_SUPPORTED_LANGUAGES: Mapping[str, str] = {
- **ISO639_1_SUPPORTED_LANGS,
- "am": "Amharic",
- "as": "Assamese",
- "ba": "Bashkir",
- "bn": "Bengali",
- "bo": "Tibetan",
- "br": "Breton",
- "eu": "Basque",
- "fo": "Faroese",
- "gu": "Gujarati",
- "ha": "Hausa",
- "haw": "Hawaiian",
- "ht": "Haitian Creole",
- "jw": "Javanese",
- "ka": "Georgian",
- "km": "Khmer",
- "la": "Latin",
- "lb": "Luxembourgish",
- "ln": "Lingala",
- "lo": "Lao",
- "mg": "Malagasy",
- "ml": "Malayalam",
- "mn": "Mongolian",
- "mt": "Maltese",
- "my": "Myanmar",
- "nn": "Nynorsk",
- "oc": "Occitan",
- "pa": "Panjabi",
- "ps": "Pashto",
- "sa": "Sanskrit",
- "sd": "Sindhi",
- "si": "Sinhala",
- "sn": "Shona",
- "so": "Somali",
- "sq": "Albanian",
- "su": "Sundanese",
- "te": "Telugu",
- "tg": "Tajik",
- "tk": "Turkmen",
- "tt": "Tatar",
- "uz": "Uzbek",
- "yi": "Yiddish",
- "yo": "Yoruba",
- "yue": "Cantonese",
-}
-
-
-@MULTIMODAL_REGISTRY.register_processor(
- FireRedLIDMultiModalProcessor,
- info=FireRedLIDProcessingInfo,
- dummy_inputs=FireRedLIDDummyInputsBuilder,
-)
-class FireRedLIDForConditionalGeneration(
- nn.Module, SupportsTranscription, SupportsMultiModal
-):
- # -- SupportsTranscription protocol attributes --
- supports_transcription_only = True
- supported_languages = _FIREREDLID_SUPPORTED_LANGUAGES
-
- hf_to_vllm_mapper = WeightsMapper(
- orig_to_new_substr={
- "encoder.": "model.encoder.",
- "lid_decoder.": "model.decoder.",
- # Encoder FFN: nn.Sequential indices → named children
- "net.0": "pre_layer_norm",
- "net.1": "linear_expand",
- "net.4": "linear_project",
- },
- orig_to_new_prefix={
- # Position encoding buffers are rebuilt at init, and the output
- # projection is tied to the embedding.
- "model.encoder.positional_encoding.pe": None,
- "model.decoder.positional_encoding.pe": None,
- "model.decoder.tgt_word_prj.weight": None,
- "proj_out.": None,
- },
- )
-
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
- config = vllm_config.model_config.hf_config
- self.config = config
- self.dtype = vllm_config.model_config.dtype
-
- with self._mark_composite_model(
- vllm_config,
- language_targets=FireRedLIDDecoder,
- tower_targets={"audio": FireRedLIDEncoder},
- ):
- self.model = FireRedLIDModel(
- vllm_config=vllm_config,
- prefix=maybe_prefix(prefix, "model"),
- )
-
- self.proj_out = ParallelLMHead(
- getattr(config, "vocab_size", 120),
- getattr(config, "d_model", 1280),
- quant_config=vllm_config.quant_config,
- prefix=maybe_prefix(prefix, "proj_out"),
- )
- self.proj_out = self.proj_out.tie_weights(self.model.decoder.tgt_word_emb)
-
- logit_scale = getattr(config, "logit_scale", 1.0)
- self.logits_processor = LogitsProcessor(
- getattr(config, "vocab_size", 120),
- scale=logit_scale,
- )
-
- def forward(
- self,
- input_ids: torch.Tensor,
- positions: torch.Tensor,
- encoder_outputs: list[torch.Tensor] | None = None,
- **kwargs,
- ) -> torch.Tensor:
- if encoder_outputs is None:
- encoder_outputs = []
- decoder_outputs = self.model(
- input_ids=input_ids,
- positions=positions,
- encoder_outputs=encoder_outputs,
- )
- return decoder_outputs
-
- def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
- """Run encoder on audio features and return per-item embeddings."""
- audio_input = self._parse_and_validate_audio_input(**kwargs)
-
- speech = audio_input["input_features"]
- speech_lengths = audio_input["speech_lengths"]
- if speech is None or speech_lengths is None:
- return []
-
- # When audio items have different time lengths, vLLM's
- # MultiModalBatchedField._reduce_data returns a plain
- # list[Tensor] instead of a stacked Tensor. The encoder
- # expects a padded [B, Tmax, feat_dim] Tensor, so we
- # normalise both speech and speech_lengths here.
- if isinstance(speech, (list, tuple)):
- # Each element: [Ti, feat_dim] (or [1, Ti, feat_dim])
- tensors = [
- s.squeeze(0) if s.dim() == 3 and s.size(0) == 1 else s for s in speech
- ]
- device = tensors[0].device
- dtype = tensors[0].dtype
- feat_dim = tensors[0].shape[-1]
- lengths = torch.tensor(
- [t.size(0) for t in tensors],
- device=device,
- dtype=torch.int32,
- )
- t_max = int(lengths.max().item())
- # Pre-allocate zero-padded batch tensor
- speech = torch.zeros(
- (len(tensors), t_max, feat_dim),
- device=device,
- dtype=dtype,
- )
- for i, t in enumerate(tensors):
- speech[i, : t.size(0)] = t
- speech_lengths = lengths
- else:
- # Already a batched Tensor [B, T, feat_dim]
- if speech.dim() == 2:
- speech = speech.unsqueeze(0)
-
- speech_lengths = torch.as_tensor(
- speech_lengths, dtype=torch.int32, device=speech.device
- )
-
- enc_output, enc_lengths = self.model.get_encoder_outputs(
- speech=speech,
- speech_lengths=speech_lengths,
- )
-
- # vLLM expects one 2D tensor per multimodal item. Slice each batch entry
- # by the true encoder length so cross-attention never sees padded frames.
- return tuple(
- enc_output[i, : max(0, int(enc_lengths[i].item()))]
- for i in range(enc_output.size(0))
- )
-
- def embed_input_ids(
- self,
- input_ids: torch.Tensor,
- multimodal_embeddings: MultiModalEmbeddings | None = None,
- *,
- is_multimodal: torch.Tensor | None = None,
- ) -> torch.Tensor:
- return self.model.decoder.embed_input_ids(input_ids)
-
- def _parse_and_validate_audio_input(
- self, **kwargs: object
- ) -> FireRedLIDAudioInputs:
- input_features = kwargs.pop("input_features", None)
- speech_lengths = kwargs.pop("speech_lengths", None)
- fake_token_lengths = kwargs.pop("fake_token_lengths", None)
- return FireRedLIDAudioInputs(
- input_features=input_features,
- speech_lengths=speech_lengths,
- fake_token_lengths=fake_token_lengths,
- )
-
- def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
- logits = self.logits_processor(self.proj_out, hidden_states)
- return logits
-
- @classmethod
- def validate_language(cls, language: str | None) -> str | None:
- # FireRedLID is a language *identification* model – the caller does
- # not need to specify a language up-front. Accept None silently.
- if language is None:
- return None
- return super().validate_language(language)
-
- @classmethod
- def get_generation_prompt(
- cls,
- stt_params: SpeechToTextParams,
- ) -> PromptType:
- """Build the prompt for the FireRedLID encoder-decoder model.
-
- The decoder receives a single token; the encoder processes
- the raw audio waveform via the multimodal pipeline.
- """
- prompt: PromptType = {
- "encoder_prompt": {
- "prompt": "",
- "multi_modal_data": {
- "audio": (
- stt_params.audio,
- int(stt_params.stt_config.sample_rate),
- ),
- },
- },
- "decoder_prompt": {
- "prompt": "",
- },
- }
- return prompt
-
- @classmethod
- def get_speech_to_text_config(
- cls,
- model_config: ModelConfig,
- task_type: Literal["transcribe", "translate"],
- ) -> SpeechToTextConfig:
- processor = cached_processor_from_config(model_config)
- return SpeechToTextConfig(
- max_audio_clip_s=processor.feature_extractor.chunk_length,
- sample_rate=processor.feature_extractor.sampling_rate,
- # LID output is at most 2 tokens – no chunking needed.
- min_energy_split_window_size=None,
- )
-
- @classmethod
- def post_process_output(cls, text: str) -> str:
- # Strip any leading/trailing whitespace from the raw LID output.
- return text.strip()
-
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
- loader = AutoWeightsLoader(self)
- return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
diff --git a/vllm/model_executor/models/gritlm.py b/vllm/model_executor/models/gritlm.py
deleted file mode 100644
index 4fb9bc7b09..0000000000
--- a/vllm/model_executor/models/gritlm.py
+++ /dev/null
@@ -1,243 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-from collections.abc import Set
-
-import numpy as np
-import torch
-
-from vllm.config import ModelConfig, VllmConfig
-from vllm.logger import init_logger
-from vllm.model_executor.layers.pooler import (
- DispatchPooler,
- PoolingParamsUpdate,
-)
-from vllm.model_executor.layers.pooler.activations import PoolerNormalize
-from vllm.model_executor.layers.pooler.seqwise import (
- EmbeddingPoolerHead,
- SequencePooler,
- SequencePoolingMethod,
- SequencePoolingMethodOutput,
- get_seq_pooling_method,
-)
-from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
-from vllm.model_executor.models.llama import LlamaForCausalLM
-from vllm.tasks import PoolingTask
-from vllm.tokenizers import cached_tokenizer_from_config
-from vllm.v1.pool.metadata import PoolingMetadata
-
-from .interfaces_base import default_pooling_type
-
-logger = init_logger(__name__)
-
-
-class GritLMMeanPool(SequencePoolingMethod):
- """As `MeanPool`, but only includes non-instruction tokens."""
-
- def __init__(self, model_config: ModelConfig):
- super().__init__()
-
- self.model_config = model_config
-
- tokenizer = cached_tokenizer_from_config(self.model_config)
-
- # Collect the tokens needed for pattern matching.
- # "▁<" is different from "_<". The former uses "▁" to indicate that
- # the next token is the start of a word.
- # "<0x0A>" is the newline token (i.e. "\n")."
- self.token_ids = {
- tok: tokenizer.convert_tokens_to_ids([tok])[0]
- for tok in ["", "▁<", "<", "|", "embed", ">", "<0x0A>", "user"]
- }
-
- def tokens_to_ids(tokens: list[str]) -> np.ndarray:
- return np.array([self.token_ids[token] for token in tokens])
-
- self.user_pattern_ids = tokens_to_ids(["▁<", "|", "user", "|", ">", "<0x0A>"])
- self.embed_newline_pattern_ids = tokens_to_ids(
- ["<0x0A>", "<", "|", "embed", "|", ">", "<0x0A>"]
- )
- self.embed_pattern_ids = tokens_to_ids(["▁<", "|", "embed", "|", ">", "<0x0A>"])
-
- def _find_array(
- self,
- arr: np.ndarray,
- target: np.ndarray,
- start_idx: int = 0,
- end_idx: int | None = None,
- ) -> int:
- """
- Find the first occurrence of `target` in `arr` starting from
- `start_idx`.
-
- Args:
- arr: The array to search within.
- target: The consecutive subsequence to find.
- start_idx: The starting index to search from (inclusive).
- end_idx: The ending index to search from (exclusive).
-
- Returns:
- The index of the first occurrence of `target` in `arr`.
- """
- if start_idx < 0:
- raise ValueError("`start_idx` must be non-negative")
- if len(arr) == 0 or len(target) == 0:
- raise ValueError("Empty `arr` or `target` not allowed")
-
- arr_len = len(arr)
- target_len = len(target)
-
- if end_idx is None:
- end_idx = arr_len
-
- for i in range(start_idx, min(end_idx, arr_len - target_len + 1)):
- if (arr[i : i + target_len] == target).all():
- return i
-
- return -1
-
- def _get_instruction_len(self, prompt_token_ids: np.ndarray) -> int:
- """
- Get the length of the instruction in the prompt.
-
- We do a pattern matching to find the instruction in the prompt,
- and then return the length of the instruction.
-
- The pattern matching is done using integers instead of strings
- because the prompt is given as a list of token IDs.
- """
- instruction_len = 0
-
- # Return no instruction in case of missing BOS token.
- if prompt_token_ids[0] != self.token_ids[""]:
- logger.warning(
- "BOS token not found in prompt, "
- "thus using empty string for instruction. "
- "GritLM requires BOS token in prompt."
- )
- return instruction_len
-
- # If user pattern is found in the prompt, that means there should be
- # a newline token before the embed pattern.
- embed_pattern_ids = self.embed_pattern_ids
- if (
- self._find_array(
- prompt_token_ids, self.user_pattern_ids, start_idx=1, end_idx=2
- )
- == 1
- ):
- embed_pattern_ids = self.embed_newline_pattern_ids
-
- # Find the embed pattern in the prompt.
- found_embed_pattern_idx = self._find_array(
- prompt_token_ids, embed_pattern_ids, start_idx=1
- )
-
- if found_embed_pattern_idx != -1:
- instruction_len = found_embed_pattern_idx + len(embed_pattern_ids)
- else:
- logger.warning(
- "Query instruction not found in prompt, "
- "thus using BOS token as instruction instead. "
- "GritLM requires query instruction in prompt."
- )
- instruction_len = 1
-
- return instruction_len
-
- def get_supported_tasks(self) -> Set[PoolingTask]:
- return {"embed"}
-
- def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
- return PoolingParamsUpdate(requires_token_ids=True)
-
- def forward(
- self,
- hidden_states: torch.Tensor,
- pooling_metadata: PoolingMetadata,
- ) -> SequencePoolingMethodOutput:
- prompt_lens = pooling_metadata.prompt_lens
- prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
- instr_lens = torch.tensor(
- [
- self._get_instruction_len(token_ids.numpy())
- for token_ids in prompt_token_ids
- ],
- device="cpu",
- )
-
- offset = 0
- pooled_data = list[torch.Tensor]()
- for prompt_len, instr_len in zip(prompt_lens, instr_lens):
- pooled_data.append(
- hidden_states[offset + instr_len : offset + prompt_len].mean(
- dim=0, dtype=torch.float32
- )
- )
- offset += prompt_len
-
- return pooled_data
-
-
-class GritLMPooler(SequencePooler):
- def __init__(self, model_config: ModelConfig):
- pooler_config = model_config.pooler_config
- assert pooler_config is not None
-
- super().__init__(
- pooling=(
- GritLMMeanPool(model_config)
- if pooler_config.seq_pooling_type == "MEAN"
- else get_seq_pooling_method(pooler_config.seq_pooling_type)
- ),
- head=EmbeddingPoolerHead(
- head_dtype=model_config.head_dtype,
- activation=PoolerNormalize(),
- ),
- )
-
-
-@default_pooling_type(seq_pooling_type="MEAN")
-class GritLM(LlamaForCausalLM):
- """This class implements the embedding model for parasail-ai/GritLM-7B-vllm.
-
- The class inherits from LlamaForCausalLM and provides a custom pooling
- layer.
-
- The main difference between the pooling layer in GritLM and the one in
- LlamaForCausalLM is that GritLM ignores the query instruction in the prompt
- when pooling the hidden states.
-
- Embedding prompts should be in the following format:
- - With instruction: "<|user|>\nINSTRUCTION\n<|embed|>\nPROMPT".
- - Without instruction: "<|embed|>\nPROMPT".
-
- Generation prompts should be in the following format:
- - "<|user|>\nPROMPT\n<|assistant|>\n"
- """
-
- is_pooling_model = True
-
- def __init__(
- self,
- vllm_config: VllmConfig,
- prefix: str = "",
- **kwargs,
- ) -> None:
- if vllm_config.model_config.runner_type == "pooling":
- hf_config = vllm_config.model_config.hf_config
- hf_config.is_causal = False
-
- vllm_config.cache_config.sliding_window = None
-
- hf_config.sliding_window = None
-
- super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
-
- pooler_config = vllm_config.model_config.pooler_config
- if pooler_config is not None:
- self.pooler = DispatchPooler(
- {
- "token_embed": pooler_for_token_embed(pooler_config),
- "embed": GritLMPooler(vllm_config.model_config),
- }
- )
diff --git a/vllm/model_executor/models/hyperclovax_vision.py b/vllm/model_executor/models/hyperclovax_vision.py
deleted file mode 100644
index 41387ad7e2..0000000000
--- a/vllm/model_executor/models/hyperclovax_vision.py
+++ /dev/null
@@ -1,1101 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-# copied from : https://github.com/huggingface/transformers
-import ast
-from collections.abc import Iterable, Mapping, Sequence
-from functools import partial
-from itertools import accumulate
-from typing import Annotated, Literal
-
-import numpy as np
-import torch
-import torch.nn as nn
-from einops import rearrange
-from timm.layers import LayerNorm, LayerNorm2d
-from timm.models.regnet import RegStage
-from transformers import BatchFeature, CLIPVisionConfig, SiglipVisionConfig
-
-from vllm.config import VllmConfig
-from vllm.config.multimodal import BaseDummyOptions
-from vllm.inputs import MultiModalDataDict
-from vllm.model_executor.layers.quantization import QuantizationConfig
-from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.inputs import (
- MultiModalFieldConfig,
- MultiModalKwargsItems,
-)
-from vllm.multimodal.parse import ImageSize, MultiModalDataItems
-from vllm.multimodal.processing import (
- BaseDummyInputsBuilder,
- BaseMultiModalProcessor,
- BaseProcessingInfo,
- PromptReplacement,
- PromptUpdate,
-)
-from vllm.sequence import IntermediateTensors
-from vllm.utils.tensor_schema import TensorSchema, TensorShape
-
-from .clip import CLIPVisionModel
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
-from .siglip import SiglipVisionModel
-from .utils import (
- AutoWeightsLoader,
- flatten_bn,
- init_vllm_registered_model,
- maybe_prefix,
-)
-from .vision import get_vision_encoder_info
-
-IMAGE_TOKEN: str = "<|dummy3|>"
-VIDEO_TOKEN: str = "<|_unuse_missing_100270|>"
-
-
-# Based on combine_frames_into_images in
-# https://huggingface.co/naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B/blob/main/processing_hyperclovax.py
-def get_num_combined_frames(
- num_frames: int,
- max_grid_shape: tuple[int, int] = (3, 3),
-) -> int:
- max_num_grids = max_grid_shape[0] * max_grid_shape[1]
-
- # Calculate the number of canvases needed.
- num_canvases = num_frames // max_num_grids
- leftover_frames = num_frames % max_num_grids
-
- return num_canvases + (leftover_frames > 0)
-
-
-class HCXVisionImagePixelInputs(TensorSchema):
- """
- Dimensions:
- - n: Number of images
- - g: Number of grids
- - c: Number of channels (3)
- - h: Height
- - w: Width
- """
-
- type: Literal["pixel_values"] = "pixel_values"
- pixel_values_images: Annotated[
- list[torch.Tensor], TensorShape("n", "g", 3, "h", "w", dynamic_dims={"g"})
- ]
- image_sizes_images: Annotated[torch.Tensor, TensorShape("n", 2)]
-
-
-HCXVisionImageInputs = HCXVisionImagePixelInputs
-
-
-class HCXVisionVideoPixelInputs(TensorSchema):
- """
- Dimensions:
- - n: Number of videos
- - f: Number of frames
- - g: Number of grids
- - c: Number of channels (3)
- - h: Height
- - w: Width
- """
-
- type: Literal["pixel_values_videos"] = "pixel_values_videos"
- pixel_values_videos: Annotated[
- list[list[torch.Tensor]],
- TensorShape("n", "f", "g", 3, "h", "w", dynamic_dims={"f", "g"}),
- ]
-
-
-HCXVisionVideoInputs = HCXVisionVideoPixelInputs
-
-
-class HCXVisionProcessingInfo(BaseProcessingInfo):
- def get_vision_encoder_info(self):
- return get_vision_encoder_info(self.get_hf_config())
-
- def get_supported_mm_limits(self) -> Mapping[str, int | None]:
- return {"image": None, "video": None}
-
- def get_num_image_tokens(
- self,
- *,
- vision_query_length: int | list[int],
- ) -> int:
- if isinstance(vision_query_length, int):
- return vision_query_length
- else:
- return sum(vision_query_length)
-
- def get_num_video_tokens(
- self,
- *,
- vision_query_length: int | list[int],
- ) -> int:
- if isinstance(vision_query_length, int):
- return vision_query_length
- else:
- return sum(vision_query_length)
-
- def get_image_size_with_most_features(self) -> ImageSize:
- vision_encoder_info = self.get_vision_encoder_info()
- width = height = vision_encoder_info.get_image_size()
- return ImageSize(width=width, height=height)
-
- def get_max_image_tokens(self) -> int:
- target_width, target_height = self.get_image_size_with_most_features()
-
- return self.get_num_image_tokens(
- image_width=target_width,
- image_height=target_height,
- )
-
-
-class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo]):
- def get_dummy_text(
- self,
- mm_counts: Mapping[str, int],
- ) -> str:
- dummy_text = IMAGE_TOKEN * mm_counts.get(
- "image", 0
- ) + VIDEO_TOKEN * mm_counts.get("video", 0)
- return dummy_text
-
- def get_dummy_mm_data(
- self,
- seq_len: int,
- mm_counts: Mapping[str, int],
- mm_options: Mapping[str, BaseDummyOptions],
- ) -> MultiModalDataDict:
- num_images = mm_counts.get("image", 0)
- num_videos = mm_counts.get("video", 0)
-
- target_width, target_height = self.info.get_image_size_with_most_features()
- target_num_frames = 32
-
- image_overrides = mm_options.get("image")
- video_overrides = mm_options.get("video")
-
- return {
- "image": self._get_dummy_images(
- width=target_width,
- height=target_height,
- num_images=num_images,
- overrides=image_overrides,
- ),
- "video": self._get_dummy_videos(
- width=target_width - 1,
- height=target_height - 1,
- num_frames=target_num_frames,
- num_videos=num_videos,
- overrides=video_overrides,
- ),
- }
-
-
-class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingInfo]):
- def _apply_hf_processor_main(
- self,
- mm_items: MultiModalDataItems,
- hf_processor_mm_kwargs: Mapping[str, object],
- ) -> BatchFeature:
- valid_mm_items = mm_items.select(
- {k for k, c in mm_items.get_all_counts().items() if c > 0}
- )
- mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
-
- if not mm_data:
- return BatchFeature(dict(passthrough_data))
-
- for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
- if video_arr.dtype != np.uint8:
- mm_data["videos"][video_idx] = video_arr.astype(np.uint8)
-
- images = mm_data.get("images")
- videos = mm_data.get("videos")
-
- # batchify input as a single item
- processed_data = self.info.ctx.call_hf_processor(
- hf_processor=self.info.get_hf_processor(**hf_processor_mm_kwargs),
- data=dict(
- images=None if images is None else [images],
- videos=None if videos is None else [videos],
- ),
- )
-
- for k, v in processed_data.items():
- if isinstance(v, list) and len(v) > 0:
- assert len(v) == 1
- processed_data[k] = v[0]
-
- if images:
- processed_data["image_sizes_images"] = torch.tensor(
- processed_data["image_sizes_images"]
- )
- processed_data["vision_query_lengths_images"] = torch.tensor(
- processed_data["vision_query_lengths_images"]
- )
-
- if videos:
- _idx_per_video = [
- 0,
- *accumulate(get_num_combined_frames(len(video)) for video in videos),
- ]
- processed_data["pixel_values_videos"] = [
- processed_data["pixel_values_videos"][
- _idx_per_video[i] : _idx_per_video[i + 1]
- ]
- for i in range(len(videos))
- ]
- processed_data["vision_query_lengths_videos"] = [
- torch.tensor(
- processed_data["vision_query_lengths_videos"][
- _idx_per_video[i] : _idx_per_video[i + 1]
- ]
- )
- for i in range(len(videos))
- ]
-
- processed_data.update(passthrough_data)
-
- return processed_data
-
- def _get_prompt_updates(
- self,
- mm_items: MultiModalDataItems,
- hf_processor_mm_kwargs: Mapping[str, object],
- out_mm_kwargs: MultiModalKwargsItems,
- ) -> Sequence[PromptUpdate]:
- hf_config = self.info.get_hf_config()
- placeholder = {
- "image": hf_config.image_token_id,
- "video": hf_config.video_token_id,
- }
-
- def get_replacement_hyperclovax(
- item_idx: int,
- modality: str,
- out_mm_kwargs: MultiModalKwargsItems,
- ):
- out_item = out_mm_kwargs[modality][item_idx]
-
- if modality == "image":
- lens = out_item["vision_query_lengths_images"].data.tolist()
- num_tokens = self.info.get_num_image_tokens(vision_query_length=lens)
- elif modality == "video":
- lens = out_item["vision_query_lengths_videos"].data.tolist()
- num_tokens = self.info.get_num_video_tokens(vision_query_length=lens)
- else:
- raise NotImplementedError(modality)
-
- return [placeholder[modality]] * num_tokens
-
- return [
- PromptReplacement(
- modality=modality,
- target=[
- placeholder[modality],
- ],
- replacement=partial(
- get_replacement_hyperclovax,
- modality=modality,
- out_mm_kwargs=out_mm_kwargs,
- ),
- )
- for modality in ("image", "video")
- ]
-
- def _get_mm_fields_config(
- self,
- hf_inputs: BatchFeature,
- hf_processor_mm_kwargs: Mapping[str, object],
- ) -> Mapping[str, MultiModalFieldConfig]:
- fields = dict(
- pixel_values_images=MultiModalFieldConfig.batched("image"),
- image_sizes_images=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
- vision_query_lengths_images=MultiModalFieldConfig.batched(
- "image", keep_on_cpu=True
- ),
- pixel_values_videos=MultiModalFieldConfig.batched("video"),
- vision_query_lengths_videos=MultiModalFieldConfig.batched(
- "video", keep_on_cpu=True
- ),
- )
-
- return fields
-
-
-def init_vision_tower_for_hcxvision(
- vision_config,
- quant_config: QuantizationConfig | None,
- *,
- use_nth_layer: int | None = None,
- require_post_norm: bool | None = None,
- prefix: str = "",
-) -> CLIPVisionModel | SiglipVisionModel:
- num_hidden_layers = vision_config.num_hidden_layers
- if not isinstance(use_nth_layer, int):
- pass
- elif use_nth_layer >= 0:
- num_hidden_layers = use_nth_layer + 1
- else:
- num_hidden_layers = num_hidden_layers + use_nth_layer + 1
-
- if isinstance(vision_config, CLIPVisionConfig):
- return CLIPVisionModel(
- vision_config,
- quant_config=quant_config,
- num_hidden_layers_override=num_hidden_layers,
- require_post_norm=require_post_norm,
- prefix=prefix,
- )
- elif isinstance(vision_config, SiglipVisionConfig):
- return SiglipVisionModel(
- vision_config,
- quant_config=quant_config,
- num_hidden_layers_override=num_hidden_layers,
- require_post_norm=require_post_norm,
- prefix=prefix,
- )
-
- msg = f"Unsupported vision config: {type(vision_config)}"
- raise NotImplementedError(msg)
-
-
-class HCXVisionMlp(nn.Module):
- def __init__(
- self,
- mm_projector_type,
- in_features,
- hidden_features=None,
- out_features=None,
- act_layer=nn.GELU,
- ):
- super().__init__()
- out_features = out_features or in_features
- hidden_features = hidden_features or in_features
- self.mm_projector_type = mm_projector_type
- if self.mm_projector_type == "mlp":
- self.fc1 = nn.Linear(in_features, hidden_features)
- self.act = act_layer()
- self.fc2 = nn.Linear(hidden_features, out_features)
- elif self.mm_projector_type == "inverted_mlp":
- self.fc1 = nn.Linear(in_features, 2 * hidden_features)
- self.act = act_layer()
- self.fc2 = nn.Linear(2 * hidden_features, out_features)
- else:
- raise NotImplementedError(
- "{} is not implemented".format(self.mm_projector_type)
- )
-
- def forward(self, x):
- x = self.fc1(x)
- x = self.act(x)
- x = self.fc2(x)
- return x
-
-
-class HCXVisionCAbstractor(nn.Module):
- """
- This module is based on C-Abstractor, whose license is under apache-2.0.
- You can check the original code at
- https://github.com/khanrc/honeybee/blob/main/honeybee/projectors/projectors.py
- and we made necessary modifications.
- """
-
- def __init__(
- self,
- num_queries: int,
- num_input_tokens: int,
- encoder_hidden_size: int,
- hidden_size: int,
- output_hidden_size: int,
- pos_emb: bool = True,
- prenorm: bool = False,
- ):
- super().__init__()
- self.num_input_tokens = num_input_tokens
- self.output_hidden_size = output_hidden_size
-
- # Positional embedding
- if pos_emb:
- self.pos_emb = torch.nn.Parameter(
- torch.zeros(1, num_input_tokens, encoder_hidden_size)
- )
- self.pos_emb.data.normal_(mean=0.0, std=0.02)
- else:
- self.pos_emb = None
-
- # (Optional) Pre-normalization layer
- if prenorm:
- self.prenorm = LayerNorm(encoder_hidden_size)
- else:
- self.prenorm = None
-
- self.build_net(
- num_queries, encoder_hidden_size, hidden_size, output_hidden_size
- )
- self.dtype = next(self.parameters()).dtype
-
- def forward(
- self,
- x: torch.Tensor,
- num_queries_vis_abstractors: list[list[int]] | None = None,
- num_grids: list[int] | None = None,
- ) -> torch.Tensor:
- if self.prenorm is not None:
- x = self.prenorm(x)
-
- if self.pos_emb is not None:
- x = x + self.pos_emb
-
- x = self._forward(
- x,
- num_queries_vis_abstractors=num_queries_vis_abstractors,
- num_grids=num_grids,
- ) # (B, L, output_hidden_size)
-
- return x
-
- def _forward(
- self,
- x: torch.Tensor,
- num_queries_vis_abstractors: list[list[int]] | None = None,
- num_grids: list[int] | None = None,
- ) -> torch.Tensor:
- # x: [B, L, dim]
- B, L, dim = x.shape
- hw = int(L**0.5)
- x = rearrange(x, "b (h w) d -> b d h w", h=hw, w=hw)
-
- if num_queries_vis_abstractors is not None:
- assert num_grids is not None
- return self._forward_adaptive_num_query(
- x, num_queries_vis_abstractors, num_grids
- )
-
- x = self.net(x)
- x = rearrange(x, "b d h w -> b (h w) d")
- x = self.readout(x)
- return x
-
- def _forward_adaptive_num_query(
- self,
- x: torch.Tensor,
- num_queries_vis_abstractors: list[list[int]] | None = None,
- num_grids: list[int] | None = None,
- ) -> list[torch.Tensor]:
- # self.net is consisted by 3 layers (s1, sampler, s2)
- assert len(self.net) == 3
-
- x = self.net[0](x) # s1
- new_x = []
- for i, num_queries in enumerate(num_queries_vis_abstractors):
- hw = int(num_queries**0.5)
- sampler = nn.AdaptiveAvgPool2d((hw, hw))
- out = sampler(x[num_grids[i] : num_grids[i + 1], :])
- out = self.net[2](out) # s2
-
- out = rearrange(out, "b d h w -> b (h w) d")
- out = self.readout(out)
-
- new_x.append(out)
- return new_x
-
- def build_net(
- self,
- n_queries: int,
- encoder_hidden_size: int,
- hidden_size: int,
- output_hidden_size: int,
- depth: int = 3,
- mlp_depth: int = 2,
- ):
- assert (n_queries**0.5).is_integer(), (
- f"n_queries must be square number. n_queries: {n_queries}"
- )
- hw = int(n_queries**0.5)
-
- # RegBlock = ResBlock + SE
- RegBlock = partial(
- RegStage,
- stride=1,
- dilation=1,
- act_layer=nn.SiLU,
- norm_layer=LayerNorm2d,
- )
-
- s1 = RegBlock(
- depth,
- encoder_hidden_size,
- hidden_size,
- )
- sampler = nn.AdaptiveAvgPool2d((hw, hw))
- s2 = RegBlock(
- depth,
- hidden_size,
- hidden_size,
- )
-
- self.net = nn.Sequential(s1, sampler, s2)
- self.readout = self.build_mlp(mlp_depth, hidden_size, output_hidden_size)
-
- def build_mlp(
- self,
- depth: int,
- hidden_size: int,
- output_hidden_size: int,
- ):
- layers = [nn.Linear(hidden_size, output_hidden_size)]
- for _ in range(1, depth):
- layers.append(nn.SiLU())
- layers.append(nn.Linear(output_hidden_size, output_hidden_size))
- return nn.Sequential(*layers)
-
-
-@MULTIMODAL_REGISTRY.register_processor(
- HCXVisionMultiModalProcessor,
- info=HCXVisionProcessingInfo,
- dummy_inputs=HCXVisionDummyInputsBuilder,
-)
-class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
- """
- HyperCLOVAX-SEED Vision-Language Model (V1 architecture).
-
- Supports:
- - HyperCLOVAX-SEED-Vision-Instruct-3B
-
- Uses CLIP/SigLIP as the vision encoder with C-Abstractor projector.
- """
-
- packed_modules_mapping = {
- "qkv_proj": ["q_proj", "k_proj", "v_proj"],
- "gate_up_proj": ["gate_proj", "up_proj"],
- }
-
- def __init__(
- self,
- *,
- vllm_config: VllmConfig,
- prefix: str = "",
- ) -> None:
- super().__init__()
-
- # init configs
- config = vllm_config.model_config.hf_config
- quant_config = vllm_config.quant_config
- # text_config
- text_config = config.text_config
- if text_config.model_type in ["gpt2", "hyperclovax", "llama"]:
- text_config._attn_implementation = "sdpa"
- if text_config.model_type != "hyperclovax":
- text_config.logits_scaling = 1.0
- # vision_config
- vision_config = config.vision_config
- vision_config.auto_map = {}
- vision_config.anyres = config.anyres
- vision_config.max_num_grids = config.max_num_grids
- self.dtype = vllm_config.model_config.dtype
-
- ## possible_resolution should be matched with preprocessor_config.json
- config.possible_resolutions = self._init_possible_resolutions(
- config, vision_config
- )
-
- with self._mark_tower_model(vllm_config, {"image", "video"}):
- self.vision_model = init_vision_tower_for_hcxvision(
- vision_config,
- quant_config=quant_config,
- use_nth_layer=getattr(config, "use_nth_layer", -1),
- require_post_norm=False,
- prefix=maybe_prefix(prefix, "vision_model"),
- )
- self.mm_projector = self._init_mm_projector(
- config, text_config, vision_config
- )
-
- if config.anyres:
- self.image_newline = nn.Parameter(
- torch.empty(text_config.hidden_size, dtype=self.dtype)
- )
-
- with self._mark_language_model(vllm_config):
- self.language_model = init_vllm_registered_model(
- vllm_config=vllm_config,
- hf_config=text_config,
- prefix=maybe_prefix(prefix, "language_model"),
- )
-
- self.config = config
- self.vision_config = vision_config
- self.text_config = text_config
-
- self.make_empty_intermediate_tensors = (
- self.language_model.make_empty_intermediate_tensors
- )
-
- @classmethod
- def get_placeholder_str(cls, modality: str, i: int) -> str | None:
- if modality.startswith("image"):
- return IMAGE_TOKEN
- if modality.startswith("video"):
- return VIDEO_TOKEN
-
- raise ValueError("Only image or video modality is supported")
-
- def _parse_and_validate_image_input(
- self,
- **kwargs: object,
- ) -> HCXVisionImageInputs | None:
- pixel_values_images = kwargs.pop("pixel_values_images", None)
-
- if pixel_values_images is None:
- return None
-
- image_sizes_images = kwargs.pop("image_sizes_images")
-
- return HCXVisionImagePixelInputs(
- pixel_values_images=pixel_values_images,
- image_sizes_images=image_sizes_images,
- )
-
- def _parse_and_validate_video_input(
- self,
- **kwargs: object,
- ) -> HCXVisionVideoInputs | None:
- pixel_values_videos = kwargs.pop("pixel_values_videos", None)
-
- if pixel_values_videos is None:
- return None
-
- return HCXVisionVideoPixelInputs(
- pixel_values_videos=pixel_values_videos,
- )
-
- def _process_image_input(
- self,
- image_input: HCXVisionImageInputs,
- ) -> tuple[torch.Tensor, ...]:
- return self.forward_images(
- pixel_values_images=image_input["pixel_values_images"],
- image_sizes_images=image_input["image_sizes_images"],
- )
-
- def _process_video_input(
- self,
- video_input: HCXVisionVideoInputs,
- ) -> tuple[torch.Tensor, ...]:
- return self.forward_videos(
- pixel_values_videos=video_input["pixel_values_videos"],
- )
-
- def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
- modalities = {}
-
- # Preserve the order of modalities if there are multiple of them
- # from the order of kwargs.
- for input_key in kwargs:
- if input_key == "pixel_values_images" and "images" not in modalities:
- modalities["images"] = self._parse_and_validate_image_input(**kwargs)
- if input_key == "pixel_values_videos" and "videos" not in modalities:
- modalities["videos"] = self._parse_and_validate_video_input(**kwargs)
-
- return modalities
-
- def embed_multimodal(
- self,
- **kwargs: object,
- ) -> MultiModalEmbeddings:
- modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
- if not modalities:
- return []
-
- # The result multimodal_embeddings is tuple of tensors, with each
- # tensor correspoending to a multimodal data item (image or video).
- multimodal_embeddings: tuple[torch.Tensor, ...] = ()
-
- # NOTE: It is important to iterate over the keys in this dictionary
- # to preserve the order of the modalities.
- for modality in modalities:
- if modality == "images":
- image_input = modalities["images"]
- image_embeddings = self._process_image_input(image_input)
- multimodal_embeddings += tuple(image_embeddings)
- if modality == "videos":
- video_input = modalities["videos"]
- video_embeddings = self._process_video_input(video_input)
- multimodal_embeddings += tuple(video_embeddings)
-
- return multimodal_embeddings
-
- def forward(
- self,
- input_ids: torch.Tensor | None,
- positions: torch.Tensor,
- intermediate_tensors: IntermediateTensors | None = None,
- inputs_embeds: torch.Tensor | None = None,
- **kwargs: object,
- ) -> torch.Tensor | IntermediateTensors:
- if intermediate_tensors is not None:
- inputs_embeds = None
-
- hidden_states = self.language_model.model(
- input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds
- )
- return hidden_states
-
- def forward_images(
- self,
- pixel_values_images: list[torch.Tensor],
- image_sizes_images: torch.Tensor,
- ) -> tuple[torch.Tensor, ...]:
- pixel_values_image_flat = flatten_bn(pixel_values_images, concat=True)
-
- visual_token_idx = 0 if "siglip" in self.vision_config.model_type else 1
- image_forward_outs = self.vision_model(pixel_values_image_flat)[
- :, visual_token_idx:
- ]
-
- image_forward_outs = image_forward_outs.to(dtype=self.mm_projector.dtype)
- image_forward_outs = self.mm_projector(image_forward_outs) # b (h w) d
-
- split_sizes = [len(item) for item in pixel_values_images]
- image_forward_outs = torch.split(image_forward_outs, split_sizes, dim=0)
-
- # newline for anyres postprocessing
- image_features = anyres_postprocessing(
- image_forward_outs=image_forward_outs,
- image_sizes=image_sizes_images.tolist(),
- num_queries_vis_abstractor=self.config.num_queries_vis_abstractor_image,
- unpad=self.config.unpad,
- patch_size=self.vision_config.patch_size,
- grid_size=self.vision_config.image_size,
- image_newline=self.image_newline,
- possible_resolutions=self.config.possible_resolutions,
- )
-
- return tuple(image_features)
-
- def forward_videos(
- self,
- pixel_values_videos: list[list[torch.Tensor]],
- ) -> tuple[torch.Tensor, ...]:
- pixel_values_videos_flat = flatten_bn(
- [frame for frames in pixel_values_videos for frame in frames],
- concat=True,
- )
-
- visual_token_idx = 0 if "siglip" in self.vision_config.model_type else 1
- video_forward_outs = self.vision_model(pixel_values_videos_flat)[
- :, visual_token_idx:
- ]
-
- video_forward_outs = video_forward_outs.to(dtype=self.mm_projector.dtype)
-
- # Run MM-Projector
- # len(num_grids) == len(num_queries_vis_abstractors) + 1
- grid_idx = 0
- # e.g. [0, 9, 18, 19, 27, 28, 36, 37, 45, 46, 54, 55, 56]
- num_grids = [grid_idx]
- # e.g. [81, 81, 81, 9, 81, 9, 81, 9, 81, 9, 81, 9]
- num_queries_vis_abstractors = []
- len_total_frames = video_forward_outs.shape[0]
-
- if self.config.first_last_frames_slow:
- # slowfast (first_last_frames_slow)
- assert len_total_frames != 0
- if len_total_frames <= 2:
- num_queries_vis_abstractors.append(
- self.config.num_queries_vis_abstractor_video_slow
- )
- grid_idx += len_total_frames
- num_grids.append(grid_idx)
- else:
- num_queries_vis_abstractors.append(
- self.config.num_queries_vis_abstractor_video_slow
- )
- grid_idx += 1
- num_grids.append(grid_idx)
-
- num_queries_vis_abstractors.append(
- self.config.num_queries_vis_abstractor_video_fast
- )
- grid_idx += len_total_frames - 2
- num_grids.append(grid_idx)
-
- num_queries_vis_abstractors.append(
- self.config.num_queries_vis_abstractor_video_slow
- )
- grid_idx += 1
- num_grids.append(grid_idx)
- else:
- # slowfast
- for pixel_values_frames in pixel_values_videos:
- for pixel_values_frame in pixel_values_frames:
- if len(pixel_values_frame) > 0:
- num_queries_vis_abstractors.append(
- self.config.num_queries_vis_abstractor_video_slow
- )
- grid_idx += 1
- num_grids.append(grid_idx)
- num_queries_vis_abstractors.append(
- self.config.num_queries_vis_abstractor_video_fast
- )
- grid_idx = grid_idx + len(pixel_values_frame) - 1
- num_grids.append(grid_idx)
-
- video_forward_outs = self.mm_projector(
- video_forward_outs, num_queries_vis_abstractors, num_grids
- )
-
- video_features = [] # what we want to return
- target_features = []
- target_group_size = 0
- group_counter = 0
- video_groups = [
- len(frame) for frames in pixel_values_videos for frame in frames
- ] # for concat video features after projector
-
- for forward_out in video_forward_outs:
- target_group_size += len(forward_out)
- target_features.append(forward_out.flatten(0, 1))
-
- video_group_size = video_groups[group_counter]
- if video_group_size == target_group_size:
- video_features.append(torch.cat(target_features, dim=0))
- target_features = []
- group_counter += 1
- target_group_size = 0
-
- elif video_group_size < target_group_size:
- raise RuntimeError(f"{video_group_size=} < {target_group_size=}")
-
- assert len(target_features) == 0, (
- f"target_features is not empty!! {target_features}"
- )
- assert len(video_groups) == len(video_features)
-
- feats_per_video = [len(video) for video in pixel_values_videos]
- idxs_per_video = [0, *accumulate(feats_per_video)]
- return tuple(
- torch.cat(video_features[idxs_per_video[i] : idxs_per_video[i + 1]])
- for i in range(len(feats_per_video))
- )
-
- def compute_logits(
- self,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor | None:
- return self.language_model.compute_logits(hidden_states)
-
- def load_weights(
- self,
- weights: Iterable[tuple[str, torch.Tensor]],
- ) -> set[str]:
- loader = AutoWeightsLoader(self)
- return loader.load_weights(weights)
-
- def _init_possible_resolutions(
- self,
- config,
- vision_config,
- ):
- if not getattr(config, "possible_resolutions", []):
- possible_resolutions = []
- if config.anyres:
- assert config.max_num_grids > 0
- for i in range(1, config.max_num_grids + 1):
- for j in range(1, config.max_num_grids + 1):
- if i == 1 and j == 1 and not config.use_1x1_grid:
- continue
- if i * j <= config.max_num_grids:
- possible_resolutions.append([i, j])
-
- possible_resolutions = [
- [ys * vision_config.image_size, xs * vision_config.image_size]
- for ys, xs in possible_resolutions
- ]
- return possible_resolutions
- else:
- return config.possible_resolutions
-
- def _init_mm_projector(
- self,
- config,
- text_config,
- vision_config,
- ):
- input_hidden_size = vision_config.hidden_size
- if config.mm_projector_type == "linear":
- mm_projector = nn.Linear(input_hidden_size, text_config.hidden_size)
- mm_projector.dtype = next(mm_projector.parameters()).dtype
- elif config.mm_projector_type == "cabstractor":
- mm_projector = HCXVisionCAbstractor(
- num_queries=config.num_queries_vis_abstractor_image,
- num_input_tokens=(vision_config.image_size // vision_config.patch_size)
- ** 2,
- encoder_hidden_size=input_hidden_size,
- hidden_size=input_hidden_size,
- output_hidden_size=text_config.hidden_size,
- pos_emb=config.proj_pos_emb,
- prenorm=config.proj_prenorm,
- )
- else:
- mm_projector = HCXVisionMlp(
- config.mm_projector_type,
- input_hidden_size,
- hidden_features=input_hidden_size,
- out_features=self.text_config.hidden_size,
- )
- return mm_projector
-
-
-def unpad_image(tensor: torch.Tensor, original_size: tuple[int, int]) -> torch.Tensor:
- original_width, original_height = original_size
- current_height, current_width = tensor.shape[1:]
-
- original_aspect_ratio = original_width / original_height
- current_aspect_ratio = current_width / current_height
-
- if original_aspect_ratio > current_aspect_ratio:
- scale_factor = current_width / original_width
- new_height = int(original_height * scale_factor)
- padding = (current_height - new_height) // 2
- unpadded_tensor = tensor[:, padding : current_height - padding, :]
- else:
- scale_factor = current_height / original_height
- new_width = int(original_width * scale_factor)
- padding = (current_width - new_width) // 2
- unpadded_tensor = tensor[:, :, padding : current_width - padding]
-
- return unpadded_tensor
-
-
-def select_best_resolution(original_size: tuple, possible_resolutions: list) -> tuple:
- original_height, original_width = original_size
- best_fit = None
- max_effective_resolution = 0
- min_wasted_resolution = float("inf")
-
- for height, width in possible_resolutions:
- scale = min(width / original_width, height / original_height)
- downscaled_width, downscaled_height = (
- int(original_width * scale),
- int(original_height * scale),
- )
- effective_resolution = min(
- downscaled_width * downscaled_height, original_width * original_height
- )
- wasted_resolution = (width * height) - effective_resolution
-
- if effective_resolution > max_effective_resolution or (
- effective_resolution == max_effective_resolution
- and wasted_resolution < min_wasted_resolution
- ):
- max_effective_resolution = effective_resolution
- min_wasted_resolution = wasted_resolution
- best_fit = (height, width)
-
- return best_fit
-
-
-def get_anyres_image_grid_shape(
- image_size: tuple[int, int],
- grid_pinpoints: str | list[tuple[int, int]],
- patch_size: int,
-) -> tuple[int, int]:
- possible_resolutions = (
- grid_pinpoints
- if isinstance(grid_pinpoints, list)
- else ast.literal_eval(grid_pinpoints)
- )
-
- original_width, original_height = image_size
- height, width = select_best_resolution(
- (original_height, original_width), possible_resolutions
- )
- return width // patch_size, height // patch_size
-
-
-def reshape_and_unpad_image_features(
- image_feature: torch.Tensor,
- height: int,
- width: int,
- image_size: tuple[int, int],
- possible_resolutions: list[tuple[int, int]],
- grid_size: int,
- unpad: bool,
- image_newline: torch.Tensor,
-) -> torch.Tensor:
- base_image_feature = image_feature[0]
- image_feature = image_feature[1:]
-
- assert height * width == base_image_feature.shape[0], (
- f"{height=} * {width=} != {base_image_feature.shape[0]=}"
- )
-
- num_patch_width, num_patch_height = get_anyres_image_grid_shape(
- image_size, possible_resolutions, grid_size
- )
- image_feature = image_feature.view(
- num_patch_height, num_patch_width, height, width, -1
- )
-
- if unpad:
- image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous()
- image_feature = image_feature.flatten(1, 2).flatten(2, 3)
- image_feature = unpad_image(image_feature, image_size)
- image_feature = torch.cat(
- (
- image_feature,
- image_newline[:, None, None]
- .expand(*image_feature.shape[:-1], 1)
- .to(image_feature.device),
- ),
- dim=-1,
- )
- image_feature = image_feature.flatten(1, 2).transpose(0, 1)
- else:
- image_feature = image_feature.permute(0, 2, 1, 3, 4).contiguous()
- image_feature = image_feature.flatten(0, 3)
- image_feature = torch.cat((base_image_feature, image_feature), dim=0)
-
- return image_feature
-
-
-def anyres_postprocessing(
- image_forward_outs: list[torch.Tensor],
- image_sizes: list[list[int]],
- possible_resolutions: list[tuple[int, int]],
- patch_size: int,
- grid_size: int,
- image_newline: torch.Tensor,
- num_queries_vis_abstractor: int = -1,
- unpad: bool = False,
-) -> list[torch.Tensor]:
- height = width = grid_size // patch_size
-
- if num_queries_vis_abstractor > 0:
- assert (num_queries_vis_abstractor**0.5).is_integer(), (
- "n_queries must be square number"
- )
- height = width = int(num_queries_vis_abstractor**0.5)
-
- # post-processing (unpad, add newline)
- new_image_features = []
- for image_idx, image_feature in enumerate(image_forward_outs):
- if image_feature.shape[0] > 1:
- image_feature = reshape_and_unpad_image_features(
- image_feature=image_feature,
- height=height,
- width=width,
- image_size=image_sizes[image_idx],
- possible_resolutions=possible_resolutions,
- grid_size=grid_size, # Pass grid info if needed by helper
- unpad=unpad,
- image_newline=image_newline,
- )
- else:
- image_feature = image_feature[0]
- image_feature = torch.cat(
- (image_feature, image_newline[None].to(image_feature.device)), dim=0
- )
- new_image_features.append(image_feature)
-
- return new_image_features
diff --git a/vllm/model_executor/models/llama.py b/vllm/model_executor/models/llama.py
index d627ec76ca..ba2e3fef09 100644
--- a/vllm/model_executor/models/llama.py
+++ b/vllm/model_executor/models/llama.py
@@ -273,7 +273,7 @@ class LlamaDecoderLayer(nn.Module):
# By default, Llama uses causal attention as it is a decoder-only model.
# You can override the HF config with `is_causal=False` to enable
# bidirectional attention, which is used in some embedding models
- # (e.g. parasail-ai/GritLM-7B-vllm)
+ # (e.g. nvidia/llama-nemotron-embed-1b-v2)
if getattr(config, "is_causal", True):
attn_type = AttentionType.DECODER
else:
diff --git a/vllm/model_executor/models/mpt.py b/vllm/model_executor/models/mpt.py
deleted file mode 100644
index b8b4c5670e..0000000000
--- a/vllm/model_executor/models/mpt.py
+++ /dev/null
@@ -1,317 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-# Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
-import math
-from collections.abc import Iterable
-from itertools import islice
-
-import torch
-import torch.nn as nn
-from transformers import MptConfig
-
-from vllm.compilation.decorators import support_torch_compile
-from vllm.config import CacheConfig, VllmConfig
-from vllm.distributed import (
- get_pp_group,
- get_tensor_model_parallel_rank,
- get_tensor_model_parallel_world_size,
-)
-from vllm.model_executor.layers.activation import get_act_fn
-from vllm.model_executor.layers.attention import Attention
-from vllm.model_executor.layers.linear import (
- ColumnParallelLinear,
- QKVParallelLinear,
- RowParallelLinear,
-)
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
-from vllm.model_executor.layers.quantization import QuantizationConfig
-from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
-from vllm.sequence import IntermediateTensors
-
-from .interfaces import SupportsPP
-from .utils import (
- AutoWeightsLoader,
- make_empty_intermediate_tensors_factory,
- make_layers,
- maybe_prefix,
-)
-
-
-def _get_alibi_slopes(
- total_num_heads: int,
- alibi_bias_max: int,
-) -> torch.Tensor:
- next_power_of_2 = 2 ** math.ceil(math.log2(total_num_heads))
- m = torch.arange(1, next_power_of_2 + 1, dtype=torch.float32)
- m = m.mul(alibi_bias_max / next_power_of_2)
- slopes = 1.0 / torch.pow(2, m)
- if next_power_of_2 != total_num_heads:
- slopes = torch.concat([slopes[1::2], slopes[::2]])[:total_num_heads]
- return slopes
-
-
-class MPTAttention(nn.Module):
- def __init__(
- self,
- config: MptConfig,
- cache_config: CacheConfig | None = None,
- quant_config: QuantizationConfig | None = None,
- prefix: str = "",
- ):
- super().__init__()
- self.d_model = config.d_model
- self.total_num_heads = config.n_heads
- self.head_dim = self.d_model // self.total_num_heads
- self.clip_qkv = config.attn_config.clip_qkv
- self.qk_ln = config.attn_config.qk_ln
- self.alibi_bias_max = config.attn_config.alibi_bias_max
- if "kv_n_heads" in config.attn_config:
- self.total_num_kv_heads = config.attn_config.kv_n_heads
- else:
- self.total_num_kv_heads = self.total_num_heads
- assert not config.attn_config.prefix_lm
- assert config.attn_config.alibi
-
- # pylint: disable=invalid-name
- self.Wqkv = QKVParallelLinear(
- self.d_model,
- self.d_model // self.total_num_heads,
- self.total_num_heads,
- self.total_num_kv_heads,
- bias=not config.no_bias,
- quant_config=quant_config,
- prefix=f"{prefix}.Wqkv",
- )
- if self.qk_ln:
- self.q_ln = nn.LayerNorm(self.d_model)
- self.k_ln = nn.LayerNorm(self.d_model)
- self.out_proj = RowParallelLinear(
- self.d_model,
- self.d_model,
- bias=not config.no_bias,
- quant_config=quant_config,
- prefix=f"{prefix}.out_proj",
- )
-
- tp_world_size = get_tensor_model_parallel_world_size()
- assert self.total_num_heads % tp_world_size == 0
- self.num_heads = self.total_num_heads // tp_world_size
-
- if self.total_num_kv_heads >= tp_world_size:
- # Number of KV heads is greater than TP size, so we partition
- # the KV heads across multiple tensor parallel GPUs.
- assert self.total_num_kv_heads % tp_world_size == 0
- else:
- # Number of KV heads is less than TP size, so we replicate
- # the KV heads across multiple tensor parallel GPUs.
- assert tp_world_size % self.total_num_kv_heads == 0
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
- self.q_size = self.num_heads * self.head_dim
- self.kv_size = self.num_kv_heads * self.head_dim
- # Create the alibi slopes and slice them.
- tp_rank = get_tensor_model_parallel_rank()
- head_start = tp_rank * self.num_heads
- head_end = (tp_rank + 1) * self.num_heads
- alibi_slopes = _get_alibi_slopes(self.total_num_heads, self.alibi_bias_max)
- alibi_slopes = alibi_slopes[head_start:head_end].tolist()
-
- self.head_dim = self.d_model // self.total_num_heads
- scaling = self.head_dim**-0.5
- self.attn = Attention(
- self.num_heads,
- self.head_dim,
- scaling,
- alibi_slopes=alibi_slopes,
- num_kv_heads=self.num_kv_heads,
- cache_config=cache_config,
- quant_config=quant_config,
- prefix=f"{prefix}.attn",
- )
-
- def forward(
- self,
- position_ids: torch.Tensor,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor:
- del position_ids # unused.
- qkv, _ = self.Wqkv(hidden_states)
- if self.clip_qkv is not None:
- qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
- if self.qk_ln:
- q = self.q_ln(q)
- k = self.k_ln(k)
- attn_output = self.attn(q, k, v)
- output, _ = self.out_proj(attn_output)
- return output
-
-
-class MPTMLP(nn.Module):
- def __init__(
- self,
- config: MptConfig,
- quant_config: QuantizationConfig | None = None,
- prefix: str = "",
- ):
- super().__init__()
- hidden_size = config.d_model
- expansion_ratio = config.expansion_ratio
- intermediate_size = expansion_ratio * hidden_size
- self.up_proj = ColumnParallelLinear(
- hidden_size,
- intermediate_size,
- bias=not config.no_bias,
- quant_config=quant_config,
- prefix=f"{prefix}.up_proj",
- )
- self.act = get_act_fn("gelu")
- self.down_proj = RowParallelLinear(
- intermediate_size,
- hidden_size,
- bias=not config.no_bias,
- quant_config=quant_config,
- prefix=f"{prefix}.down_proj",
- )
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- x, _ = self.up_proj(x)
- x = self.act(x)
- x, _ = self.down_proj(x)
- return x
-
-
-class MPTBlock(nn.Module):
- def __init__(
- self,
- config: MptConfig,
- cache_config: CacheConfig | None = None,
- quant_config: QuantizationConfig | None = None,
- prefix: str = "",
- ):
- super().__init__()
- hidden_size = config.d_model
- self.norm_1 = nn.LayerNorm(hidden_size)
- self.attn = MPTAttention(
- config, cache_config, quant_config, prefix=f"{prefix}.attn"
- )
- self.norm_2 = nn.LayerNorm(hidden_size)
- self.ffn = MPTMLP(config, quant_config, prefix=f"{prefix}.ffn")
-
- def forward(
- self,
- position_ids: torch.Tensor,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor:
- x = self.norm_1(hidden_states)
- x = self.attn(
- position_ids=position_ids,
- hidden_states=x,
- )
- hidden_states = hidden_states + x
- x = self.norm_2(hidden_states)
- x = self.ffn(x)
- hidden_states = hidden_states + x
- return hidden_states
-
-
-@support_torch_compile
-class MPTModel(nn.Module):
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
-
- config = vllm_config.model_config.hf_config
- cache_config = vllm_config.cache_config
- quant_config = vllm_config.quant_config
-
- assert config.embedding_fraction == 1.0
- assert config.norm_type == "low_precision_layernorm"
-
- self.wte = VocabParallelEmbedding(
- config.vocab_size,
- config.d_model,
- )
- self.start_layer, self.end_layer, self.blocks = make_layers(
- config.n_layers,
- lambda prefix: MPTBlock(config, cache_config, quant_config, prefix=prefix),
- prefix=f"{prefix}.blocks",
- )
- self.norm_f = nn.LayerNorm(config.d_model)
- if config.no_bias:
- for module in self.modules():
- if hasattr(module, "bias") and isinstance(module.bias, nn.Parameter):
- # Remove the bias term in Linear and LayerNorm.
- module.register_parameter("bias", None)
- self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
- ["hidden_states"], config.d_model
- )
-
- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
- return self.wte(input_ids)
-
- def forward(
- self,
- input_ids: torch.Tensor | None,
- position_ids: torch.Tensor,
- intermediate_tensors: IntermediateTensors | None,
- inputs_embeds: torch.Tensor | None = None,
- ) -> torch.Tensor | IntermediateTensors:
- if get_pp_group().is_first_rank:
- if inputs_embeds is not None:
- hidden_states = inputs_embeds
- else:
- hidden_states = self.embed_input_ids(input_ids)
- else:
- assert intermediate_tensors is not None
- hidden_states = intermediate_tensors["hidden_states"]
-
- for block in islice(self.blocks, self.start_layer, self.end_layer):
- hidden_states = block(position_ids, hidden_states)
- if not get_pp_group().is_last_rank:
- return IntermediateTensors({"hidden_states": hidden_states})
- hidden_states = self.norm_f(hidden_states)
- return hidden_states
-
-
-class MPTForCausalLM(nn.Module, SupportsPP):
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
- config = vllm_config.model_config.hf_config
- quant_config = vllm_config.quant_config
- self.config = config
- self.quant_config = quant_config
-
- self.transformer = MPTModel(
- vllm_config=vllm_config, prefix=maybe_prefix(prefix, "transformer")
- )
- self.lm_head = self.transformer.wte
- self.logits_processor = LogitsProcessor(config.vocab_size)
- self.make_empty_intermediate_tensors = (
- self.transformer.make_empty_intermediate_tensors
- )
-
- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
- return self.transformer.embed_input_ids(input_ids)
-
- def forward(
- self,
- input_ids: torch.Tensor | None,
- positions: torch.Tensor,
- intermediate_tensors: IntermediateTensors | None = None,
- inputs_embeds: torch.Tensor | None = None,
- ) -> torch.Tensor | IntermediateTensors:
- hidden_states = self.transformer(
- input_ids, positions, intermediate_tensors, inputs_embeds
- )
- return hidden_states
-
- def compute_logits(
- self,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor | None:
- logits = self.logits_processor(self.lm_head, hidden_states)
- return logits
-
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
- loader = AutoWeightsLoader(self)
- return loader.load_weights(weights)
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index c6043716a8..8c3bf4c07b 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -74,7 +74,6 @@ _TEXT_GENERATION_MODELS = {
"AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
"ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
"ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
- "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
"AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
"BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
"BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
@@ -102,7 +101,6 @@ _TEXT_GENERATION_MODELS = {
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
"Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
"ExaoneMoeForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
- "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
"FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
"FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
@@ -129,12 +127,10 @@ _TEXT_GENERATION_MODELS = {
"GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
"GraniteMoeSWAForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
- "GritLM": ("gritlm", "GritLM"),
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
"HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
"HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
- "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
@@ -178,9 +174,6 @@ _TEXT_GENERATION_MODELS = {
"MistralForCausalLM": ("mistral", "MistralForCausalLM"),
"MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
- # transformers's mpt class has lower case
- "MptForCausalLM": ("mpt", "MPTForCausalLM"),
- "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
"MiMoForCausalLM": ("mimo", "MiMoForCausalLM"),
"MiMoV2FlashForCausalLM": ("mimo_v2", "MiMoV2FlashForCausalLM"),
"MiMoV2ForCausalLM": ("mimo_v2", "MiMoV2ForCausalLM"),
@@ -207,14 +200,12 @@ _TEXT_GENERATION_MODELS = {
"Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
"Qwen3_5ForCausalLM": ("qwen3_5", "Qwen3_5ForCausalLM"),
"Qwen3_5MoeForCausalLM": ("qwen3_5", "Qwen3_5MoeForCausalLM"),
- "RWForCausalLM": ("falcon", "FalconForCausalLM"),
"SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
"SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
"SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
"Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
"Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
"Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
- "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
"SolarForCausalLM": ("solar", "SolarForCausalLM"),
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
@@ -234,7 +225,6 @@ _EMBEDDING_MODELS = {
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
"Gemma3TextModel": ("gemma3", "Gemma3Model"),
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
- "GritLM": ("gritlm", "GritLM"),
"GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
"GteNewModel": ("bert_with_rope", "GteNewModel"),
"JinaEmbeddingsV5Model": ("jina", "JinaEmbeddingsV5Model"),
@@ -274,7 +264,6 @@ _EMBEDDING_MODELS = {
# Technically Terratorch models work on images, both in
# input and output. I am adding it here because it piggy-backs on embedding
# models for the time being.
- "PrithviGeoSpatialMAE": ("terratorch", "Terratorch"),
"Terratorch": ("terratorch", "Terratorch"),
}
@@ -360,12 +349,6 @@ _MULTIMODAL_MODELS = {
"BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
"BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
- "ChameleonForConditionalGeneration": (
- "chameleon",
- "ChameleonForConditionalGeneration",
- ),
- "Cheers": ("cheers", "CheersForConditionalGeneration"),
- "CheersForConditionalGeneration": ("cheers", "CheersForConditionalGeneration"),
"Cohere2VisionForConditionalGeneration": (
"cohere2_vision",
"Cohere2VisionForConditionalGeneration",
@@ -401,10 +384,6 @@ _MULTIMODAL_MODELS = {
"FireRedASR2ForConditionalGeneration",
),
"FunASRForConditionalGeneration": ("funasr", "FunASRForConditionalGeneration"),
- "FireRedLIDForConditionalGeneration": (
- "fireredlid",
- "FireRedLIDForConditionalGeneration",
- ),
"FunAudioChatForConditionalGeneration": (
"funaudiochat",
"FunAudioChatForConditionalGeneration",
@@ -811,6 +790,20 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
"FuyuForCausalLM": "0.25.0",
"Plamo2ForCausalLM": "0.26.0",
"OuroForCausalLM": "0.26.0",
+ "ArcticForCausalLM": "0.28.0",
+ "ChameleonForConditionalGeneration": "0.28.0",
+ "Cheers": "0.28.0",
+ "CheersForConditionalGeneration": "0.28.0",
+ "Fairseq2LlamaForCausalLM": "0.28.0",
+ "FireRedLIDForConditionalGeneration": "0.28.0",
+ "GritLM": "0.28.0",
+ "HCXVisionForCausalLM": "0.28.0",
+ "MPTForCausalLM": "0.28.0",
+ "MptForCausalLM": "0.28.0",
+ "RWForCausalLM": "0.28.0",
+ "StableLMEpochForCausalLM": "0.28.0",
+ # Superseded by the generic "Terratorch" architecture
+ "PrithviGeoSpatialMAE": "0.28.0",
}
_OOT_SUPPORTED_MODELS = {
diff --git a/vllm/transformers_utils/chat_templates/registry.py b/vllm/transformers_utils/chat_templates/registry.py
index 9af4217be6..422854ab3d 100644
--- a/vllm/transformers_utils/chat_templates/registry.py
+++ b/vllm/transformers_utils/chat_templates/registry.py
@@ -24,7 +24,6 @@ def _get_minicpmv_chat_template_fallback(tokenizer_name_or_path: str) -> Path |
_MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
"blip-2": CHAT_TEMPLATES_DIR / "template_blip2.jinja",
- "chameleon": CHAT_TEMPLATES_DIR / "template_basic.jinja",
"clip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
"colpali": CHAT_TEMPLATES_DIR / "template_basic.jinja",
"deepseek_ocr": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py
index 26d4bd6e1b..65b6b133a9 100644
--- a/vllm/transformers_utils/config.py
+++ b/vllm/transformers_utils/config.py
@@ -71,10 +71,8 @@ class LazyConfigDict(dict):
_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
afmoe="AfmoeConfig",
- arctic="ArcticConfig",
axk1="AXK1Config",
bagel="BagelConfig",
- umm="CheersConfig",
chatglm="ChatGLMConfig",
modernvbert="ColModernVBertConfig",
colpali="ColPaliConfig",
@@ -90,11 +88,9 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
dots3_note="Dots3NoteConfig",
k3_dspark="K3DSparkConfig",
flex_olmo="FlexOlmoConfig",
- fireredlid="FireRedLIDConfig",
funaudiochat="FunAudioChatConfig",
granite4_vision="Granite4VisionConfig",
hyperclovax="HyperCLOVAXConfig",
- hyperclovax_vlm="HCXVisionConfig",
hunyuan_vl="HunYuanVLConfig",
hy_v3="HYV3Config",
isaac="IsaacConfig",
diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py
index c38cc4b4b8..625f0636b8 100644
--- a/vllm/transformers_utils/configs/__init__.py
+++ b/vllm/transformers_utils/configs/__init__.py
@@ -16,10 +16,8 @@ import importlib
_CLASS_TO_MODULE: dict[str, str] = {
"AfmoeConfig": "vllm.transformers_utils.configs.afmoe",
- "ArcticConfig": "vllm.transformers_utils.configs.arctic",
"AXK1Config": "vllm.transformers_utils.configs.AXK1",
"BagelConfig": "vllm.transformers_utils.configs.bagel",
- "CheersConfig": "vllm.transformers_utils.configs.cheers",
"ChatGLMConfig": "vllm.transformers_utils.configs.chatglm",
"ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert",
"ColPaliConfig": "vllm.transformers_utils.configs.colpali",
@@ -39,7 +37,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"K3DSparkConfig": "vllm.transformers_utils.configs.k3_dspark",
"DotsOCRConfig": "vllm.transformers_utils.configs.dotsocr",
"EAGLEConfig": "vllm.transformers_utils.configs.eagle",
- "FireRedLIDConfig": "vllm.transformers_utils.configs.fireredlid",
"FlexOlmoConfig": "vllm.transformers_utils.configs.flex_olmo",
"FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
@@ -47,7 +44,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
"HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl",
"HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl",
"HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl",
- "HCXVisionConfig": "vllm.transformers_utils.configs.hyperclovax",
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
@@ -112,10 +108,8 @@ _CLASS_TO_MODULE: dict[str, str] = {
__all__ = [
"AfmoeConfig",
- "ArcticConfig",
"AXK1Config",
"BagelConfig",
- "CheersConfig",
"ChatGLMConfig",
"ColModernVBertConfig",
"ColPaliConfig",
@@ -137,14 +131,12 @@ __all__ = [
"DotsOCRConfig",
"EAGLEConfig",
"FlexOlmoConfig",
- "FireRedLIDConfig",
"FunAudioChatConfig",
"FunAudioChatAudioEncoderConfig",
"Granite4VisionConfig",
"HunYuanVLConfig",
"HunYuanVLTextConfig",
"HunYuanVLVisionConfig",
- "HCXVisionConfig",
"HYV3Config",
"HyperCLOVAXConfig",
"IsaacConfig",
diff --git a/vllm/transformers_utils/configs/arctic.py b/vllm/transformers_utils/configs/arctic.py
deleted file mode 100644
index ba4b1a8f70..0000000000
--- a/vllm/transformers_utils/configs/arctic.py
+++ /dev/null
@@ -1,216 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-# ruff: noqa: E501
-# coding=utf-8
-# Copied from
-# https://huggingface.co/Snowflake/snowflake-arctic-instruct/blob/main/configuration_arctic.py
-"""Arctic model configuration"""
-
-from dataclasses import asdict, dataclass
-from typing import Any
-
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import logging
-
-logger = logging.get_logger(__name__)
-
-ARCTIC_PRETRAINED_CONFIG_ARCHIVE_MAP = {
- "arctic": "https://huggingface.co/Snowflake/snowflake-arctic-instruct/tree/main/config.json",
-}
-
-
-@dataclass
-class ArcticLoRAConfig:
- lora_r: int = 64
- lora_alpha: float = 16
- shard_base_weights: bool = False
-
-
-@dataclass
-class ArcticQuantizationConfig:
- q_bits: int = 8
- rounding: str = "nearest"
- mantissa_bits: int = 3
- group_size: int = 128
-
-
-class ArcticConfig(PretrainedConfig):
- r"""
- This is the configuration class to store the configuration of a [`ArcticModel`]. It is used to instantiate an
- Arctic model according to the specified arguments, defining the model architecture. Instantiating a configuration
- with the defaults will yield a similar configuration to that of the #TODO(rsamdani): add what model has the default config..
-
-
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
- documentation from [`PretrainedConfig`] for more information.
-
-
- Args:
- vocab_size (`int`, *optional*, defaults to 32000):
- Vocabulary size of the Arctic model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`ArcticModel`]
- hidden_size (`int`, *optional*, defaults to 4096):
- Dimension of the hidden representations.
- intermediate_size (`int`, *optional*, defaults to 14336):
- Dimension of the MLP representations.
- num_hidden_layers (`int`, *optional*, defaults to 32):
- Number of hidden layers in the Transformer encoder.
- num_attention_heads (`int`, *optional*, defaults to 32):
- Number of attention heads for each attention layer in the Transformer encoder.
- num_key_value_heads (`int`, *optional*, defaults to 8):
- This is the number of key_value heads that should be used to implement Grouped Query Attention. If
- `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
- `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
- converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
- by meanpooling all the original heads within that group. For more details checkout [this
- paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
- hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
- The non-linear activation function (function or string) in the decoder.
- max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
- The maximum sequence length that this model might ever be used with. Arctic's sliding window attention
- allows sequence of up to 4096*32 tokens.
- initializer_range (`float`, *optional*, defaults to 0.02):
- The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
- rms_norm_eps (`float`, *optional*, defaults to 1e-05):
- The epsilon used by the rms normalization layers.
- use_cache (`bool`, *optional*, defaults to `True`):
- Whether or not the model should return the last key/values attentions (not used by all models). Only
- relevant if `config.is_decoder=True`.
- pad_token_id (`int`, *optional*):
- The id of the padding token.
- bos_token_id (`int`, *optional*, defaults to 1):
- The id of the "beginning-of-sequence" token.
- eos_token_id (`int`, *optional*, defaults to 2):
- The id of the "end-of-sequence" token.
- tie_word_embeddings (`bool`, *optional*, defaults to `False`):
- Whether the model's input and output word embeddings should be tied.
- rope_parameters (`dict`, *optional*):
- Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
- and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
- accordingly.
- Expected contents:
- `rope_theta` (`float`): The base period of the RoPE embeddings.
- `rope_type` (`str`):
- The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
- 'llama3'], with 'default' being the original RoPE implementation.
- sliding_window (`int`, *optional*):
- Sliding window attention window size. If not specified, will default to `4096`.
- attention_dropout (`float`, *optional*, defaults to 0.0):
- The dropout ratio for the attention probabilities.
- num_experts_per_tok (`int`, *optional*, defaults to 2):
- The number of experts to root per-token, can be also interpreted as the `top-p` routing
- parameter
- num_local_experts (`int`, *optional*, defaults to 8):
- Number of experts per Sparse MLP layer.
- router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
- The aux loss factor for the total loss.
-
- ```python
- >>> from transformers import ArcticModel, ArcticConfig
-
- >>> # Initializing a Arctic 7B style configuration TODO(rsamdani): verify which model does the default configuration correspond to.
- >>> configuration = ArcticConfig()
-
- >>> # Initializing a model from the Arctic 7B style configuration
- >>> model = ArcticModel(configuration)
-
- >>> # Accessing the model configuration
- >>> configuration = model.config
- ```"""
-
- model_type = "arctic"
- keys_to_ignore_at_inference = ["past_key_values"]
-
- def __init__(
- self,
- vocab_size=32000,
- hidden_size=4096,
- intermediate_size=14336,
- num_hidden_layers=32,
- num_attention_heads=32,
- num_key_value_heads=None,
- hidden_act="silu",
- max_position_embeddings=4096,
- initializer_range=0.02,
- rms_norm_eps=1e-5,
- use_cache=True,
- pad_token_id=None,
- bos_token_id=1,
- eos_token_id=2,
- tie_word_embeddings=False,
- rope_parameters: dict[str, Any] | None = None,
- sliding_window=None,
- attention_dropout=0.0,
- num_experts_per_tok=1,
- num_local_experts=8,
- router_aux_loss_coef=0.001,
- moe_layer_frequency=2,
- parallel_attn_mlp_res=False,
- moe_train_capacity_factor=1,
- moe_eval_capacity_factor=1,
- enable_expert_tensor_parallelism=False,
- moe_min_capacity=0,
- moe_token_dropping=True,
- quantization=None,
- **kwargs,
- ):
- self.vocab_size = vocab_size
- self.max_position_embeddings = max_position_embeddings
- self.hidden_size = hidden_size
- self.intermediate_size = intermediate_size
- self.num_hidden_layers = num_hidden_layers
- self.num_attention_heads = num_attention_heads
- self.sliding_window = sliding_window
-
- # for backward compatibility
- if num_key_value_heads is None:
- num_key_value_heads = num_attention_heads
-
- self.num_key_value_heads = num_key_value_heads
- self.hidden_act = hidden_act
- self.initializer_range = initializer_range
- self.rms_norm_eps = rms_norm_eps
- self.use_cache = use_cache
- rope_theta = kwargs.pop("rope_theta", 1e6)
- if rope_parameters is None:
- rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
- self.rope_parameters = rope_parameters
- self.attention_dropout = attention_dropout
-
- self.num_experts_per_tok = num_experts_per_tok
- self.num_local_experts = num_local_experts
- self.router_aux_loss_coef = router_aux_loss_coef
- self.moe_layer_frequency = moe_layer_frequency
- self.moe_train_capacity_factor = moe_train_capacity_factor
- self.moe_eval_capacity_factor = moe_eval_capacity_factor
- self.enable_expert_tensor_parallelism = enable_expert_tensor_parallelism
- self.moe_min_capacity = moe_min_capacity
- self.moe_token_dropping = moe_token_dropping
- self.parallel_attn_mlp_res = parallel_attn_mlp_res
- if isinstance(quantization, dict):
- self.quantization = ArcticQuantizationConfig(**quantization)
- else:
- self.quantization = quantization
-
- super().__init__(
- pad_token_id=pad_token_id,
- bos_token_id=bos_token_id,
- eos_token_id=eos_token_id,
- tie_word_embeddings=tie_word_embeddings,
- **kwargs,
- )
-
- @classmethod
- def from_dict(cls, config_dict: dict[str, Any], **kwargs) -> "ArcticConfig":
- result = super().from_dict(config_dict, **kwargs)
- config = result[0] if isinstance(result, tuple) else result
- if isinstance(config.quantization, dict):
- config.quantization = ArcticQuantizationConfig(**config.quantization)
- return result
-
- def to_dict(self) -> dict[str, Any]:
- ret = super().to_dict()
- if isinstance(ret["quantization"], ArcticQuantizationConfig):
- ret["quantization"] = asdict(ret["quantization"])
- return ret
diff --git a/vllm/transformers_utils/configs/cheers.py b/vllm/transformers_utils/configs/cheers.py
deleted file mode 100644
index e00d19761a..0000000000
--- a/vllm/transformers_utils/configs/cheers.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-from transformers import PretrainedConfig, SiglipVisionConfig
-from transformers.modeling_rope_utils import rope_config_validation
-
-
-class CheersTextConfig(PretrainedConfig):
- """Qwen2-based text config with Cheers-specific defaults."""
-
- model_type = "umm"
- base_config_key = "text_config"
-
- def __init__(
- self,
- vocab_size=152064,
- hidden_size=3584,
- intermediate_size=18944,
- num_hidden_layers=28,
- num_attention_heads=28,
- num_key_value_heads=4,
- hidden_act="silu",
- max_position_embeddings=131072,
- initializer_range=0.02,
- rms_norm_eps=1e-6,
- use_cache=True,
- tie_word_embeddings=False,
- rope_theta=1000000.0,
- rope_scaling=None,
- use_sliding_window=False,
- sliding_window=131072,
- max_window_layers=28,
- layer_types=None,
- attention_dropout=0.0,
- **kwargs,
- ):
- self.vocab_size = vocab_size
- self.max_position_embeddings = max_position_embeddings
- self.hidden_size = hidden_size
- self.intermediate_size = intermediate_size
- self.num_hidden_layers = num_hidden_layers
- self.num_attention_heads = num_attention_heads
- self.use_sliding_window = use_sliding_window
- self.sliding_window = sliding_window if self.use_sliding_window else None
- self.max_window_layers = max_window_layers
- if num_key_value_heads is None:
- num_key_value_heads = num_attention_heads
- self.num_key_value_heads = num_key_value_heads
- self.hidden_act = hidden_act
- self.initializer_range = initializer_range
- self.rms_norm_eps = rms_norm_eps
- self.use_cache = use_cache
- self.rope_theta = rope_theta
- self.rope_scaling = rope_scaling
- self.attention_dropout = attention_dropout
- if self.rope_scaling is not None and "type" in self.rope_scaling:
- self.rope_scaling["rope_type"] = self.rope_scaling["type"]
- rope_config_validation(self)
-
- self.layer_types = layer_types
- if self.layer_types is None:
- self.layer_types = [
- "sliding_attention"
- if self.sliding_window is not None and i >= self.max_window_layers
- else "full_attention"
- for i in range(self.num_hidden_layers)
- ]
-
- super().__init__(
- tie_word_embeddings=tie_word_embeddings,
- **kwargs,
- )
-
-
-class CheersConfig(PretrainedConfig):
- """Configuration class for Cheers (UMM) model."""
-
- model_type = "umm"
-
- def __init__(
- self,
- text_config: dict | CheersTextConfig | None = None,
- vision_representation_config: dict | SiglipVisionConfig | None = None,
- vae_encoder_config: dict | None = None,
- vae_decoder_config: dict | None = None,
- **kwargs,
- ):
- super().__init__(**kwargs)
-
- if isinstance(text_config, dict):
- self.text_config = CheersTextConfig(**text_config)
- else:
- self.text_config = text_config or CheersTextConfig()
-
- if isinstance(vision_representation_config, dict):
- self.vision_representation_config = SiglipVisionConfig(
- **vision_representation_config
- )
- else:
- self.vision_representation_config = (
- vision_representation_config or SiglipVisionConfig()
- )
-
- self.vae_encoder_config = vae_encoder_config or {"resolution": 512}
- self.vae_decoder_config = vae_decoder_config or {"resolution": 512}
-
- @property
- def hidden_size(self) -> int:
- """Return the hidden size of the language model."""
- return self.text_config.hidden_size
diff --git a/vllm/transformers_utils/configs/fireredlid.py b/vllm/transformers_utils/configs/fireredlid.py
deleted file mode 100644
index a71062d084..0000000000
--- a/vllm/transformers_utils/configs/fireredlid.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-from __future__ import annotations
-
-import contextlib
-
-from transformers import AutoConfig
-from transformers.configuration_utils import PretrainedConfig
-
-
-class FireRedLIDConfig(PretrainedConfig):
- """Minimal config class for native vLLM FireRedLID support."""
-
- model_type = "fireredlid"
- keys_to_ignore_at_inference = ["past_key_values"]
-
- def __init__(
- self,
- vocab_size: int = 120,
- lid_odim: int = 120,
- idim: int = 80,
- d_model: int = 1280,
- n_head: int = 20,
- n_layers_enc: int = 16,
- n_layers_lid_dec: int = 6,
- kernel_size: int = 33,
- residual_dropout: float = 0.05,
- dropout_rate: float = 0.05,
- pe_maxlen: int = 5000,
- pad_token_id: int = 2,
- bos_token_id: int = 3,
- eos_token_id: int = 4,
- decoder_start_token_id: int = 3,
- tie_word_embeddings: bool = True,
- is_encoder_decoder: bool = True,
- architectures: list[str] | None = None,
- **kwargs,
- ):
- self.vocab_size = vocab_size
- self.lid_odim = lid_odim
- self.idim = idim
- self.d_model = d_model
- self.hidden_size = d_model
- self.n_head = n_head
- self.num_attention_heads = n_head
- self.n_layers_enc = n_layers_enc
- self.encoder_layers = n_layers_enc
- self.n_layers_lid_dec = n_layers_lid_dec
- self.decoder_layers = n_layers_lid_dec
- self.num_hidden_layers = n_layers_lid_dec
- self.kernel_size = kernel_size
- self.residual_dropout = residual_dropout
- self.dropout_rate = dropout_rate
- self.pe_maxlen = pe_maxlen
- self.tie_word_embeddings = tie_word_embeddings
- self.is_encoder_decoder = is_encoder_decoder
- self.architectures = architectures or ["FireRedLIDForConditionalGeneration"]
-
- super().__init__(
- pad_token_id=pad_token_id,
- bos_token_id=bos_token_id,
- eos_token_id=eos_token_id,
- decoder_start_token_id=decoder_start_token_id,
- tie_word_embeddings=tie_word_embeddings,
- is_encoder_decoder=is_encoder_decoder,
- architectures=self.architectures,
- **kwargs,
- )
-
-
-with contextlib.suppress(ValueError):
- AutoConfig.register(FireRedLIDConfig.model_type, FireRedLIDConfig)
diff --git a/vllm/transformers_utils/configs/hyperclovax.py b/vllm/transformers_utils/configs/hyperclovax.py
index d1a3218fe4..9fa823743d 100644
--- a/vllm/transformers_utils/configs/hyperclovax.py
+++ b/vllm/transformers_utils/configs/hyperclovax.py
@@ -17,7 +17,6 @@
# limitations under the License.
"""HyperCLOVA X model configuration."""
-from transformers import AutoConfig
from transformers.configuration_utils import PretrainedConfig
@@ -276,74 +275,3 @@ class HyperCLOVAXConfig(PretrainedConfig):
auto_map=auto_map,
**kwargs,
)
-
-
-class HCXVisionConfig(PretrainedConfig):
- """Vendored HyperCLOVAX Vision config with transformers v5 fix.
-
- The original remote code config does not handle empty initialization
- (text_config=None), which breaks transformers v5's @strict validation.
-
- TODO: Remove this class once HyperCLOVAX is upstreamed to transformers.
- Tracking PR: https://github.com/huggingface/transformers/pull/44956
- """
-
- model_type = "hyperclovax_vlm"
- keys_to_ignore_at_inference = ["past_key_values"]
-
- text_config_attribute_map = {
- "n_embd": "hidden_size",
- "n_positions": "max_position_embeddings",
- "n_head": "num_attention_heads",
- "n_layer": "num_hidden_layers",
- }
-
- def __init__(
- self,
- text_config=None,
- vision_config=None,
- use_nth_layer=-2,
- img_start_id=100009,
- decoder_max_length=4096,
- anyres=False,
- unpad=False,
- max_num_grids=-1,
- num_queries_vis_abstractor=-1,
- ignore_index=-100,
- proj_pos_emb=True,
- proj_prenorm=False,
- use_1x1_grid=False,
- **kwargs,
- ):
- for key, val in self.text_config_attribute_map.items():
- if text_config is not None and key in text_config:
- text_config[val] = text_config.pop(key)
-
- self.text_config = None
- if text_config is not None:
- _text_config = AutoConfig.for_model(text_config["model_type"])
- self.text_config = _text_config.from_dict(text_config)
- self.hidden_size = self.text_config.hidden_size
-
- self.vision_config = None
- if vision_config is not None:
- _vision_config = AutoConfig.for_model(vision_config["model_type"])
- self.vision_config = _vision_config.from_dict(vision_config)
-
- self.use_nth_layer = use_nth_layer
- self.decoder_max_length = decoder_max_length
- self.anyres = anyres
- self.unpad = unpad
- self.max_num_grids = max_num_grids
- self.num_queries_vis_abstractor = num_queries_vis_abstractor
- self.img_start_id = img_start_id
- self.ignore_index = ignore_index
- self.proj_pos_emb = proj_pos_emb
- self.proj_prenorm = proj_prenorm
- self.use_1x1_grid = use_1x1_grid
- super().__init__(**kwargs)
-
- def get_text_config(self, decoder=False):
- if self.text_config is not None:
- return self.text_config
- return self
diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py
index 4b65e2584c..b6d720b7af 100644
--- a/vllm/transformers_utils/model_arch_config_convertor.py
+++ b/vllm/transformers_utils/model_arch_config_convertor.py
@@ -373,7 +373,6 @@ class ModelArchConfigConvertorBase:
"molmo2",
"moondream3",
"paligemma",
- "umm",
)
if not hasattr(self.hf_config, "model_type"):
return False
@@ -545,13 +544,6 @@ class FalconModelArchConfigConvertor(ModelArchConfigConvertorBase):
return super().get_total_num_kv_heads()
-class MPTModelArchConfigConvertor(ModelArchConfigConvertorBase):
- def get_total_num_kv_heads(self) -> int:
- if "kv_n_heads" in self.hf_text_config.attn_config:
- return self.hf_text_config.attn_config["kv_n_heads"]
- return self.hf_text_config.num_attention_heads
-
-
class DbrxModelArchConfigConvertor(ModelArchConfigConvertorBase):
def get_total_num_kv_heads(self) -> int:
return getattr(
@@ -792,7 +784,6 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
"mimo_v2_mtp": MimoV2MTPModelArchConfigConvertor,
"mimo_v2_omni_mtp": MimoV2MTPModelArchConfigConvertor,
"moss_audio": MossAudioModelArchConfigConvertor,
- "mpt": MPTModelArchConfigConvertor,
"nemotron-nas": NemotronNasModelArchConfigConvertor,
"bailing_hybrid_v3_mtp": BailingHybridV3MTPModelArchConfigConvertor,
"pangu_ultra_moe_mtp": PanguUltraMoeMTPModelArchConfigConvertor,
diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py
index 64a94950d4..8fd924fcf3 100644
--- a/vllm/transformers_utils/processor.py
+++ b/vllm/transformers_utils/processor.py
@@ -37,9 +37,7 @@ def _transformers_v4_compatibility_import():
subset of `ProcessorChatTemplateKwargs` as defined in Transformers v4.
In Transformers v5 these were merged into `ProcessorChatTemplateKwargs` and
`ChatTemplateLoadKwargs` was removed. For backward compatibility, we add an alias
- for `ChatTemplateLoadKwargs` if it doesn't exist.
-
- This can be removed if `HCXVisionForCausalLM` is upstreamed to Transformers."""
+ for `ChatTemplateLoadKwargs` if it doesn't exist."""
old_import = getattr(processing_utils, "ChatTemplateLoadKwargs", None)
new_import = getattr(processing_utils, "ProcessorChatTemplateKwargs", None)
if old_import is None and new_import is not None:
diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py
index 4fa7d2fee8..b5e4607761 100644
--- a/vllm/transformers_utils/processors/__init__.py
+++ b/vllm/transformers_utils/processors/__init__.py
@@ -12,12 +12,10 @@ import importlib
__all__ = [
"BagelProcessor",
- "CheersProcessor",
"CohereASRProcessor",
"Cosmos3EdgeProcessor",
"DeepseekVLV2Processor",
"FireRedASR2Processor",
- "FireRedLIDProcessor",
"FunASRProcessor",
"GLM4VProcessor",
"Granite4VisionProcessor",
@@ -52,12 +50,10 @@ __all__ = [
_CLASS_TO_MODULE: dict[str, str] = {
"BagelProcessor": "vllm.transformers_utils.processors.bagel",
- "CheersProcessor": "vllm.transformers_utils.processors.cheers",
"CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr",
"Cosmos3EdgeProcessor": "vllm.transformers_utils.processors.cosmos3_edge",
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
- "FireRedLIDProcessor": "vllm.transformers_utils.processors.fireredlid",
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
"Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
diff --git a/vllm/transformers_utils/processors/cheers.py b/vllm/transformers_utils/processors/cheers.py
deleted file mode 100644
index 68eecbcffe..0000000000
--- a/vllm/transformers_utils/processors/cheers.py
+++ /dev/null
@@ -1,89 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Cheers (UMM) processor for image and text inputs."""
-
-from transformers import AutoProcessor
-from transformers.feature_extraction_utils import BatchFeature
-from transformers.image_utils import ImageInput
-from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
-from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
-
-
-class CheersProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
- _defaults = {
- "images_kwargs": {
- "return_tensors": "pt",
- },
- }
-
-
-class CheersProcessor(ProcessorMixin):
- """
- Constructs a Cheers processor which wraps a
- SigLIP image processor and a Qwen2 tokenizer.
- """
-
- attributes = ["image_processor", "tokenizer"]
- image_processor_class = "AutoImageProcessor"
- tokenizer_class = "AutoTokenizer"
-
- def __call__(
- self,
- text: TextInput
- | PreTokenizedInput
- | list[TextInput]
- | list[PreTokenizedInput] = None,
- images: ImageInput = None,
- **kwargs: Unpack[CheersProcessorKwargs],
- ):
- output_kwargs = self._merge_kwargs(
- CheersProcessorKwargs,
- tokenizer_init_kwargs=self.tokenizer.init_kwargs,
- **kwargs,
- )
-
- if images is not None:
- import torch
-
- if isinstance(images, (list, tuple)):
- all_pv = []
- all_ghw = []
- for img in images:
- result = self.image_processor(img, **output_kwargs["images_kwargs"])
- all_pv.append(result["pixel_values"])
- if "grid_hws" in result:
- all_ghw.append(result["grid_hws"])
- pixel_values = {
- "pixel_values": torch.cat(all_pv, dim=0),
- }
- if all_ghw:
- pixel_values["grid_hws"] = torch.cat(all_ghw, dim=0)
- else:
- pixel_values = self.image_processor(
- images, **output_kwargs["images_kwargs"]
- )
- else:
- pixel_values = {}
-
- text_inputs = (
- self.tokenizer(text, **output_kwargs["text_kwargs"])
- if text is not None
- else {}
- )
-
- return BatchFeature(data={**pixel_values, **text_inputs})
-
- def batch_decode(self, *args, **kwargs):
- return self.tokenizer.batch_decode(*args, **kwargs)
-
- def decode(self, *args, **kwargs):
- return self.tokenizer.decode(*args, **kwargs)
-
- @property
- def model_input_names(self):
- tokenizer_input_names = self.tokenizer.model_input_names
- image_processor_input_names = self.image_processor.model_input_names
- return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
-
-
-AutoProcessor.register("CheersProcessor", CheersProcessor)
diff --git a/vllm/transformers_utils/processors/fireredlid.py b/vllm/transformers_utils/processors/fireredlid.py
deleted file mode 100644
index 3afdca4c7b..0000000000
--- a/vllm/transformers_utils/processors/fireredlid.py
+++ /dev/null
@@ -1,276 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-FireRedLID feature extractor and processor.
-
-The FeatureExtractor handles:
- - Raw waveform → 80-dim log-mel filterbank (via kaldi_native_fbank)
- - CMVN normalization (means / inverse_std_variences from preprocessor_config)
- - Padding + length tracking
-
-The Processor wraps the FeatureExtractor and a tokenizer.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import numpy as np
-import torch
-import torch.nn.functional as F
-from transformers import (
- AutoFeatureExtractor,
- BatchFeature,
-)
-from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
-from transformers.processing_utils import ProcessorMixin
-from transformers.utils import TensorType
-
-from vllm.logger import init_logger
-from vllm.utils.import_utils import LazyLoader
-
-if TYPE_CHECKING:
- import kaldi_native_fbank as knf
-else:
- knf = LazyLoader("knf", globals(), "kaldi_native_fbank")
-
-
-logger = init_logger(__name__)
-
-
-# ---------------------------------------------------------------------------
-# Helpers (shared with FireRedASR2 processor)
-# ---------------------------------------------------------------------------
-
-
-class CMVN:
- def __init__(self, dim, means, inverse_std_variences):
- self.dim = dim
- self.means = np.array(means)
- self.inverse_std_variences = np.array(inverse_std_variences)
-
- def __call__(self, x):
- assert x.shape[-1] == self.dim, "CMVN dim mismatch"
- out = x - self.means
- out = out * self.inverse_std_variences
- return out
-
-
-class KaldifeatFbank:
- def __init__(
- self,
- num_mel_bins: int = 80,
- frame_length: int = 25,
- frame_shift: int = 10,
- dither: float = 0.0,
- ):
- self.dither = dither
- opts = knf.FbankOptions()
- opts.frame_opts.dither = dither
- opts.mel_opts.num_bins = num_mel_bins
- opts.frame_opts.snip_edges = True
- opts.mel_opts.debug_mel = False
- self.opts = opts
-
- def __call__(self, sample_rate, wav_np, is_train=False):
- dither = self.dither if is_train else 0.0
- self.opts.frame_opts.dither = dither
- fbank = knf.OnlineFbank(self.opts)
- fbank.accept_waveform(sample_rate, wav_np.tolist())
- feat = []
- for i in range(fbank.num_frames_ready):
- feat.append(fbank.get_frame(i))
- if len(feat) == 0:
- return np.zeros((0, self.opts.mel_opts.num_bins))
- return np.vstack(feat)
-
-
-# ---------------------------------------------------------------------------
-# Feature Extractor
-# ---------------------------------------------------------------------------
-
-
-class FireRedLIDFeatureExtractor(SequenceFeatureExtractor):
- """
- Extracts 80-dim log-mel filterbank features from raw waveforms,
- applies CMVN, and returns padded feature tensors with lengths.
-
- Also computes ``fake_token_lengths`` — the actual encoder output
- length for each audio — so that vLLM can allocate the correct
- number of cross-attention KV cache slots.
- """
-
- model_input_names = ["input_features"]
-
- def __init__(
- self,
- feature_size=80,
- sampling_rate=16000,
- chunk_length=30,
- padding_value=0.0,
- return_attention_mask=False,
- dim=80,
- means=None,
- inverse_std_variences=None,
- num_mel_bins=80,
- frame_length=25,
- frame_shift=10,
- dither=0.0,
- left_context=3,
- right_context=3,
- **kwargs,
- ):
- super().__init__(
- feature_size=feature_size,
- sampling_rate=sampling_rate,
- padding_value=padding_value,
- return_attention_mask=return_attention_mask,
- **kwargs,
- )
- self.chunk_length = chunk_length
- self.dim = dim
- self.means = means
- self.inverse_std_variences = inverse_std_variences
- self.num_mel_bins = num_mel_bins
- self.frame_length = frame_length
- self.frame_shift = frame_shift
- self.dither = dither
- self.sampling_rate = sampling_rate
- self.context = left_context + 1 + right_context
-
- def __call__(
- self,
- raw_speech: np.ndarray | list[float] | list[np.ndarray] | list[list[float]],
- truncation: bool = True,
- pad_to_multiple_of: int | None = None,
- return_tensors: str | TensorType | None = None,
- return_attention_mask: bool | None = None,
- padding: str | None = "max_length",
- max_length: int | None = None,
- sampling_rate: int | None = None,
- do_normalize: bool | None = None,
- **kwargs,
- ) -> BatchFeature:
- if sampling_rate is not None and sampling_rate != self.sampling_rate:
- raise ValueError(
- f"FireRedLIDFeatureExtractor expects sampling_rate="
- f"{self.sampling_rate}, got {sampling_rate}."
- )
-
- # Initialize helpers
- cmvn = CMVN(self.dim, self.means, self.inverse_std_variences)
- fbank = KaldifeatFbank(
- num_mel_bins=self.num_mel_bins,
- frame_length=self.frame_length,
- frame_shift=self.frame_shift,
- dither=self.dither,
- )
-
- def padding_position_is_0(padded_input, input_lengths):
- N, T = padded_input.size()[:2]
- mask = torch.ones((N, T)).to(padded_input.device)
- for i in range(N):
- mask[i, input_lengths[i] :] = 0
- mask = mask.unsqueeze(dim=1)
- return mask.to(torch.uint8)
-
- feats = []
- speech_lengths = []
- fake_token_lengths = []
-
- for speech in raw_speech:
- # vLLM loads audio via librosa (float32 in [-1,1]),
- # but kaldi_native_fbank expects int16-scale values.
- speech_scaled = speech * 32768
- feat = fbank(self.sampling_rate, speech_scaled)
- feat = cmvn(feat)
- feat = torch.from_numpy(feat).float()
- length = feat.size(0)
- feats.append(feat)
- speech_lengths.append(length)
-
- # Compute the actual Conv2dSubsampling output length.
- # This mirrors the mask logic in Conv2dSubsampling.forward:
- # pad context frames, then mask[:, :, :-2:2][:, :, :-2:2].sum()
- padded_input = F.pad(feat, (0, 0, 0, self.context - 1), "constant", 0.0)
- src_mask = padding_position_is_0(
- padded_input[None, :, :],
- torch.tensor([length], dtype=torch.int32),
- )
- mask = src_mask[:, :, :-2:2][:, :, :-2:2]
- enc_len = mask[:, -1, :].sum(dim=-1)
- fake_token_len = torch.clamp(enc_len, min=1)
- fake_token_lengths.append(fake_token_len)
-
- if len(feats) == 0:
- return BatchFeature()
-
- # Pad to uniform length
- max_feat_len = max(f.size(0) for f in feats)
- padded = feats[0].new_zeros(len(feats), max_feat_len, feats[0].size(1))
- for i, feat in enumerate(feats):
- padded[i, : feat.size(0)] = feat
-
- result = BatchFeature({"input_features": padded})
-
- if return_tensors is not None:
- result = result.convert_to_tensors(return_tensors)
-
- result["speech_lengths"] = torch.tensor(speech_lengths, dtype=torch.long)
- result["fake_token_lengths"] = torch.concat(fake_token_lengths)
- return result
-
-
-# ---------------------------------------------------------------------------
-# Processor
-# ---------------------------------------------------------------------------
-
-
-class FireRedLIDProcessor(ProcessorMixin):
- """
- Wraps FireRedLIDFeatureExtractor + a tokenizer.
- """
-
- feature_extractor_class = "FireRedLIDFeatureExtractor"
- tokenizer_class = ("PythonBackend", "TokenizersBackend")
-
- def __init__(self, feature_extractor, tokenizer):
- super().__init__(feature_extractor, tokenizer)
- self.current_processor = self.feature_extractor
- self._in_target_context_manager = False
-
- def __call__(self, *args, **kwargs):
- if self._in_target_context_manager:
- return self.current_processor(*args, **kwargs)
-
- audio = kwargs.pop("audio", None)
- sampling_rate = kwargs.pop("sampling_rate", None)
- text = kwargs.pop("text", None)
- if len(args) > 0:
- audio = args[0]
- args = args[1:]
-
- if audio is not None:
- inputs = self.feature_extractor(
- audio, *args, sampling_rate=sampling_rate, **kwargs
- )
- else:
- inputs = BatchFeature()
-
- if text is not None:
- if isinstance(text, str):
- text = [text]
- encodings = self.tokenizer(text, **kwargs)
- if audio is not None:
- inputs["labels"] = encodings["input_ids"]
- else:
- return encodings
-
- return inputs
-
-
-# ---------------------------------------------------------------------------
-# Registration
-# ---------------------------------------------------------------------------
-
-AutoFeatureExtractor.register("FireRedLIDFeatureExtractor", FireRedLIDFeatureExtractor)
diff --git a/vllm/v1/worker/gpu/model_states/encoder_decoder.py b/vllm/v1/worker/gpu/model_states/encoder_decoder.py
index 93c738fb22..5bd069312e 100644
--- a/vllm/v1/worker/gpu/model_states/encoder_decoder.py
+++ b/vllm/v1/worker/gpu/model_states/encoder_decoder.py
@@ -43,7 +43,7 @@ class EncoderDecoderAttnMetadata(ModelSpecificAttnMetadata):
class EncoderDecoderModelState(ModelState):
"""ModelState for cross-attention encoder-decoder models
- (Whisper, CohereASR, NemotronParse, FireRedLID, ...)
+ (Whisper, CohereASR, NemotronParse, ...)
"""
def __init__(