[Bugfix] Honor OPT token embedding width for prompt embeddings (#61736)

Signed-off-by: maithilijoshi20 <maithilij2003@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
maithilijoshi20
2026-10-16 04:41:50 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent b466fbacd2
commit 92dc3dd393
9 changed files with 60 additions and 9 deletions
+8 -2
View File
@@ -91,11 +91,17 @@ def safe_load_prompt_embeds(
parameter="prompt_embeds",
)
# Pin each tensor to the model's hidden_size. Validating here
# Pin each tensor to the model's text embedding width. Validating here
# also transitively guarantees cross-tensor consistency for requests that
# include multiple `prompt_embeds` parts, which is required by downstream
# concatenation in `_build_mixed_prompt_embeds`.
expected_hidden_size = model_config.get_hidden_size()
# OPT projects token embeddings to hidden_size inside its decoder. CLIP
# and SigLIP still accept text embeddings at hidden_size, not projection_dim.
expected_hidden_size = getattr(
model_config.hf_text_config,
"word_embed_proj_dim",
model_config.get_hidden_size(),
)
if tensor.shape[1] != expected_hidden_size:
raise VLLMValidationError(
f"`prompt_embeds` hidden_size {tensor.shape[1]} does not match "