[Bugfix] Honor OPT token embedding width for prompt embeddings (#61736)
Signed-off-by: maithilijoshi20 <maithilij2003@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
b466fbacd2
commit
92dc3dd393
@@ -91,11 +91,17 @@ def safe_load_prompt_embeds(
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
|
||||
# Pin each tensor to the model's hidden_size. Validating here
|
||||
# Pin each tensor to the model's text embedding width. Validating here
|
||||
# also transitively guarantees cross-tensor consistency for requests that
|
||||
# include multiple `prompt_embeds` parts, which is required by downstream
|
||||
# concatenation in `_build_mixed_prompt_embeds`.
|
||||
expected_hidden_size = model_config.get_hidden_size()
|
||||
# OPT projects token embeddings to hidden_size inside its decoder. CLIP
|
||||
# and SigLIP still accept text embeddings at hidden_size, not projection_dim.
|
||||
expected_hidden_size = getattr(
|
||||
model_config.hf_text_config,
|
||||
"word_embed_proj_dim",
|
||||
model_config.get_hidden_size(),
|
||||
)
|
||||
if tensor.shape[1] != expected_hidden_size:
|
||||
raise VLLMValidationError(
|
||||
f"`prompt_embeds` hidden_size {tensor.shape[1]} does not match "
|
||||
|
||||
Reference in New Issue
Block a user