[Tests] Release HF embedding weights after prompt extraction (#57868)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
6654bcbfca
commit
8ec00ec2bc
@@ -158,7 +158,9 @@ def test_models(
|
|||||||
hf_model.model.device
|
hf_model.model.device
|
||||||
)
|
)
|
||||||
if prompt_embeds is not None:
|
if prompt_embeds is not None:
|
||||||
embed = hf_model.model.get_input_embeddings()(token_ids)
|
# Retained prompt values must not keep reference weights alive.
|
||||||
|
with torch.no_grad():
|
||||||
|
embed = hf_model.model.get_input_embeddings()(token_ids)
|
||||||
|
|
||||||
# MiniCPM models apply scale_emb to embeddings internally.
|
# MiniCPM models apply scale_emb to embeddings internally.
|
||||||
# vLLM expects pre-scaled embeddings when using inputs_embeds.
|
# vLLM expects pre-scaled embeddings when using inputs_embeds.
|
||||||
|
|||||||
Reference in New Issue
Block a user