Support embedding models in V1 (#16188)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Signed-off-by: Max de Bayser <maxdebayser@gmail.com>
Signed-off-by: 22quinn <33176974+22quinn@users.noreply.github.com>
Co-authored-by: 22quinn <33176974+22quinn@users.noreply.github.com>
This commit is contained in:
Maximilien de Bayser
2025-06-18 21:36:33 -07:00
committed by GitHub
co-authored by 22quinn
parent 4959915089
commit 799397ee4f
56 changed files with 889 additions and 281 deletions
+1 -1
View File
@@ -1266,7 +1266,7 @@ class LLM:
# the tokenizer for models such as
# "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
# lists of tokens to the `text` and `text_pair` kwargs
tokenizer = self.llm_engine.get_tokenizer()
tokenizer = self.get_tokenizer()
def ensure_str(prompt: SingletonPrompt):
if isinstance(prompt, dict):
+3 -1
View File
@@ -9,6 +9,7 @@ from typing import Final, Literal, Optional, Union, cast
import jinja2
import numpy as np
import torch
from fastapi import Request
from typing_extensions import assert_never
@@ -39,7 +40,8 @@ def _get_data(
elif encoding_format == "base64":
# Force to use float32 for base64 encoding
# to match the OpenAI python client behavior
pooling_bytes = np.array(output.data, dtype="float32").tobytes()
pt_float32 = output.data.to(dtype=torch.float32)
pooling_bytes = np.array(pt_float32, dtype="float32").tobytes()
return base64.b64encode(pooling_bytes).decode("utf-8")
assert_never(encoding_format)