[Frontend] Serve LiquidAI/d1-3B on /v1/systemone (#60676)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Simon Veitner <sveitner@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Misha Goin
2026-10-12 13:00:47 -04:00
committed by GitHub
co-authored by Simon Veitner Claude
parent d79b52b182
commit fa39582756
3 changed files with 222 additions and 17 deletions
@@ -18,6 +18,7 @@ from functools import cached_property
from typing import Any, ClassVar, TypeVar
import numpy as np
from PIL import Image
import vllm.envs as envs
from vllm.config import ModelConfig
@@ -92,6 +93,16 @@ class ReadStrategy(ABC):
def __init__(self, context: ReadContext):
self.context = context
@cached_property
def _connector(self) -> MediaConnector:
model_config = self.context.engine_client.model_config
return MEDIA_CONNECTOR_REGISTRY.load(
envs.VLLM_MEDIA_CONNECTOR,
media_io_kwargs=model_config.get_multimodal_config().media_io_kwargs,
allowed_local_media_path=model_config.allowed_local_media_path,
allowed_media_domains=model_config.allowed_media_domains,
)
@abstractmethod
def limits(self) -> DecisionLimits: ...
@@ -162,12 +173,25 @@ def type_label_ids(
return label_ids
def cap_pixels(image: Image.Image, max_pixels: int) -> Image.Image:
"""Shrink ``image`` to at most ``max_pixels`` pixels (bicubic)."""
w, h = image.size
if w * h <= max_pixels:
return image
scale = math.sqrt(max_pixels / (w * h))
size = (max(1, int(w * scale)), max(1, int(h * scale)))
return image.resize(size, Image.Resampling.BICUBIC)
class NextTokenStrategy(ReadStrategy):
"""Autoregressive models. Each question is one request: the state, then the
question with its labeled options, and the logprobs of the label tokens as
the reply's first token. The requests share the state. With prefix caching,
it is prefilled once."""
#: Images are shrunk to at most this many pixels before the processor.
max_image_pixels: ClassVar[int | None] = None
def __init__(self, context: ReadContext):
super().__init__(context)
# Every prompt ends with the same generation prompt, so a label's token
@@ -210,6 +234,27 @@ class NextTokenStrategy(ReadStrategy):
"""The engine input for a read of the rendered prompt."""
return engine_input
def _label_groups(self, question: Question) -> list[list[int]]:
"""The tokens that count for each label. A label scores its best one."""
return [[self.label_ids[label]] for label in question.labels]
def _content(self, state: str, question: Question) -> str:
"""The user turn: the state, then the question."""
return f"{state}\n\n{question.type.prompt(question)}"
async def _image_parts(self, urls: Sequence[str]) -> list[dict[str, Any]]:
"""The images as chat content parts, fetched once so that every
question reads the same pictures."""
if not urls:
return []
try:
images = await asyncio.gather(*map(self._connector.fetch_image_async, urls))
except ValueError as e:
raise StructuredDecisionError(f"media: {e}") from e
if self.max_image_pixels is not None:
images = [cap_pixels(image, self.max_image_pixels) for image in images]
return [{"type": "image_pil", "image_pil": image} for image in images]
def limits(self) -> DecisionLimits:
return DecisionLimits(max_questions=64, max_options=len(LABELS))
@@ -234,6 +279,9 @@ class NextTokenStrategy(ReadStrategy):
default_template=ctx.chat_template,
default_template_content_format=ctx.chat_template_content_format,
default_template_kwargs=ctx.default_chat_template_kwargs,
# A read that fails before it reaches the engine must not leave the
# sender cache believing the engine has the images.
skip_mm_cache=True,
)
prompt_ids = extract_prompt_components(
ctx.engine_client.model_config, engine_input
@@ -256,10 +304,14 @@ class NextTokenStrategy(ReadStrategy):
) -> list[QuestionRead]:
ctx = self.context
read_request = self._read_request(chat_template_kwargs, cache_salt)
image_parts = await self._image_parts((media or {}).get("image", ()))
slots, engine_inputs, params = [], [], []
for q in questions:
messages = [{"role": "user", "content": f"{state}\n\n{q.type.prompt(q)}"}]
content: Any = self._content(state, q)
if image_parts:
content = [*image_parts, {"type": "text", "text": content}]
messages = [{"role": "user", "content": content}]
if instructions:
messages.insert(0, {"role": "system", "content": instructions})
engine_input, prompt_ids = await self._render(read_request, messages)
@@ -268,9 +320,11 @@ class NextTokenStrategy(ReadStrategy):
"these chat options end the prompt differently, so the "
"labels' tokens are unknown"
)
slots.append([self.label_ids[label] for label in q.labels])
slots.append(self._label_groups(q))
engine_inputs.append(self._read_input(engine_input, prompt_ids))
params.append(self._sampling_params(slots[-1], prompt_ids))
params.append(
self._sampling_params([i for ids in slots[-1] for i in ids], prompt_ids)
)
label_reads = await next_token_label_reads(
ctx.engine_client,
@@ -283,14 +337,17 @@ class NextTokenStrategy(ReadStrategy):
)
reads = []
for ids, label_read in zip(slots, label_reads):
for groups, label_read in zip(slots, label_reads):
output = label_read.result.outputs[0]
logprobs = iter(label_read.logprobs)
reads.append(
QuestionRead(
probs=label_softmax(label_read.logprobs),
probs=label_softmax(
[max(next(logprobs) for _ in g) for g in groups]
),
label_mass=sum(math.exp(lp) for lp in label_read.logprobs),
argmax_is_label=bool(output.token_ids)
and output.token_ids[0] in ids,
and any(output.token_ids[0] in ids for ids in groups),
input_tokens=len(label_read.result.prompt_token_ids or ()),
output_tokens=len(output.token_ids),
)
@@ -373,6 +430,88 @@ class DiffusionGemmaCanvasStrategy(NextTokenStrategy):
)
class LiquidStrategy(NextTokenStrategy):
"""Liquid decision models (d1). The prompt and the label tokens are the
ones the checkpoint was trained with, from the `prompt.py` it ships."""
modalities = frozenset({"image"})
# runner.py bounds every picture at 1024 x 1024 pixels before the processor.
max_image_pixels = 1024 * 1024
def render_state(self, state: Any) -> str:
if state is None:
return ""
if isinstance(state, str):
return state
return json.dumps(state, ensure_ascii=False, indent=2)
def _single_tokens(self, texts: Sequence[str]) -> list[int]:
ids: list[int] = []
for text in texts:
encoded = self._tokenizer().encode(text, add_special_tokens=False)
if len(encoded) == 1 and encoded[0] not in ids:
ids.append(encoded[0])
return ids
@staticmethod
def _codes(question: Question) -> list[str]:
"""A choice's codes are A to Z, or the option names when those are
already single letters."""
names = [o.name.strip() for o in question.options]
if all(len(n) == 1 and n.isalpha() for n in names):
return names
return list(question.labels)
def _label_groups(self, question: Question) -> list[list[int]]:
if question.type.name == "noul":
forms: list[Sequence[str]] = [("yes", "Yes", "YES"), ("no", "No", "NO")]
elif question.type.name == "choice":
forms = [(code, f" {code}") for code in self._codes(question)]
else:
forms = [(label,) for label in question.labels]
groups = [self._single_tokens(f) for f in forms]
seen: set[int] = set()
for (label, *_), ids in zip(forms, groups):
if not ids or self._single_tokens([label]) != ids[:1] or seen & set(ids):
raise StructuredDecisionError(
f"question {question.id!r}: label {label!r} is not one "
"distinct token for this model"
)
seen.update(ids)
return groups
def _content(self, state: str, question: Question) -> str:
q = question
if q.type.name == "choice":
lines = "\n".join(
f"{code} {o.description or o.name.replace('_', ' ')}"
for code, o in zip(self._codes(q), q.options)
)
ask = (
f"{q.instructions}\n\nOptions:\n{lines}\n\n"
"Reply with the option code only."
)
elif q.type.name == "noul":
yes, no = (o.description for o in q.options)
sides = f"\nYes: {yes}\nNo: {no}" if yes or no else ""
ask = f"{q.instructions}{sides}\n\nReply with yes or no only."
elif q.type.name == "score":
legend = "\n".join(
f"{label} {o.name}" for label, o in zip(q.labels, q.options)
)
ask = (
f"{q.instructions}\n\n{legend}\n\n"
f"Reply with a single digit 0-{len(q.options) - 1} only."
)
else:
raise StructuredDecisionError(
f"question {q.id!r}: this model does not answer {q.type.name!r} "
"questions"
)
# A null state leaves the images, or the question alone, as the prompt.
return f"{state}\n\n\nQUESTION:\n{ask}" if state else ask
class JointSchemaStrategy(ReadStrategy):
"""Clef-Omni. The whole request is one pooling read: the state, then every
question with its options, as the model was trained to read them. The
@@ -442,16 +581,6 @@ class JointSchemaStrategy(ReadStrategy):
def _tokens(self, text: str) -> list[int]:
return self.tokenizer.encode(text, add_special_tokens=False)
@cached_property
def _connector(self) -> MediaConnector:
model_config = self.context.engine_client.model_config
return MEDIA_CONNECTOR_REGISTRY.load(
envs.VLLM_MEDIA_CONNECTOR,
media_io_kwargs=model_config.get_multimodal_config().media_io_kwargs,
allowed_local_media_path=model_config.allowed_local_media_path,
allowed_media_domains=model_config.allowed_media_domains,
)
async def _fetch(
self, urls: Sequence[str], media_io: MediaIO[_M], timeout: int
) -> list[_M]:
@@ -682,6 +811,7 @@ class JointSchemaStrategy(ReadStrategy):
READ_STRATEGIES: dict[str, type[ReadStrategy]] = {
"ClefOmniForJointSchema": JointSchemaStrategy,
"Lfm2VlForConditionalGeneration": LiquidStrategy,
"Qwen3ForCausalLM": NextTokenStrategy,
"Qwen3_5ForConditionalGeneration": NextTokenStrategy,
"Qwen3_5MoeForConditionalGeneration": NextTokenStrategy,