[DisaggEverything] Tokens in<>out /generate endpoint (#24261)

Signed-off-by: NickLucche <nlucches@redhat.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Nicolò Lucchesi
2025-11-14 09:58:01 -07:00
committed by GitHub
co-authored by Harry Mellor
parent d54a18a47e
commit 6f1e7f7226
12 changed files with 822 additions and 9 deletions
+63 -2
View File
@@ -8,7 +8,7 @@ from collections.abc import Callable, Sequence
from functools import partial
from inspect import isclass
from types import FunctionType
from typing import Any, TypeAlias
from typing import Any, TypeAlias, get_type_hints
import cloudpickle
import msgspec
@@ -16,6 +16,8 @@ import numpy as np
import torch
import zmq
from msgspec import msgpack
from pydantic import GetCoreSchemaHandler
from pydantic_core import core_schema
from vllm import envs
from vllm.logger import init_logger
@@ -32,7 +34,6 @@ from vllm.multimodal.inputs import (
NestedTensors,
)
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.v1.engine import UtilityResult
from vllm.v1.utils import tensor_data
logger = init_logger(__name__)
@@ -104,6 +105,13 @@ def _decode_type_info_recursive(
return convert_fn(type_info, data)
class UtilityResult:
"""Wrapper for special handling when serializing/deserializing."""
def __init__(self, r: Any = None):
self.result = r
class MsgpackEncoder:
"""Encoder with custom torch tensor and numpy array serialization.
@@ -469,3 +477,56 @@ def run_method(
else:
func = partial(method, obj) # type: ignore
return func(*args, **kwargs)
class PydanticMsgspecMixin:
@classmethod
def __get_pydantic_core_schema__(
cls, source_type: Any, handler: GetCoreSchemaHandler
) -> core_schema.CoreSchema:
"""
Make msgspec.Struct compatible with Pydantic, respecting defaults.
Handle JSON=>msgspec.Struct. Used when exposing msgspec.Struct to the
API as input or in `/docs`. Note this is cached by Pydantic and not
called on every validation.
"""
msgspec_fields = {f.name: f for f in msgspec.structs.fields(source_type)}
type_hints = get_type_hints(source_type)
# Build the Pydantic typed_dict_field for each msgspec field
fields = {}
for name, hint in type_hints.items():
msgspec_field = msgspec_fields[name]
# typed_dict_field using the handler to get the schema
field_schema = handler(hint)
# Add default value to the schema.
if msgspec_field.default_factory is not msgspec.NODEFAULT:
wrapped_schema = core_schema.with_default_schema(
schema=field_schema,
default_factory=msgspec_field.default_factory,
)
fields[name] = core_schema.typed_dict_field(wrapped_schema)
elif msgspec_field.default is not msgspec.NODEFAULT:
wrapped_schema = core_schema.with_default_schema(
schema=field_schema,
default=msgspec_field.default,
)
fields[name] = core_schema.typed_dict_field(wrapped_schema)
else:
# No default, so Pydantic will treat it as required
fields[name] = core_schema.typed_dict_field(field_schema)
return core_schema.no_info_after_validator_function(
cls._validate_msgspec,
core_schema.typed_dict_schema(fields),
)
@classmethod
def _validate_msgspec(cls, value: Any) -> Any:
"""Validate and convert input to msgspec.Struct instance."""
if isinstance(value, cls):
return value
if isinstance(value, dict):
return cls(**value)
return msgspec.convert(value, type=cls)