[DisaggEverything] Tokens in<>out /generate endpoint (#24261)
Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
d54a18a47e
commit
6f1e7f7226
+63
-2
@@ -8,7 +8,7 @@ from collections.abc import Callable, Sequence
|
||||
from functools import partial
|
||||
from inspect import isclass
|
||||
from types import FunctionType
|
||||
from typing import Any, TypeAlias
|
||||
from typing import Any, TypeAlias, get_type_hints
|
||||
|
||||
import cloudpickle
|
||||
import msgspec
|
||||
@@ -16,6 +16,8 @@ import numpy as np
|
||||
import torch
|
||||
import zmq
|
||||
from msgspec import msgpack
|
||||
from pydantic import GetCoreSchemaHandler
|
||||
from pydantic_core import core_schema
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
@@ -32,7 +34,6 @@ from vllm.multimodal.inputs import (
|
||||
NestedTensors,
|
||||
)
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.v1.engine import UtilityResult
|
||||
from vllm.v1.utils import tensor_data
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -104,6 +105,13 @@ def _decode_type_info_recursive(
|
||||
return convert_fn(type_info, data)
|
||||
|
||||
|
||||
class UtilityResult:
|
||||
"""Wrapper for special handling when serializing/deserializing."""
|
||||
|
||||
def __init__(self, r: Any = None):
|
||||
self.result = r
|
||||
|
||||
|
||||
class MsgpackEncoder:
|
||||
"""Encoder with custom torch tensor and numpy array serialization.
|
||||
|
||||
@@ -469,3 +477,56 @@ def run_method(
|
||||
else:
|
||||
func = partial(method, obj) # type: ignore
|
||||
return func(*args, **kwargs)
|
||||
|
||||
|
||||
class PydanticMsgspecMixin:
|
||||
@classmethod
|
||||
def __get_pydantic_core_schema__(
|
||||
cls, source_type: Any, handler: GetCoreSchemaHandler
|
||||
) -> core_schema.CoreSchema:
|
||||
"""
|
||||
Make msgspec.Struct compatible with Pydantic, respecting defaults.
|
||||
Handle JSON=>msgspec.Struct. Used when exposing msgspec.Struct to the
|
||||
API as input or in `/docs`. Note this is cached by Pydantic and not
|
||||
called on every validation.
|
||||
"""
|
||||
msgspec_fields = {f.name: f for f in msgspec.structs.fields(source_type)}
|
||||
type_hints = get_type_hints(source_type)
|
||||
|
||||
# Build the Pydantic typed_dict_field for each msgspec field
|
||||
fields = {}
|
||||
for name, hint in type_hints.items():
|
||||
msgspec_field = msgspec_fields[name]
|
||||
|
||||
# typed_dict_field using the handler to get the schema
|
||||
field_schema = handler(hint)
|
||||
|
||||
# Add default value to the schema.
|
||||
if msgspec_field.default_factory is not msgspec.NODEFAULT:
|
||||
wrapped_schema = core_schema.with_default_schema(
|
||||
schema=field_schema,
|
||||
default_factory=msgspec_field.default_factory,
|
||||
)
|
||||
fields[name] = core_schema.typed_dict_field(wrapped_schema)
|
||||
elif msgspec_field.default is not msgspec.NODEFAULT:
|
||||
wrapped_schema = core_schema.with_default_schema(
|
||||
schema=field_schema,
|
||||
default=msgspec_field.default,
|
||||
)
|
||||
fields[name] = core_schema.typed_dict_field(wrapped_schema)
|
||||
else:
|
||||
# No default, so Pydantic will treat it as required
|
||||
fields[name] = core_schema.typed_dict_field(field_schema)
|
||||
return core_schema.no_info_after_validator_function(
|
||||
cls._validate_msgspec,
|
||||
core_schema.typed_dict_schema(fields),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _validate_msgspec(cls, value: Any) -> Any:
|
||||
"""Validate and convert input to msgspec.Struct instance."""
|
||||
if isinstance(value, cls):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return cls(**value)
|
||||
return msgspec.convert(value, type=cls)
|
||||
|
||||
Reference in New Issue
Block a user