[V1] Use msgpack for core request serialization (#12918)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
+14
-28
@@ -1,20 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import enum
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, List, Optional, Union
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import msgspec
|
||||
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal import MultiModalKwargs
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.metrics.stats import SchedulerStats
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal import MultiModalKwargs
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
# These are possible values of RequestOutput.finish_reason,
|
||||
# so form part of the external API.
|
||||
FINISH_REASON_STRINGS = ("stop", "length", "abort")
|
||||
@@ -39,8 +36,11 @@ class FinishReason(enum.IntEnum):
|
||||
return FINISH_REASON_STRINGS[self.value]
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineCoreRequest:
|
||||
class EngineCoreRequest(
|
||||
msgspec.Struct,
|
||||
array_like=True, # type: ignore[call-arg]
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False): # type: ignore[call-arg]
|
||||
|
||||
# NOTE: prompt and prompt_token_ids should be DecoderOnlyInput,
|
||||
# but this object is currently not playing well with msgspec
|
||||
@@ -51,13 +51,13 @@ class EngineCoreRequest:
|
||||
# Detokenizer, but set to None when it is added to EngineCoreClient.
|
||||
prompt: Optional[str]
|
||||
prompt_token_ids: List[int]
|
||||
mm_inputs: Optional[List[Optional["MultiModalKwargs"]]]
|
||||
mm_inputs: Optional[List[Optional[MultiModalKwargs]]]
|
||||
mm_hashes: Optional[List[str]]
|
||||
mm_placeholders: Optional[List["PlaceholderRange"]]
|
||||
sampling_params: "SamplingParams"
|
||||
mm_placeholders: Optional[List[PlaceholderRange]]
|
||||
sampling_params: SamplingParams
|
||||
eos_token_id: Optional[int]
|
||||
arrival_time: float
|
||||
lora_request: Optional["LoRARequest"]
|
||||
lora_request: Optional[LoRARequest]
|
||||
|
||||
|
||||
class EngineCoreOutput(
|
||||
@@ -94,16 +94,6 @@ class EngineCoreOutputs(
|
||||
scheduler_stats: SchedulerStats
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineCoreProfile:
|
||||
is_start: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineCoreResetPrefixCache:
|
||||
pass
|
||||
|
||||
|
||||
class EngineCoreRequestType(enum.Enum):
|
||||
"""
|
||||
Request types defined as hex byte strings, so it can be sent over sockets
|
||||
@@ -113,7 +103,3 @@ class EngineCoreRequestType(enum.Enum):
|
||||
ABORT = b'\x01'
|
||||
PROFILE = b'\x02'
|
||||
RESET_PREFIX_CACHE = b'\x03'
|
||||
|
||||
|
||||
EngineCoreRequestUnion = Union[EngineCoreRequest, EngineCoreProfile,
|
||||
EngineCoreResetPrefixCache, List[str]]
|
||||
|
||||
Reference in New Issue
Block a user