[Misc] Support more collective_rpc return types (#21845)

Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Nick Hill
2025-07-30 10:20:20 -07:00
committed by GitHub
parent 8f0d516715
commit 56bd537dde
5 changed files with 121 additions and 6 deletions
+8 -1
View File
@@ -123,6 +123,13 @@ class EngineCoreOutput(
return self.finish_reason is not None
class UtilityResult:
"""Wrapper for special handling when serializing/deserializing."""
def __init__(self, r: Any = None):
self.result = r
class UtilityOutput(
msgspec.Struct,
array_like=True, # type: ignore[call-arg]
@@ -132,7 +139,7 @@ class UtilityOutput(
# Non-None implies the call failed, result should be None.
failure_message: Optional[str] = None
result: Any = None
result: Optional[UtilityResult] = None
class EngineCoreOutputs(
+3 -3
View File
@@ -36,7 +36,7 @@ from vllm.v1.core.sched.scheduler import Scheduler as V1Scheduler
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
EngineCoreRequestType,
ReconfigureDistributedRequest, ReconfigureRankType,
UtilityOutput)
UtilityOutput, UtilityResult)
from vllm.v1.engine.mm_input_cache import MirroredProcessingCache
from vllm.v1.engine.utils import EngineHandshakeMetadata, EngineZmqAddresses
from vllm.v1.executor.abstract import Executor
@@ -715,8 +715,8 @@ class EngineCoreProc(EngineCore):
output = UtilityOutput(call_id)
try:
method = getattr(self, method_name)
output.result = method(
*self._convert_msgspec_args(method, args))
result = method(*self._convert_msgspec_args(method, args))
output.result = UtilityResult(result)
except BaseException as e:
logger.exception("Invocation of %s method failed", method_name)
output.failure_message = (f"Call to {method_name} method"
+2 -1
View File
@@ -552,7 +552,8 @@ def _process_utility_output(output: UtilityOutput,
if output.failure_message is not None:
future.set_exception(Exception(output.failure_message))
else:
future.set_result(output.result)
assert output.result is not None
future.set_result(output.result.result)
class SyncMPClient(MPClient):
+44
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import importlib
import pickle
from collections.abc import Sequence
from inspect import isclass
@@ -9,6 +10,7 @@ from types import FunctionType
from typing import Any, Optional, Union
import cloudpickle
import msgspec
import numpy as np
import torch
import zmq
@@ -22,6 +24,7 @@ from vllm.multimodal.inputs import (BaseMultiModalField,
MultiModalFlatField, MultiModalKwargs,
MultiModalKwargsItem,
MultiModalSharedField, NestedTensors)
from vllm.v1.engine import UtilityResult
logger = init_logger(__name__)
@@ -46,6 +49,10 @@ def _log_insecure_serialization_warning():
"VLLM_ALLOW_INSECURE_SERIALIZATION=1")
def _typestr(t: type):
return t.__module__, t.__qualname__
class MsgpackEncoder:
"""Encoder with custom torch tensor and numpy array serialization.
@@ -122,6 +129,18 @@ class MsgpackEncoder:
for itemlist in mm._items_by_modality.values()
for item in itemlist]
if isinstance(obj, UtilityResult):
result = obj.result
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION or result is None:
return None, result
# Since utility results are not strongly typed, we also encode
# the type (or a list of types in the case it's a list) to
# help with correct msgspec deserialization.
cls = result.__class__
return _typestr(cls) if cls is not list else [
_typestr(type(v)) for v in result
], result
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
raise TypeError(f"Object of type {type(obj)} is not serializable"
"Set VLLM_ALLOW_INSECURE_SERIALIZATION=1 to allow "
@@ -237,8 +256,33 @@ class MsgpackDecoder:
k: self._decode_nested_tensors(v)
for k, v in obj.items()
})
if t is UtilityResult:
return self._decode_utility_result(obj)
return obj
def _decode_utility_result(self, obj: Any) -> UtilityResult:
result_type, result = obj
if result_type is not None:
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
raise TypeError("VLLM_ALLOW_INSECURE_SERIALIZATION must "
"be set to use custom utility result types")
assert isinstance(result_type, list)
if len(result_type) == 2 and isinstance(result_type[0], str):
result = self._convert_result(result_type, result)
else:
assert isinstance(result, list)
result = [
self._convert_result(rt, r)
for rt, r in zip(result_type, result)
]
return UtilityResult(result)
def _convert_result(self, result_type: Sequence[str], result: Any):
mod_name, name = result_type
mod = importlib.import_module(mod_name)
result_type = getattr(mod, name)
return msgspec.convert(result, result_type, dec_hook=self.dec_hook)
def _decode_ndarray(self, arr: Any) -> np.ndarray:
dtype, shape, data = arr
# zero-copy decode. We assume the ndarray will not be kept around,