[docs][misc] IOProcessor plugins fixes (#24046)

Signed-off-by: Christian Pinto <christian.pinto@ibm.com>
This commit is contained in:
Christian Pinto
2025-09-01 09:17:41 -07:00
committed by GitHub
parent 39a22dcaac
commit cf91a89dd2
6 changed files with 14 additions and 23 deletions
+2 -1
View File
@@ -1424,9 +1424,10 @@ class IOProcessorRequest(OpenAIBaseModel, Generic[T]):
When using plugins IOProcessor plugins, the actual input is processed
by the plugin itself. Hence, we use a generic type for the request data
"""
softmax: bool = True
def to_pooling_params(self):
return PoolingParams(task="encode")
return PoolingParams(task="encode", softmax=self.softmax)
class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
+7 -2
View File
@@ -49,7 +49,12 @@ class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):
request_id: Optional[str] = None,
**kwargs,
) -> IOProcessorOutput:
collected_output = [item async for i, item in model_output]
# We cannot guarantee outputs are returned in the same order they were
# fed to vLLM.
# Let's sort them by id before post_processing
sorted_output = sorted([(i, item) async for i, item in model_output],
key=lambda output: output[0])
collected_output = [output[1] for output in sorted_output]
return self.post_process(collected_output, request_id, **kwargs)
@abstractmethod
@@ -59,4 +64,4 @@ class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):
@abstractmethod
def output_to_response(
self, plugin_output: IOProcessorOutput) -> IOProcessorResponse:
raise NotImplementedError
raise NotImplementedError