[Frontend] Offload blocking preprocessing & postprocessing ops to thread pool for pooling entrypoints. (#39763)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
wang.yuqi
2026-04-14 08:29:25 +00:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent 0008729abf
commit c0ecaed950
10 changed files with 68 additions and 83 deletions
+1 -1
View File
@@ -69,9 +69,9 @@ class PoolingServeContext(Generic[PoolingRequestT]):
raw_request: Request | None = None
model_name: str
request_id: str
pooling_params: PoolingParams | list[PoolingParams]
created_time: int = field(default_factory=lambda: int(time.time()))
lora_request: LoRARequest | None = None
pooling_params: PoolingParams | list[PoolingParams] | None = None
engine_inputs: Sequence[EngineInput] | None = None
prompt_request_ids: list[str] | None = None
intermediates: Any | None = None