[Bugfix][Frontend] Emit kv/ec transfer params at the top level of render output (#61241)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-10-15 12:34:15 +00:00
committed by GitHub
parent e8e1e96196
commit c6b772cf6b
2 changed files with 79 additions and 3 deletions
+29 -3
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from fastapi import Request
@@ -33,6 +33,7 @@ from vllm.renderers.inputs.preprocess import (
extract_prompt_len,
)
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
logger = init_logger(__name__)
@@ -40,6 +41,24 @@ if TYPE_CHECKING:
from vllm.entrypoints.mcp.tool_server import ToolServer
def _hoist_transfer_params(
params: SamplingParams,
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
"""Move KV/EC transfer params out of `extra_args`.
Render emits them only as top-level `GenerateRequest` fields. Generate
copies them back into `extra_args` for the engine.
"""
if not params.extra_args:
return None, None
# to_sampling_params aliases request.vllm_xargs, so copy before popping.
extra = dict(params.extra_args)
kv = extra.pop("kv_transfer_params", None)
ec = extra.pop("ec_transfer_params", None)
params.extra_args = extra or None
return kv, ec
class ServingRender(BaseServing):
def __init__(
self,
@@ -126,6 +145,7 @@ class ServingRender(BaseServing):
truncate_prompt_tokens=request.truncate_prompt_tokens,
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
# Resolve here what OpenAIServingChat passes to the engine:
# `_grammar_from_parser` (set by `adjust_request`) is not serialized.
@@ -152,6 +172,8 @@ class ServingRender(BaseServing):
engine_input, include_mm_kwargs=request.return_mm_kwargs
),
sampling_params=params,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
model=request.model,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs=reasoning_parser_kwargs,
@@ -226,6 +248,7 @@ class ServingRender(BaseServing):
params = request.to_sampling_params(
max_tokens, self.default_sampling_params
)
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
request_id = base_id if len(result) == 1 else f"{base_id}-{i}"
@@ -237,6 +260,8 @@ class ServingRender(BaseServing):
engine_input, include_mm_kwargs=request.return_mm_kwargs
),
sampling_params=params,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
model=request.model,
stream=bool(request.stream),
stream_options=(request.stream_options if request.stream else None),
@@ -291,6 +316,7 @@ class ServingRender(BaseServing):
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
return GenerateRequest(
request_id=request.request_id,
@@ -304,8 +330,8 @@ class ServingRender(BaseServing):
stream=bool(request.stream),
cache_salt=request.cache_salt,
priority=request.priority,
kv_transfer_params=request.kv_transfer_params,
ec_transfer_params=request.ec_transfer_params,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
token_offsets=engine_input.get("prompt_token_offsets"),
)