[Bugfix][Frontend] Emit kv/ec transfer params at the top level of render output (#61241)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_len,
|
||||
)
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -40,6 +41,24 @@ if TYPE_CHECKING:
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
|
||||
|
||||
def _hoist_transfer_params(
|
||||
params: SamplingParams,
|
||||
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
||||
"""Move KV/EC transfer params out of `extra_args`.
|
||||
|
||||
Render emits them only as top-level `GenerateRequest` fields. Generate
|
||||
copies them back into `extra_args` for the engine.
|
||||
"""
|
||||
if not params.extra_args:
|
||||
return None, None
|
||||
# to_sampling_params aliases request.vllm_xargs, so copy before popping.
|
||||
extra = dict(params.extra_args)
|
||||
kv = extra.pop("kv_transfer_params", None)
|
||||
ec = extra.pop("ec_transfer_params", None)
|
||||
params.extra_args = extra or None
|
||||
return kv, ec
|
||||
|
||||
|
||||
class ServingRender(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -126,6 +145,7 @@ class ServingRender(BaseServing):
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
|
||||
|
||||
# Resolve here what OpenAIServingChat passes to the engine:
|
||||
# `_grammar_from_parser` (set by `adjust_request`) is not serialized.
|
||||
@@ -152,6 +172,8 @@ class ServingRender(BaseServing):
|
||||
engine_input, include_mm_kwargs=request.return_mm_kwargs
|
||||
),
|
||||
sampling_params=params,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
model=request.model,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
@@ -226,6 +248,7 @@ class ServingRender(BaseServing):
|
||||
params = request.to_sampling_params(
|
||||
max_tokens, self.default_sampling_params
|
||||
)
|
||||
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
|
||||
|
||||
request_id = base_id if len(result) == 1 else f"{base_id}-{i}"
|
||||
|
||||
@@ -237,6 +260,8 @@ class ServingRender(BaseServing):
|
||||
engine_input, include_mm_kwargs=request.return_mm_kwargs
|
||||
),
|
||||
sampling_params=params,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
model=request.model,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
@@ -291,6 +316,7 @@ class ServingRender(BaseServing):
|
||||
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request.request_id,
|
||||
@@ -304,8 +330,8 @@ class ServingRender(BaseServing):
|
||||
stream=bool(request.stream),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
kv_transfer_params=request.kv_transfer_params,
|
||||
ec_transfer_params=request.ec_transfer_params,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user