[Bugfix] Thread kv_transfer_params into engine for /inference/v1/generate (disagg) (#42644)
Signed-off-by: hallerite <git@hallerite.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
2ab187430b
commit
c71f6f8a81
@@ -203,6 +203,16 @@ class ServingTokens(GenerateBaseServing):
|
|||||||
# Schedule the request and get the result generator.
|
# Schedule the request and get the result generator.
|
||||||
result_generator: AsyncGenerator[RequestOutput, None] | None = None
|
result_generator: AsyncGenerator[RequestOutput, None] | None = None
|
||||||
|
|
||||||
|
# Pass disaggregated-serving parameters through to the engine.
|
||||||
|
if request.kv_transfer_params is not None:
|
||||||
|
extra = sampling_params.extra_args or {}
|
||||||
|
extra["kv_transfer_params"] = request.kv_transfer_params
|
||||||
|
sampling_params.extra_args = extra
|
||||||
|
if request.ec_transfer_params is not None:
|
||||||
|
extra = sampling_params.extra_args or {}
|
||||||
|
extra["ec_transfer_params"] = request.ec_transfer_params
|
||||||
|
sampling_params.extra_args = extra
|
||||||
|
|
||||||
# Apply server-side ``max_tokens`` defaulting when the client did
|
# Apply server-side ``max_tokens`` defaulting when the client did
|
||||||
# not set it, matching the OpenAI-compat endpoints. ``SamplingParams``
|
# not set it, matching the OpenAI-compat endpoints. ``SamplingParams``
|
||||||
# defaults ``max_tokens`` to 16, which would otherwise silently cap
|
# defaults ``max_tokens`` to 16, which would otherwise silently cap
|
||||||
|
|||||||
Reference in New Issue
Block a user