[EC Connector] Add EC Transfer Params (#42433)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -137,6 +137,12 @@ class AnthropicMessagesRequest(BaseModel):
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ECTransfer parameters used for encoder-cache disaggregated serving."
|
||||
),
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
@@ -218,6 +224,9 @@ class AnthropicMessagesResponse(BaseModel):
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="ECTransfer parameters."
|
||||
)
|
||||
|
||||
def model_post_init(self, __context):
|
||||
if not self.id:
|
||||
|
||||
@@ -491,6 +491,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
top_p=anthropic_request.top_p,
|
||||
top_k=anthropic_request.top_k,
|
||||
kv_transfer_params=anthropic_request.kv_transfer_params,
|
||||
ec_transfer_params=anthropic_request.ec_transfer_params,
|
||||
chat_template_kwargs=anthropic_request.chat_template_kwargs,
|
||||
)
|
||||
|
||||
@@ -630,6 +631,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
generator.usage,
|
||||
),
|
||||
kv_transfer_params=generator.kv_transfer_params,
|
||||
ec_transfer_params=generator.ec_transfer_params,
|
||||
)
|
||||
choice = generator.choices[0]
|
||||
if choice.finish_reason == "stop":
|
||||
|
||||
@@ -134,6 +134,9 @@ class ChatCompletionResponse(OpenAIBaseModel):
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="ECTransfer parameters."
|
||||
)
|
||||
metrics: PerRequestTimingMetrics | None = None
|
||||
|
||||
|
||||
@@ -439,6 +442,13 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ECTransfer parameters used for encoder-cache disaggregated serving."
|
||||
),
|
||||
)
|
||||
|
||||
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
@@ -665,6 +675,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
if self.kv_transfer_params:
|
||||
# Pass in kv_transfer_params via extra_args
|
||||
extra_args["kv_transfer_params"] = self.kv_transfer_params
|
||||
if self.ec_transfer_params:
|
||||
# Pass in ec_transfer_params via extra_args
|
||||
extra_args["ec_transfer_params"] = self.ec_transfer_params
|
||||
return SamplingParams.from_optional(
|
||||
n=self.n,
|
||||
presence_penalty=self.presence_penalty,
|
||||
|
||||
@@ -1057,6 +1057,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
),
|
||||
prompt_text=prompt_text,
|
||||
kv_transfer_params=final_res.kv_transfer_params,
|
||||
ec_transfer_params=final_res.ec_transfer_params,
|
||||
metrics=per_request_metrics,
|
||||
)
|
||||
|
||||
|
||||
@@ -189,6 +189,13 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ECTransfer parameters used for encoder-cache disaggregated serving."
|
||||
),
|
||||
)
|
||||
|
||||
vllm_xargs: dict[str, str | int | float] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
@@ -346,6 +353,9 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
if self.kv_transfer_params:
|
||||
# Pass in kv_transfer_params via extra_args
|
||||
extra_args["kv_transfer_params"] = self.kv_transfer_params
|
||||
if self.ec_transfer_params:
|
||||
# Pass in ec_transfer_params via extra_args
|
||||
extra_args["ec_transfer_params"] = self.ec_transfer_params
|
||||
return SamplingParams.from_optional(
|
||||
n=self.n,
|
||||
presence_penalty=self.presence_penalty,
|
||||
@@ -595,6 +605,9 @@ class CompletionResponse(OpenAIBaseModel):
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="ECTransfer parameters."
|
||||
)
|
||||
metrics: PerRequestTimingMetrics | None = None
|
||||
|
||||
|
||||
|
||||
@@ -510,6 +510,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
num_prompt_tokens = 0
|
||||
num_generated_tokens = 0
|
||||
kv_transfer_params = None
|
||||
ec_transfer_params = None
|
||||
last_final_res = None
|
||||
for final_res in final_res_batch:
|
||||
last_final_res = final_res
|
||||
@@ -632,6 +633,8 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
|
||||
if final_res_batch:
|
||||
kv_transfer_params = final_res_batch[0].kv_transfer_params
|
||||
ec_transfer_params = final_res_batch[0].ec_transfer_params
|
||||
|
||||
return CompletionResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
@@ -640,6 +643,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
usage=usage,
|
||||
system_fingerprint=self.system_fingerprint,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
metrics=per_request_metrics,
|
||||
)
|
||||
|
||||
|
||||
@@ -200,6 +200,7 @@ class SimpleContext(ConversationContext):
|
||||
|
||||
self.input_messages: list[ResponseRawMessageAndToken] = []
|
||||
self.kv_transfer_params: dict[str, Any] | None = None
|
||||
self.ec_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
def append_output(self, output) -> None:
|
||||
self.last_output = output
|
||||
@@ -210,6 +211,8 @@ class SimpleContext(ConversationContext):
|
||||
self.num_output_tokens += len(output.outputs[0].token_ids or [])
|
||||
if output.kv_transfer_params is not None:
|
||||
self.kv_transfer_params = output.kv_transfer_params
|
||||
if output.ec_transfer_params is not None:
|
||||
self.ec_transfer_params = output.ec_transfer_params
|
||||
|
||||
# Accumulate text, token_ids, and logprobs for streaming mode
|
||||
delta_output = output.outputs[0]
|
||||
@@ -328,6 +331,7 @@ class ParsableContext(ConversationContext):
|
||||
self.output_messages: list[ResponseRawMessageAndToken] = []
|
||||
self._accumulated_token_ids: list[int] = []
|
||||
self.kv_transfer_params: dict[str, Any] | None = None
|
||||
self.ec_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
def append_output(self, output: RequestOutput) -> None:
|
||||
self.num_prompt_tokens = len(output.prompt_token_ids or [])
|
||||
@@ -336,6 +340,9 @@ class ParsableContext(ConversationContext):
|
||||
if output.kv_transfer_params is not None:
|
||||
self.kv_transfer_params = output.kv_transfer_params
|
||||
|
||||
if output.ec_transfer_params is not None:
|
||||
self.ec_transfer_params = output.ec_transfer_params
|
||||
|
||||
completion = output.outputs[0]
|
||||
self.finish_reason = completion.finish_reason
|
||||
|
||||
@@ -630,6 +637,7 @@ class HarmonyContext(ConversationContext):
|
||||
self.is_first_turn = True
|
||||
self.first_tok_of_message = True
|
||||
self.kv_transfer_params: dict[str, Any] | None = None
|
||||
self.ec_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
def append_output(self, output: RequestOutput) -> None:
|
||||
if self.first_tok_of_message:
|
||||
@@ -645,6 +653,8 @@ class HarmonyContext(ConversationContext):
|
||||
self._update_decode_token_usage(output)
|
||||
if output.kv_transfer_params is not None:
|
||||
self.kv_transfer_params = output.kv_transfer_params
|
||||
if output.ec_transfer_params is not None:
|
||||
self.ec_transfer_params = output.ec_transfer_params
|
||||
|
||||
if output.finished:
|
||||
self.finish_reason = output.outputs[0].finish_reason
|
||||
|
||||
@@ -285,6 +285,12 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ECTransfer parameters used for encoder-cache disaggregated serving."
|
||||
),
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
@@ -409,6 +415,8 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
|
||||
if self.kv_transfer_params:
|
||||
extra_args["kv_transfer_params"] = self.kv_transfer_params
|
||||
if self.ec_transfer_params:
|
||||
extra_args["ec_transfer_params"] = self.ec_transfer_params
|
||||
|
||||
return SamplingParams.from_optional(
|
||||
temperature=temperature,
|
||||
@@ -675,6 +683,9 @@ class ResponsesResponse(OpenAIBaseModel):
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="ECTransfer parameters."
|
||||
)
|
||||
|
||||
# --8<-- [start:responses-response-extra-params]
|
||||
# These are populated when enable_response_messages is set to True
|
||||
@@ -720,6 +731,7 @@ class ResponsesResponse(OpenAIBaseModel):
|
||||
input_messages: ResponseInputOutputMessage | None = None,
|
||||
output_messages: ResponseInputOutputMessage | None = None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
) -> "ResponsesResponse":
|
||||
incomplete_details: IncompleteDetails | None = None
|
||||
if status == "incomplete":
|
||||
@@ -758,6 +770,7 @@ class ResponsesResponse(OpenAIBaseModel):
|
||||
user=request.user,
|
||||
usage=usage,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -932,6 +932,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
status=status,
|
||||
usage=usage,
|
||||
kv_transfer_params=context.kv_transfer_params,
|
||||
ec_transfer_params=context.ec_transfer_params,
|
||||
)
|
||||
|
||||
if request.store:
|
||||
|
||||
@@ -132,6 +132,12 @@ class GenerateRequest(BaseModel):
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ECTransfer parameters used for encoder-cache disaggregated serving."
|
||||
),
|
||||
)
|
||||
|
||||
# Tracks which keys the caller explicitly set inside ``sampling_params``
|
||||
# when the request was parsed from a JSON body. Lets the server tell
|
||||
@@ -238,6 +244,12 @@ class GenerateResponse(BaseModel):
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ECTransfer parameters used for encoder-cache disaggregated serving."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
####### Derender (postprocessing) #######
|
||||
|
||||
@@ -330,6 +330,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
usage=usage,
|
||||
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
|
||||
kv_transfer_params=final_res.kv_transfer_params,
|
||||
ec_transfer_params=final_res.ec_transfer_params,
|
||||
)
|
||||
|
||||
# Log complete response if output logging is enabled
|
||||
|
||||
@@ -207,6 +207,8 @@ if TYPE_CHECKING:
|
||||
VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: bool = False
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
|
||||
VLLM_EC_SIDE_CHANNEL_HOST: str = "localhost"
|
||||
VLLM_EC_SIDE_CHANNEL_PORT: int = 5601
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
|
||||
VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
|
||||
VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1
|
||||
@@ -1565,6 +1567,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int(
|
||||
os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600")
|
||||
),
|
||||
# IP address used for the EC connector's ZMQ side channel
|
||||
# (producer ROUTER bind, consumer DEALER dial).
|
||||
"VLLM_EC_SIDE_CHANNEL_HOST": lambda: os.getenv(
|
||||
"VLLM_EC_SIDE_CHANNEL_HOST", "localhost"
|
||||
),
|
||||
# Port for the EC connector's ZMQ side channel; advertised to peers
|
||||
# via `ec_transfer_params.peer_port` on the producer's response.
|
||||
"VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(
|
||||
os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")
|
||||
),
|
||||
# Port used for Mooncake handshake between remote agents.
|
||||
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
|
||||
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
|
||||
|
||||
@@ -104,6 +104,7 @@ class RequestOutput:
|
||||
None if decoder-only.
|
||||
num_cached_tokens: The number of tokens with prefix cache hit.
|
||||
kv_transfer_params: The params for remote K/V transfer.
|
||||
ec_transfer_params: The params for remote encoder-cache transfer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -121,6 +122,7 @@ class RequestOutput:
|
||||
num_cached_tokens: int | None = None,
|
||||
*,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
# Forward compatibility, code that uses args added in new release can
|
||||
# still run with older versions of vLLM without breaking.
|
||||
**kwargs: Any,
|
||||
@@ -141,12 +143,14 @@ class RequestOutput:
|
||||
self.encoder_prompt_token_ids = encoder_prompt_token_ids
|
||||
self.num_cached_tokens = num_cached_tokens
|
||||
self.kv_transfer_params = kv_transfer_params
|
||||
self.ec_transfer_params = ec_transfer_params
|
||||
|
||||
def add(self, next_output: "RequestOutput", aggregate: bool) -> None:
|
||||
"""Merge subsequent RequestOutput into this one"""
|
||||
|
||||
self.finished |= next_output.finished
|
||||
self.kv_transfer_params = next_output.kv_transfer_params
|
||||
self.ec_transfer_params = next_output.ec_transfer_params
|
||||
|
||||
for next_completion in next_output.outputs:
|
||||
for i, completion in enumerate(self.outputs):
|
||||
|
||||
@@ -9,6 +9,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorBase
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorBase_V1
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.engine import EngineCoreOutputs
|
||||
@@ -248,3 +249,6 @@ class SchedulerInterface(ABC):
|
||||
|
||||
def get_kv_connector(self) -> "KVConnectorBase_V1 | None":
|
||||
return None
|
||||
|
||||
def get_ec_connector(self) -> "ECConnectorBase | None":
|
||||
return None
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import Any
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorMetadata,
|
||||
ECConnectorRole,
|
||||
)
|
||||
@@ -1675,6 +1676,7 @@ class Scheduler(SchedulerInterface):
|
||||
new_token_ids = generated_token_ids
|
||||
pooler_output = pooler_outputs[req_index] if pooler_outputs else None
|
||||
kv_transfer_params = None
|
||||
ec_transfer_params = None
|
||||
status_before_stop = request.status
|
||||
num_output_tokens_before = len(request._output_token_ids)
|
||||
|
||||
@@ -1761,7 +1763,7 @@ class Scheduler(SchedulerInterface):
|
||||
finish_reason = request.get_finished_reason()
|
||||
finished = self._handle_stopped_request(request)
|
||||
if finished:
|
||||
kv_transfer_params = self._free_request(request)
|
||||
kv_transfer_params, ec_transfer_params = self._free_request(request)
|
||||
|
||||
if status_before_stop == RequestStatus.RUNNING:
|
||||
stopped_running_reqs.add(request)
|
||||
@@ -1785,6 +1787,7 @@ class Scheduler(SchedulerInterface):
|
||||
new_token_ids
|
||||
or pooler_output is not None
|
||||
or kv_transfer_params
|
||||
or ec_transfer_params
|
||||
or stopped
|
||||
):
|
||||
# Add EngineCoreOutput for this Request.
|
||||
@@ -1800,6 +1803,7 @@ class Scheduler(SchedulerInterface):
|
||||
events=request.take_events(),
|
||||
prefill_stats=request.take_prefill_stats(),
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
trace_headers=request.trace_headers,
|
||||
routed_experts=routed_experts,
|
||||
num_nans_in_logits=request.num_nans_in_logits,
|
||||
@@ -2151,11 +2155,21 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
def _free_request(
|
||||
self, request: Request, delay_free_blocks: bool = False
|
||||
) -> dict[str, Any] | None:
|
||||
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
||||
assert request.is_finished()
|
||||
|
||||
self._inflight_prefills.discard(request)
|
||||
connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request)
|
||||
|
||||
# EC Connector: mirror the KV hook. The contract requires firing
|
||||
# before the encoder cache is freed so the connector can inspect
|
||||
# per-request state (e.g. which mm_hashes it recorded during
|
||||
# save_caches()) and emit ec_transfer_params for the response body.
|
||||
ec_xfer_params: dict[str, Any] | None = None
|
||||
if self.ec_connector is not None:
|
||||
ec_delay_free, ec_xfer_params = self.ec_connector.request_finished(request)
|
||||
connector_delay_free_blocks |= ec_delay_free
|
||||
|
||||
self.encoder_cache_manager.free(request)
|
||||
request_id = request.request_id
|
||||
self.finished_req_ids.add(request_id)
|
||||
@@ -2166,7 +2180,7 @@ class Scheduler(SchedulerInterface):
|
||||
if not delay_free_blocks:
|
||||
self._free_blocks(request)
|
||||
|
||||
return kv_xfer_params
|
||||
return kv_xfer_params, ec_xfer_params
|
||||
|
||||
def _free_blocks(self, request: Request):
|
||||
assert request.is_finished()
|
||||
@@ -2414,6 +2428,9 @@ class Scheduler(SchedulerInterface):
|
||||
def get_kv_connector(self) -> KVConnectorBase_V1 | None:
|
||||
return self.connector
|
||||
|
||||
def get_ec_connector(self) -> ECConnectorBase | None:
|
||||
return self.ec_connector
|
||||
|
||||
def _connector_finished(
|
||||
self, request: Request
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
|
||||
@@ -190,6 +190,7 @@ class EngineCoreOutput(
|
||||
stop_reason: int | str | None = None
|
||||
events: list[EngineCoreEvent] | None = None
|
||||
kv_transfer_params: dict[str, Any] | None = None
|
||||
ec_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
trace_headers: Mapping[str, str] | None = None
|
||||
|
||||
|
||||
@@ -400,6 +400,15 @@ class EngineCore:
|
||||
"Disabling KVTransfer for this request."
|
||||
)
|
||||
|
||||
if (
|
||||
request.ec_transfer_params is not None
|
||||
and self.scheduler.get_ec_connector() is None
|
||||
):
|
||||
logger.warning(
|
||||
"Got ec_transfer_params, but no ECConnector found. "
|
||||
"Disabling ECTransfer for this request."
|
||||
)
|
||||
|
||||
self.scheduler.add_request(request)
|
||||
if request.abort_immediately:
|
||||
# Immediately abort so the connector's request_finished hook runs
|
||||
|
||||
@@ -276,6 +276,7 @@ class RequestState:
|
||||
finish_reason: FinishReason | None,
|
||||
stop_reason: int | str | None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput | None:
|
||||
finished = finish_reason is not None
|
||||
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
|
||||
@@ -327,7 +328,11 @@ class RequestState:
|
||||
external_req_id = self.parent_req.external_req_id
|
||||
|
||||
return self._new_request_output(
|
||||
external_req_id, outputs, finished, kv_transfer_params
|
||||
external_req_id,
|
||||
outputs,
|
||||
finished,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
)
|
||||
|
||||
def _new_request_output(
|
||||
@@ -336,6 +341,7 @@ class RequestState:
|
||||
outputs: list[CompletionOutput] | list[PoolingOutput],
|
||||
finished: bool,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput:
|
||||
# If prompt embeds were used, put placeholder prompt token ids
|
||||
prompt_token_ids = self.prompt_token_ids
|
||||
@@ -369,6 +375,7 @@ class RequestState:
|
||||
outputs=cast(list[CompletionOutput], outputs),
|
||||
finished=finished,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
num_cached_tokens=self.num_cached_tokens,
|
||||
metrics=self.stats,
|
||||
)
|
||||
@@ -497,6 +504,7 @@ class OutputProcessor:
|
||||
finish_reason=FinishReason.ABORT,
|
||||
stop_reason=None,
|
||||
kv_transfer_params=None,
|
||||
ec_transfer_params=None,
|
||||
)
|
||||
):
|
||||
req_state.queue.put(request_output)
|
||||
@@ -620,6 +628,7 @@ class OutputProcessor:
|
||||
finish_reason = engine_core_output.finish_reason
|
||||
stop_reason = engine_core_output.stop_reason
|
||||
kv_transfer_params = engine_core_output.kv_transfer_params
|
||||
ec_transfer_params = engine_core_output.ec_transfer_params
|
||||
if engine_core_output.routed_experts is not None:
|
||||
req_state.routed_experts_chunks.append(
|
||||
engine_core_output.routed_experts
|
||||
@@ -654,6 +663,7 @@ class OutputProcessor:
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
):
|
||||
if req_state.streaming_input:
|
||||
request_output.finished = False
|
||||
|
||||
@@ -100,6 +100,8 @@ class Request:
|
||||
|
||||
# P/D: Connector-specific KV transfer parameters.
|
||||
self.kv_transfer_params: dict[str, Any] | None = None
|
||||
# E/P/D: Connector-specific encoder-cache transfer parameters.
|
||||
self.ec_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
if pooling_params is not None:
|
||||
# Pooling models.
|
||||
@@ -115,6 +117,9 @@ class Request:
|
||||
self.kv_transfer_params = sampling_params.extra_args.get(
|
||||
"kv_transfer_params"
|
||||
)
|
||||
self.ec_transfer_params = sampling_params.extra_args.get(
|
||||
"ec_transfer_params"
|
||||
)
|
||||
self.kv_cache_report_mode = sampling_params.extra_args.get(
|
||||
"kv_cache_report_mode", "incremental"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user