[EC Connector] Add EC Transfer Params (#42433)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
omerpaz95
2026-07-12 14:35:33 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 370b678a02
commit 8df14cfc8c
55 changed files with 416 additions and 12 deletions
+9
View File
@@ -137,6 +137,12 @@ class AnthropicMessagesRequest(BaseModel):
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None,
description=(
"ECTransfer parameters used for encoder-cache disaggregated serving."
),
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
@@ -218,6 +224,9 @@ class AnthropicMessagesResponse(BaseModel):
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None, description="ECTransfer parameters."
)
def model_post_init(self, __context):
if not self.id:
+2
View File
@@ -491,6 +491,7 @@ class AnthropicServingMessages(OpenAIServingChat):
top_p=anthropic_request.top_p,
top_k=anthropic_request.top_k,
kv_transfer_params=anthropic_request.kv_transfer_params,
ec_transfer_params=anthropic_request.ec_transfer_params,
chat_template_kwargs=anthropic_request.chat_template_kwargs,
)
@@ -630,6 +631,7 @@ class AnthropicServingMessages(OpenAIServingChat):
generator.usage,
),
kv_transfer_params=generator.kv_transfer_params,
ec_transfer_params=generator.ec_transfer_params,
)
choice = generator.choices[0]
if choice.finish_reason == "stop":
@@ -134,6 +134,9 @@ class ChatCompletionResponse(OpenAIBaseModel):
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None, description="ECTransfer parameters."
)
metrics: PerRequestTimingMetrics | None = None
@@ -439,6 +442,13 @@ class ChatCompletionRequest(OpenAIBaseModel):
description="KVTransfer parameters used for disaggregated serving.",
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None,
description=(
"ECTransfer parameters used for encoder-cache disaggregated serving."
),
)
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
default=None,
description=(
@@ -665,6 +675,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
if self.kv_transfer_params:
# Pass in kv_transfer_params via extra_args
extra_args["kv_transfer_params"] = self.kv_transfer_params
if self.ec_transfer_params:
# Pass in ec_transfer_params via extra_args
extra_args["ec_transfer_params"] = self.ec_transfer_params
return SamplingParams.from_optional(
n=self.n,
presence_penalty=self.presence_penalty,
@@ -1057,6 +1057,7 @@ class OpenAIServingChat(GenerateBaseServing):
),
prompt_text=prompt_text,
kv_transfer_params=final_res.kv_transfer_params,
ec_transfer_params=final_res.ec_transfer_params,
metrics=per_request_metrics,
)
@@ -189,6 +189,13 @@ class CompletionRequest(OpenAIBaseModel):
description="KVTransfer parameters used for disaggregated serving.",
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None,
description=(
"ECTransfer parameters used for encoder-cache disaggregated serving."
),
)
vllm_xargs: dict[str, str | int | float] | None = Field(
default=None,
description=(
@@ -346,6 +353,9 @@ class CompletionRequest(OpenAIBaseModel):
if self.kv_transfer_params:
# Pass in kv_transfer_params via extra_args
extra_args["kv_transfer_params"] = self.kv_transfer_params
if self.ec_transfer_params:
# Pass in ec_transfer_params via extra_args
extra_args["ec_transfer_params"] = self.ec_transfer_params
return SamplingParams.from_optional(
n=self.n,
presence_penalty=self.presence_penalty,
@@ -595,6 +605,9 @@ class CompletionResponse(OpenAIBaseModel):
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None, description="ECTransfer parameters."
)
metrics: PerRequestTimingMetrics | None = None
@@ -510,6 +510,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
num_prompt_tokens = 0
num_generated_tokens = 0
kv_transfer_params = None
ec_transfer_params = None
last_final_res = None
for final_res in final_res_batch:
last_final_res = final_res
@@ -632,6 +633,8 @@ class OpenAIServingCompletion(GenerateBaseServing):
if final_res_batch:
kv_transfer_params = final_res_batch[0].kv_transfer_params
ec_transfer_params = final_res_batch[0].ec_transfer_params
return CompletionResponse(
id=request_id,
created=created_time,
@@ -640,6 +643,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
usage=usage,
system_fingerprint=self.system_fingerprint,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
metrics=per_request_metrics,
)
@@ -200,6 +200,7 @@ class SimpleContext(ConversationContext):
self.input_messages: list[ResponseRawMessageAndToken] = []
self.kv_transfer_params: dict[str, Any] | None = None
self.ec_transfer_params: dict[str, Any] | None = None
def append_output(self, output) -> None:
self.last_output = output
@@ -210,6 +211,8 @@ class SimpleContext(ConversationContext):
self.num_output_tokens += len(output.outputs[0].token_ids or [])
if output.kv_transfer_params is not None:
self.kv_transfer_params = output.kv_transfer_params
if output.ec_transfer_params is not None:
self.ec_transfer_params = output.ec_transfer_params
# Accumulate text, token_ids, and logprobs for streaming mode
delta_output = output.outputs[0]
@@ -328,6 +331,7 @@ class ParsableContext(ConversationContext):
self.output_messages: list[ResponseRawMessageAndToken] = []
self._accumulated_token_ids: list[int] = []
self.kv_transfer_params: dict[str, Any] | None = None
self.ec_transfer_params: dict[str, Any] | None = None
def append_output(self, output: RequestOutput) -> None:
self.num_prompt_tokens = len(output.prompt_token_ids or [])
@@ -336,6 +340,9 @@ class ParsableContext(ConversationContext):
if output.kv_transfer_params is not None:
self.kv_transfer_params = output.kv_transfer_params
if output.ec_transfer_params is not None:
self.ec_transfer_params = output.ec_transfer_params
completion = output.outputs[0]
self.finish_reason = completion.finish_reason
@@ -630,6 +637,7 @@ class HarmonyContext(ConversationContext):
self.is_first_turn = True
self.first_tok_of_message = True
self.kv_transfer_params: dict[str, Any] | None = None
self.ec_transfer_params: dict[str, Any] | None = None
def append_output(self, output: RequestOutput) -> None:
if self.first_tok_of_message:
@@ -645,6 +653,8 @@ class HarmonyContext(ConversationContext):
self._update_decode_token_usage(output)
if output.kv_transfer_params is not None:
self.kv_transfer_params = output.kv_transfer_params
if output.ec_transfer_params is not None:
self.ec_transfer_params = output.ec_transfer_params
if output.finished:
self.finish_reason = output.outputs[0].finish_reason
@@ -285,6 +285,12 @@ class ResponsesRequest(OpenAIBaseModel):
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None,
description=(
"ECTransfer parameters used for encoder-cache disaggregated serving."
),
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
@@ -409,6 +415,8 @@ class ResponsesRequest(OpenAIBaseModel):
extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
if self.kv_transfer_params:
extra_args["kv_transfer_params"] = self.kv_transfer_params
if self.ec_transfer_params:
extra_args["ec_transfer_params"] = self.ec_transfer_params
return SamplingParams.from_optional(
temperature=temperature,
@@ -675,6 +683,9 @@ class ResponsesResponse(OpenAIBaseModel):
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None, description="ECTransfer parameters."
)
# --8<-- [start:responses-response-extra-params]
# These are populated when enable_response_messages is set to True
@@ -720,6 +731,7 @@ class ResponsesResponse(OpenAIBaseModel):
input_messages: ResponseInputOutputMessage | None = None,
output_messages: ResponseInputOutputMessage | None = None,
kv_transfer_params: dict[str, Any] | None = None,
ec_transfer_params: dict[str, Any] | None = None,
) -> "ResponsesResponse":
incomplete_details: IncompleteDetails | None = None
if status == "incomplete":
@@ -758,6 +770,7 @@ class ResponsesResponse(OpenAIBaseModel):
user=request.user,
usage=usage,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
)
@@ -932,6 +932,7 @@ class OpenAIServingResponses(GenerateBaseServing):
status=status,
usage=usage,
kv_transfer_params=context.kv_transfer_params,
ec_transfer_params=context.ec_transfer_params,
)
if request.store:
@@ -132,6 +132,12 @@ class GenerateRequest(BaseModel):
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None,
description=(
"ECTransfer parameters used for encoder-cache disaggregated serving."
),
)
# Tracks which keys the caller explicitly set inside ``sampling_params``
# when the request was parsed from a JSON body. Lets the server tell
@@ -238,6 +244,12 @@ class GenerateResponse(BaseModel):
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None,
description=(
"ECTransfer parameters used for encoder-cache disaggregated serving."
),
)
####### Derender (postprocessing) #######
@@ -330,6 +330,7 @@ class ServingTokens(GenerateBaseServing):
usage=usage,
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
kv_transfer_params=final_res.kv_transfer_params,
ec_transfer_params=final_res.ec_transfer_params,
)
# Log complete response if output logging is enabled
+12
View File
@@ -207,6 +207,8 @@ if TYPE_CHECKING:
VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: bool = False
VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
VLLM_EC_SIDE_CHANNEL_HOST: str = "localhost"
VLLM_EC_SIDE_CHANNEL_PORT: int = 5601
VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1
@@ -1565,6 +1567,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int(
os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600")
),
# IP address used for the EC connector's ZMQ side channel
# (producer ROUTER bind, consumer DEALER dial).
"VLLM_EC_SIDE_CHANNEL_HOST": lambda: os.getenv(
"VLLM_EC_SIDE_CHANNEL_HOST", "localhost"
),
# Port for the EC connector's ZMQ side channel; advertised to peers
# via `ec_transfer_params.peer_port` on the producer's response.
"VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(
os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")
),
# Port used for Mooncake handshake between remote agents.
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
+4
View File
@@ -104,6 +104,7 @@ class RequestOutput:
None if decoder-only.
num_cached_tokens: The number of tokens with prefix cache hit.
kv_transfer_params: The params for remote K/V transfer.
ec_transfer_params: The params for remote encoder-cache transfer.
"""
def __init__(
@@ -121,6 +122,7 @@ class RequestOutput:
num_cached_tokens: int | None = None,
*,
kv_transfer_params: dict[str, Any] | None = None,
ec_transfer_params: dict[str, Any] | None = None,
# Forward compatibility, code that uses args added in new release can
# still run with older versions of vLLM without breaking.
**kwargs: Any,
@@ -141,12 +143,14 @@ class RequestOutput:
self.encoder_prompt_token_ids = encoder_prompt_token_ids
self.num_cached_tokens = num_cached_tokens
self.kv_transfer_params = kv_transfer_params
self.ec_transfer_params = ec_transfer_params
def add(self, next_output: "RequestOutput", aggregate: bool) -> None:
"""Merge subsequent RequestOutput into this one"""
self.finished |= next_output.finished
self.kv_transfer_params = next_output.kv_transfer_params
self.ec_transfer_params = next_output.ec_transfer_params
for next_completion in next_output.outputs:
for i, completion in enumerate(self.outputs):
+4
View File
@@ -9,6 +9,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorBase
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorBase_V1
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.engine import EngineCoreOutputs
@@ -248,3 +249,6 @@ class SchedulerInterface(ABC):
def get_kv_connector(self) -> "KVConnectorBase_V1 | None":
return None
def get_ec_connector(self) -> "ECConnectorBase | None":
return None
+20 -3
View File
@@ -10,6 +10,7 @@ from typing import Any
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorBase,
ECConnectorMetadata,
ECConnectorRole,
)
@@ -1675,6 +1676,7 @@ class Scheduler(SchedulerInterface):
new_token_ids = generated_token_ids
pooler_output = pooler_outputs[req_index] if pooler_outputs else None
kv_transfer_params = None
ec_transfer_params = None
status_before_stop = request.status
num_output_tokens_before = len(request._output_token_ids)
@@ -1761,7 +1763,7 @@ class Scheduler(SchedulerInterface):
finish_reason = request.get_finished_reason()
finished = self._handle_stopped_request(request)
if finished:
kv_transfer_params = self._free_request(request)
kv_transfer_params, ec_transfer_params = self._free_request(request)
if status_before_stop == RequestStatus.RUNNING:
stopped_running_reqs.add(request)
@@ -1785,6 +1787,7 @@ class Scheduler(SchedulerInterface):
new_token_ids
or pooler_output is not None
or kv_transfer_params
or ec_transfer_params
or stopped
):
# Add EngineCoreOutput for this Request.
@@ -1800,6 +1803,7 @@ class Scheduler(SchedulerInterface):
events=request.take_events(),
prefill_stats=request.take_prefill_stats(),
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
trace_headers=request.trace_headers,
routed_experts=routed_experts,
num_nans_in_logits=request.num_nans_in_logits,
@@ -2151,11 +2155,21 @@ class Scheduler(SchedulerInterface):
def _free_request(
self, request: Request, delay_free_blocks: bool = False
) -> dict[str, Any] | None:
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
assert request.is_finished()
self._inflight_prefills.discard(request)
connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request)
# EC Connector: mirror the KV hook. The contract requires firing
# before the encoder cache is freed so the connector can inspect
# per-request state (e.g. which mm_hashes it recorded during
# save_caches()) and emit ec_transfer_params for the response body.
ec_xfer_params: dict[str, Any] | None = None
if self.ec_connector is not None:
ec_delay_free, ec_xfer_params = self.ec_connector.request_finished(request)
connector_delay_free_blocks |= ec_delay_free
self.encoder_cache_manager.free(request)
request_id = request.request_id
self.finished_req_ids.add(request_id)
@@ -2166,7 +2180,7 @@ class Scheduler(SchedulerInterface):
if not delay_free_blocks:
self._free_blocks(request)
return kv_xfer_params
return kv_xfer_params, ec_xfer_params
def _free_blocks(self, request: Request):
assert request.is_finished()
@@ -2414,6 +2428,9 @@ class Scheduler(SchedulerInterface):
def get_kv_connector(self) -> KVConnectorBase_V1 | None:
return self.connector
def get_ec_connector(self) -> ECConnectorBase | None:
return self.ec_connector
def _connector_finished(
self, request: Request
) -> tuple[bool, dict[str, Any] | None]:
+1
View File
@@ -190,6 +190,7 @@ class EngineCoreOutput(
stop_reason: int | str | None = None
events: list[EngineCoreEvent] | None = None
kv_transfer_params: dict[str, Any] | None = None
ec_transfer_params: dict[str, Any] | None = None
trace_headers: Mapping[str, str] | None = None
+9
View File
@@ -400,6 +400,15 @@ class EngineCore:
"Disabling KVTransfer for this request."
)
if (
request.ec_transfer_params is not None
and self.scheduler.get_ec_connector() is None
):
logger.warning(
"Got ec_transfer_params, but no ECConnector found. "
"Disabling ECTransfer for this request."
)
self.scheduler.add_request(request)
if request.abort_immediately:
# Immediately abort so the connector's request_finished hook runs
+11 -1
View File
@@ -276,6 +276,7 @@ class RequestState:
finish_reason: FinishReason | None,
stop_reason: int | str | None,
kv_transfer_params: dict[str, Any] | None = None,
ec_transfer_params: dict[str, Any] | None = None,
) -> RequestOutput | PoolingRequestOutput | None:
finished = finish_reason is not None
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
@@ -327,7 +328,11 @@ class RequestState:
external_req_id = self.parent_req.external_req_id
return self._new_request_output(
external_req_id, outputs, finished, kv_transfer_params
external_req_id,
outputs,
finished,
kv_transfer_params,
ec_transfer_params,
)
def _new_request_output(
@@ -336,6 +341,7 @@ class RequestState:
outputs: list[CompletionOutput] | list[PoolingOutput],
finished: bool,
kv_transfer_params: dict[str, Any] | None = None,
ec_transfer_params: dict[str, Any] | None = None,
) -> RequestOutput | PoolingRequestOutput:
# If prompt embeds were used, put placeholder prompt token ids
prompt_token_ids = self.prompt_token_ids
@@ -369,6 +375,7 @@ class RequestState:
outputs=cast(list[CompletionOutput], outputs),
finished=finished,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
num_cached_tokens=self.num_cached_tokens,
metrics=self.stats,
)
@@ -497,6 +504,7 @@ class OutputProcessor:
finish_reason=FinishReason.ABORT,
stop_reason=None,
kv_transfer_params=None,
ec_transfer_params=None,
)
):
req_state.queue.put(request_output)
@@ -620,6 +628,7 @@ class OutputProcessor:
finish_reason = engine_core_output.finish_reason
stop_reason = engine_core_output.stop_reason
kv_transfer_params = engine_core_output.kv_transfer_params
ec_transfer_params = engine_core_output.ec_transfer_params
if engine_core_output.routed_experts is not None:
req_state.routed_experts_chunks.append(
engine_core_output.routed_experts
@@ -654,6 +663,7 @@ class OutputProcessor:
finish_reason,
stop_reason,
kv_transfer_params,
ec_transfer_params,
):
if req_state.streaming_input:
request_output.finished = False
+5
View File
@@ -100,6 +100,8 @@ class Request:
# P/D: Connector-specific KV transfer parameters.
self.kv_transfer_params: dict[str, Any] | None = None
# E/P/D: Connector-specific encoder-cache transfer parameters.
self.ec_transfer_params: dict[str, Any] | None = None
if pooling_params is not None:
# Pooling models.
@@ -115,6 +117,9 @@ class Request:
self.kv_transfer_params = sampling_params.extra_args.get(
"kv_transfer_params"
)
self.ec_transfer_params = sampling_params.extra_args.get(
"ec_transfer_params"
)
self.kv_cache_report_mode = sampling_params.extra_args.get(
"kv_cache_report_mode", "incremental"
)