[EC Connector] Add EC Transfer Params (#42433)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -190,6 +190,7 @@ class EngineCoreOutput(
|
||||
stop_reason: int | str | None = None
|
||||
events: list[EngineCoreEvent] | None = None
|
||||
kv_transfer_params: dict[str, Any] | None = None
|
||||
ec_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
trace_headers: Mapping[str, str] | None = None
|
||||
|
||||
|
||||
@@ -400,6 +400,15 @@ class EngineCore:
|
||||
"Disabling KVTransfer for this request."
|
||||
)
|
||||
|
||||
if (
|
||||
request.ec_transfer_params is not None
|
||||
and self.scheduler.get_ec_connector() is None
|
||||
):
|
||||
logger.warning(
|
||||
"Got ec_transfer_params, but no ECConnector found. "
|
||||
"Disabling ECTransfer for this request."
|
||||
)
|
||||
|
||||
self.scheduler.add_request(request)
|
||||
if request.abort_immediately:
|
||||
# Immediately abort so the connector's request_finished hook runs
|
||||
|
||||
@@ -276,6 +276,7 @@ class RequestState:
|
||||
finish_reason: FinishReason | None,
|
||||
stop_reason: int | str | None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput | None:
|
||||
finished = finish_reason is not None
|
||||
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
|
||||
@@ -327,7 +328,11 @@ class RequestState:
|
||||
external_req_id = self.parent_req.external_req_id
|
||||
|
||||
return self._new_request_output(
|
||||
external_req_id, outputs, finished, kv_transfer_params
|
||||
external_req_id,
|
||||
outputs,
|
||||
finished,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
)
|
||||
|
||||
def _new_request_output(
|
||||
@@ -336,6 +341,7 @@ class RequestState:
|
||||
outputs: list[CompletionOutput] | list[PoolingOutput],
|
||||
finished: bool,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput:
|
||||
# If prompt embeds were used, put placeholder prompt token ids
|
||||
prompt_token_ids = self.prompt_token_ids
|
||||
@@ -369,6 +375,7 @@ class RequestState:
|
||||
outputs=cast(list[CompletionOutput], outputs),
|
||||
finished=finished,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
num_cached_tokens=self.num_cached_tokens,
|
||||
metrics=self.stats,
|
||||
)
|
||||
@@ -497,6 +504,7 @@ class OutputProcessor:
|
||||
finish_reason=FinishReason.ABORT,
|
||||
stop_reason=None,
|
||||
kv_transfer_params=None,
|
||||
ec_transfer_params=None,
|
||||
)
|
||||
):
|
||||
req_state.queue.put(request_output)
|
||||
@@ -620,6 +628,7 @@ class OutputProcessor:
|
||||
finish_reason = engine_core_output.finish_reason
|
||||
stop_reason = engine_core_output.stop_reason
|
||||
kv_transfer_params = engine_core_output.kv_transfer_params
|
||||
ec_transfer_params = engine_core_output.ec_transfer_params
|
||||
if engine_core_output.routed_experts is not None:
|
||||
req_state.routed_experts_chunks.append(
|
||||
engine_core_output.routed_experts
|
||||
@@ -654,6 +663,7 @@ class OutputProcessor:
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
):
|
||||
if req_state.streaming_input:
|
||||
request_output.finished = False
|
||||
|
||||
Reference in New Issue
Block a user