[EC Connector] Add EC Transfer Params (#42433)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
omerpaz95
2026-07-12 14:35:33 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 370b678a02
commit 8df14cfc8c
55 changed files with 416 additions and 12 deletions
+5
View File
@@ -100,6 +100,8 @@ class Request:
# P/D: Connector-specific KV transfer parameters.
self.kv_transfer_params: dict[str, Any] | None = None
# E/P/D: Connector-specific encoder-cache transfer parameters.
self.ec_transfer_params: dict[str, Any] | None = None
if pooling_params is not None:
# Pooling models.
@@ -115,6 +117,9 @@ class Request:
self.kv_transfer_params = sampling_params.extra_args.get(
"kv_transfer_params"
)
self.ec_transfer_params = sampling_params.extra_args.get(
"ec_transfer_params"
)
self.kv_cache_report_mode = sampling_params.extra_args.get(
"kv_cache_report_mode", "incremental"
)