[Perf] Optimize hidden state extraction logic (#37374)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
f743254143
commit
4e2eba28be
@@ -734,6 +734,17 @@ class VllmConfig:
|
||||
Right now, this function reads the offloading settings from
|
||||
CacheConfig and configures the KVTransferConfig accordingly.
|
||||
"""
|
||||
# Check if KV connector requires chunked prefill to be disabled.
|
||||
if (
|
||||
self.kv_transfer_config is not None
|
||||
and self.kv_transfer_config.kv_connector == "ExampleHiddenStatesConnector"
|
||||
and self.scheduler_config.enable_chunked_prefill
|
||||
):
|
||||
raise ValueError(
|
||||
"ExampleHiddenStatesConnector does not support chunked prefill. "
|
||||
"Please disable chunked prefill (--no-enable-chunked-prefill)."
|
||||
)
|
||||
|
||||
# KV offloading is only activated when kv_offloading_size is set.
|
||||
if (kv_offloading_size := self.cache_config.kv_offloading_size) is None:
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user