[Bugfix][KV Offload] Propagate EAGLE mode to SimpleCPU coordinator (#49071)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
5c9f6557d7
commit
9bc266d923
@@ -117,12 +117,13 @@ class SimpleCPUOffloadScheduler:
|
|||||||
"lazy" if lazy_offload else "eager",
|
"lazy" if lazy_offload else "eager",
|
||||||
)
|
)
|
||||||
|
|
||||||
# TODO (yifan): maybe need to enable kv_cache_events and metrics_collector here.
|
spec_config = vllm_config.speculative_config
|
||||||
|
use_eagle = spec_config is not None and spec_config.use_eagle()
|
||||||
self.cpu_coordinator: KVCacheCoordinator = get_kv_cache_coordinator(
|
self.cpu_coordinator: KVCacheCoordinator = get_kv_cache_coordinator(
|
||||||
kv_cache_config=self.cpu_kv_cache_config,
|
kv_cache_config=self.cpu_kv_cache_config,
|
||||||
max_model_len=vllm_config.model_config.max_model_len,
|
max_model_len=vllm_config.model_config.max_model_len,
|
||||||
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
||||||
use_eagle=False,
|
use_eagle=use_eagle,
|
||||||
enable_caching=True,
|
enable_caching=True,
|
||||||
enable_kv_cache_events=self.enable_kv_cache_events,
|
enable_kv_cache_events=self.enable_kv_cache_events,
|
||||||
dcp_world_size=dcp_world_size,
|
dcp_world_size=dcp_world_size,
|
||||||
@@ -131,7 +132,6 @@ class SimpleCPUOffloadScheduler:
|
|||||||
hash_block_size=self.hash_block_size,
|
hash_block_size=self.hash_block_size,
|
||||||
)
|
)
|
||||||
self.cpu_block_pool: BlockPool = self.cpu_coordinator.block_pool
|
self.cpu_block_pool: BlockPool = self.cpu_coordinator.block_pool
|
||||||
|
|
||||||
# GPU block pool reference - bound after scheduler builds kv_cache_manager
|
# GPU block pool reference - bound after scheduler builds kv_cache_manager
|
||||||
self._gpu_block_pool: BlockPool | None = None
|
self._gpu_block_pool: BlockPool | None = None
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user