[Feature][Spec] Support disabling trailing prefix-cache block dropping (#53388)
Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com> Signed-off-by: Ziming Huang <48115868+ZeldaHuang@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
9e905f7450
commit
481839ad9e
@@ -1593,16 +1593,17 @@ class MooncakeStoreWorker:
|
||||
for group in kv_cache_config.transfer_groups
|
||||
]
|
||||
spec_cfg = getattr(vllm_config, "speculative_config", None)
|
||||
use_eagle = bool(
|
||||
spec_cfg.use_eagle()
|
||||
if spec_cfg is not None and callable(getattr(spec_cfg, "use_eagle", None))
|
||||
use_eagle_block_drop = bool(
|
||||
spec_cfg.use_eagle_block_drop()
|
||||
if spec_cfg is not None
|
||||
and callable(getattr(spec_cfg, "use_eagle_block_drop", None))
|
||||
else False
|
||||
)
|
||||
self.coord = MooncakeStoreCoordinator(
|
||||
self._kv_cache_groups,
|
||||
scheduler_block_size=self.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
use_eagle=use_eagle,
|
||||
use_eagle=use_eagle_block_drop,
|
||||
retention_interval=kv_cache_config.prefix_cache_retention_interval,
|
||||
dcp_world_size=self.dcp_size,
|
||||
)
|
||||
|
||||
@@ -223,11 +223,11 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
if g.is_eagle_group
|
||||
}
|
||||
|
||||
use_eagle = (
|
||||
use_eagle_block_drop = (
|
||||
vllm_config.speculative_config is not None
|
||||
and vllm_config.speculative_config.use_eagle()
|
||||
and vllm_config.speculative_config.use_eagle_block_drop()
|
||||
)
|
||||
if use_eagle and not eagle_groups:
|
||||
if use_eagle_block_drop and not eagle_groups:
|
||||
eagle_groups = set(range(len(kv_cache_config.kv_cache_groups)))
|
||||
|
||||
if eagle_groups:
|
||||
|
||||
Reference in New Issue
Block a user