[Bugfix] Reject a prefix_match_unit that a single KV cache group cannot honor (#58021)

Signed-off-by: QHarshil <harshil_c@hotmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harshil
2026-09-29 23:12:15 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent eec2a86b1e
commit be255076d0
2 changed files with 21 additions and 2 deletions
@@ -310,7 +310,9 @@ def _make_vllm_config(
kv_cache_layout: KVCacheLayout = KVCacheLayout.LBHNC, kv_cache_layout: KVCacheLayout = KVCacheLayout.LBHNC,
disable_hybrid_kv_cache_manager: bool = True, disable_hybrid_kv_cache_manager: bool = True,
) -> SimpleNamespace: ) -> SimpleNamespace:
cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=10) cache_config = SimpleNamespace(
block_size=16, num_gpu_blocks=10, prefix_match_unit=None
)
cache_config.get_resolved_kv_cache_layout = lambda: kv_cache_layout cache_config.get_resolved_kv_cache_layout = lambda: kv_cache_layout
return SimpleNamespace( return SimpleNamespace(
model_config=_FakeModelConfig(), model_config=_FakeModelConfig(),
+18 -1
View File
@@ -714,7 +714,9 @@ def resolve_kv_cache_block_sizes(
group's effective block size. Attention groups are scaled by DCP; group's effective block size. Attention groups are scaled by DCP;
Mamba groups keep their full per-rank state and are not scaled. Mamba groups keep their full per-rank state and are not scaled.
- ``hash_block_size`` is the granularity at which ``Request.block_hashes`` - ``hash_block_size`` is the granularity at which ``Request.block_hashes``
is computed. Single group: equals scheduler block size. Multiple groups: is computed. Single group: equals scheduler block size, and any other
``cache_config.prefix_match_unit`` is rejected while block hashing is
active. Multiple groups:
``cache_config.prefix_match_unit`` override if set, else the GCD of ``cache_config.prefix_match_unit`` override if set, else the GCD of
group block sizes; every group's block size must be divisible by it. group block sizes; every group's block size must be divisible by it.
Returns the scheduler block size (i.e. disables finer hashing) if block Returns the scheduler block size (i.e. disables finer hashing) if block
@@ -728,6 +730,21 @@ def resolve_kv_cache_block_sizes(
if groups and not groups[0].kv_cache_spec.dcp_sharded: if groups and not groups[0].kv_cache_spec.dcp_sharded:
dcp = 1 dcp = 1
bs = cache_config.block_size * dcp bs = cache_config.block_size * dcp
# The Mamba prefill checkpoint builder reads prefix_match_unit directly,
# so a value dropped here puts its checkpoints off the scheduler's grid.
if (
cache_config.prefix_match_unit not in (None, bs)
and len(groups) == 1
and (
cache_config.enable_prefix_caching
or vllm_config.kv_transfer_config is not None
)
):
raise ValueError(
f"Invalid prefix_match_unit={cache_config.prefix_match_unit}; "
"with a single KV cache group, prefix-cache hits land on the "
f"block size ({bs}). Unset it or set it to {bs}."
)
return bs, bs return bs, bs
group_block_sizes = [ group_block_sizes = [