[Bugfix] Scale KV page size for hidden states extraction with TP (#50894)
Signed-off-by: Claude <noreply@anthropic.com> Signed-off-by: Orestis Zambounis <orestis-z@users.noreply.github.com> Signed-off-by: Orestis Zambounis <23146389+orestis-z@users.noreply.github.com> Signed-off-by: Speculators Bot <bot@speculators.ai> Signed-off-by: Speculators Bot <speculatorsbot@gmail.com> Co-authored-by: Orestis Zambounis <orestis-z@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: Speculators Bot <speculatorsbot@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Orestis Zambounis
Claude Opus 4.6
Benjamin Chislett
Speculators Bot
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c676e4930b
commit
ff5f6d41b1
@@ -2217,6 +2217,38 @@ def _largest_divisor_at_most(value: int, limit: int) -> int:
|
||||
return 1
|
||||
|
||||
|
||||
def _ensure_min_page_size(
|
||||
groups: list[KVCacheGroupSpec],
|
||||
common_page: int,
|
||||
hidden_specs: dict[str, HiddenStateCacheSpec],
|
||||
) -> tuple[list[KVCacheGroupSpec], int]:
|
||||
"""Scale up group block sizes so the common page is at least as large as
|
||||
the biggest hidden-state per-token cost.
|
||||
|
||||
This protects the hidden state extraction feature, where we store hidden states in
|
||||
the KV cache, and return them to clients using a custom hidden state connector.
|
||||
|
||||
Returns the (possibly scaled) groups and updated common page size.
|
||||
"""
|
||||
max_per_token = max(
|
||||
spec.num_kv_heads * spec.head_size * get_dtype_size(spec.dtype)
|
||||
for spec in hidden_specs.values()
|
||||
)
|
||||
if max_per_token <= common_page:
|
||||
return groups, common_page
|
||||
|
||||
scale = cdiv(max_per_token, common_page)
|
||||
common_page *= scale
|
||||
scaled: list[KVCacheGroupSpec] = []
|
||||
for g in groups:
|
||||
s = g.kv_cache_spec
|
||||
kw: dict[str, int] = {"block_size": s.block_size * scale}
|
||||
if isinstance(s, (AttentionSpec, MambaSpec)) and s.page_size_padded is not None:
|
||||
kw["page_size_padded"] = s.page_size_padded * scale
|
||||
scaled.append(KVCacheGroupSpec(g.layer_names, replace(s, **kw)))
|
||||
return scaled, common_page
|
||||
|
||||
|
||||
def get_kv_cache_groups(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
@@ -2288,6 +2320,8 @@ def get_kv_cache_groups(
|
||||
# Add hidden-state layers back with page aligned to the common page.
|
||||
if hidden_specs:
|
||||
common_page = get_uniform_page_size([g.kv_cache_spec for g in groups])
|
||||
# TP may shrink the common page below hidden-state per-token cost.
|
||||
groups, common_page = _ensure_min_page_size(groups, common_page, hidden_specs)
|
||||
group_block_size = math.gcd(*(g.kv_cache_spec.block_size for g in groups))
|
||||
for name, spec in hidden_specs.items():
|
||||
per_token = spec.num_kv_heads * spec.head_size * get_dtype_size(spec.dtype)
|
||||
|
||||
Reference in New Issue
Block a user