[Bugfix] Scale KV page size for hidden states extraction with TP (#50894)
Signed-off-by: Claude <noreply@anthropic.com> Signed-off-by: Orestis Zambounis <orestis-z@users.noreply.github.com> Signed-off-by: Orestis Zambounis <23146389+orestis-z@users.noreply.github.com> Signed-off-by: Speculators Bot <bot@speculators.ai> Signed-off-by: Speculators Bot <speculatorsbot@gmail.com> Co-authored-by: Orestis Zambounis <orestis-z@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: Speculators Bot <speculatorsbot@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Orestis Zambounis
Claude Opus 4.6
Benjamin Chislett
Speculators Bot
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c676e4930b
commit
ff5f6d41b1
@@ -3208,6 +3208,45 @@ def test_mla_with_incompatible_swa_uses_one_full_allocation_group(caplog_vllm):
|
||||
assert "attention compute is unchanged" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_hidden_states_with_tp_scales_page_size():
|
||||
"""When TP shrinks KV pages below the hidden-state per-token cost,
|
||||
get_kv_cache_groups must scale up target block sizes so that the
|
||||
common page accommodates the unsharded hidden states."""
|
||||
# Simulate TP=4 sharding a model with 8 KV heads → 2 per rank.
|
||||
# KV page = block_size(16) * num_kv_heads(2) * head_size(64) * dtype(2)
|
||||
# = 16 * 2 * 64 * 2 = 4096 bytes.
|
||||
kv_spec = new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
# Hidden-state per-token cost = num_hidden_states(6) * hidden_size(512)
|
||||
# * dtype(2) = 6144 bytes, which exceeds the 4096-byte KV page.
|
||||
hs_spec = HiddenStateCacheSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=6,
|
||||
head_size=512,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
specs = {
|
||||
"target.0.attn": kv_spec,
|
||||
"target.1.attn": kv_spec,
|
||||
"cache_only_layers.48": hs_spec,
|
||||
}
|
||||
|
||||
groups = get_kv_cache_groups(_grouping_config(), specs)
|
||||
|
||||
# The hidden-state layer should be present and no assertion should fire.
|
||||
all_layers = {name for g in groups for name in g.layer_names}
|
||||
assert "cache_only_layers.48" in all_layers
|
||||
|
||||
# The target group block sizes must have been scaled up.
|
||||
for g in groups:
|
||||
if "cache_only_layers.48" not in g.layer_names:
|
||||
assert g.kv_cache_spec.block_size > kv_spec.block_size
|
||||
|
||||
|
||||
def test_get_kv_cache_spec_kind_prefers_specific_attention_subclasses():
|
||||
assert get_kv_cache_spec_kind(new_mla_spec()) == KVCacheSpecKind.MLA_ATTENTION
|
||||
|
||||
|
||||
Reference in New Issue
Block a user