[Bugfix][MLA] Restore DSpark cache-group capability under optimized Python (#55234)
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -224,9 +224,10 @@ class KVCacheSpec:
|
||||
"""
|
||||
Merge a list of KVCacheSpec objects into a single KVCacheSpec object.
|
||||
"""
|
||||
assert all(spec == specs[0] for spec in specs[1:]), (
|
||||
"All layers in the same KV cache group must be the same."
|
||||
)
|
||||
if not all(spec == specs[0] for spec in specs[1:]):
|
||||
raise AssertionError(
|
||||
"All layers in the same KV cache group must be the same."
|
||||
)
|
||||
return copy.deepcopy(specs[0])
|
||||
|
||||
def is_uniform_with_collection(
|
||||
@@ -558,7 +559,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
model_version: str | None = None
|
||||
storage_block_size: int | None = None
|
||||
"""Token width used to view storage when it differs from the kernel block."""
|
||||
# Marks draft groups that flatten a non-causal query block into decode rows.
|
||||
# Group capability enabled when any member flattens a non-causal query block
|
||||
# into decode rows. Runtime metadata still selects causal vs. non-causal mode.
|
||||
non_causal_multi_token_decode: bool = False
|
||||
# MLA stores a single latent vector per state; there is no separate V.
|
||||
head_size_v: int = 0
|
||||
@@ -586,11 +588,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
"quantization method, tokens per state, model version, and storage "
|
||||
"block size."
|
||||
)
|
||||
non_causal_mtd_set = {spec.non_causal_multi_token_decode for spec in specs}
|
||||
assert len(non_causal_mtd_set) == 1, (
|
||||
"All attention layers in the same KV cache group must agree on "
|
||||
"non_causal_multi_token_decode."
|
||||
)
|
||||
merged_spec = cls(
|
||||
block_size=specs[0].block_size,
|
||||
num_kv_heads=specs[0].num_kv_heads,
|
||||
@@ -604,7 +601,9 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
tokens_per_state=tokens_per_state_set.pop(),
|
||||
model_version=model_version_set.pop(),
|
||||
storage_block_size=storage_block_size_set.pop(),
|
||||
non_causal_multi_token_decode=non_causal_mtd_set.pop(),
|
||||
non_causal_multi_token_decode=any(
|
||||
spec.non_causal_multi_token_decode for spec in specs
|
||||
),
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
|
||||
Reference in New Issue
Block a user