[Bugfix][MLA] Scope DSpark non-causal capability to builder layers (#57992)
Signed-off-by: Xavier Aguilar <xavier.aguilarfruto@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
71f9a74920
commit
9367d8b969
@@ -19,6 +19,21 @@ class _NonCausalMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
supports_non_causal_multi_token_decode = True
|
||||
|
||||
|
||||
class _CausalOnlyMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
pass
|
||||
|
||||
|
||||
class _DraftMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
"""Triton-like: non-causal decode is supported, DCP draft decode is not."""
|
||||
|
||||
supports_non_causal_multi_token_decode = True
|
||||
|
||||
|
||||
class _DcpCapableMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
supports_non_causal_multi_token_decode = True
|
||||
supports_non_causal_multi_token_dcp = True
|
||||
|
||||
|
||||
def _metadata(
|
||||
query_start_loc: list[int],
|
||||
num_tokens: int | None = None,
|
||||
@@ -60,6 +75,54 @@ def _builder(marked: bool = True) -> _NonCausalMLAMetadataBuilder:
|
||||
return builder
|
||||
|
||||
|
||||
def _mla_layer(*, non_causal: bool = False) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
non_causal_multi_token_decode=non_causal,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
prefill_backend=SimpleNamespace(clone=lambda: None),
|
||||
)
|
||||
|
||||
|
||||
def _merged_mla_spec() -> MLAAttentionSpec:
|
||||
return MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
non_causal_multi_token_decode=True,
|
||||
)
|
||||
|
||||
|
||||
def _dspark_dcp_vllm_config(
|
||||
static_forward_context: dict[str, SimpleNamespace],
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
method="dspark", num_speculative_tokens=None
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=2,
|
||||
prefill_context_parallel_size=1,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
static_forward_context=static_forward_context
|
||||
),
|
||||
model_config=SimpleNamespace(
|
||||
dtype=torch.bfloat16,
|
||||
max_model_len=128,
|
||||
get_num_attention_heads=lambda _parallel_config: 16,
|
||||
),
|
||||
cache_config=SimpleNamespace(cache_dtype="auto", block_size=64),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=1),
|
||||
attention_config=SimpleNamespace(use_prefill_query_quantization=False),
|
||||
)
|
||||
|
||||
|
||||
def test_group_capability_keeps_runtime_causality_per_step():
|
||||
builder = _builder()
|
||||
|
||||
@@ -137,3 +200,61 @@ def test_mla_cache_marker_is_promoted_to_group_capability():
|
||||
[unmarked, unmarked]
|
||||
).non_causal_multi_token_decode
|
||||
assert MLAAttentionSpec.merge([unmarked, marked]).non_causal_multi_token_decode
|
||||
|
||||
|
||||
def test_builder_scopes_noncausal_capability_to_its_layers():
|
||||
merged_spec = _merged_mla_spec()
|
||||
static_forward_context = {
|
||||
"target": _mla_layer(non_causal=False),
|
||||
"draft": _mla_layer(non_causal=True),
|
||||
# Indexer and compressor caches share an MLA group but predate the flag.
|
||||
"indexer": SimpleNamespace(),
|
||||
}
|
||||
vllm_config = _dspark_dcp_vllm_config(static_forward_context)
|
||||
device = torch.device("cpu")
|
||||
|
||||
def _flag(layer_names: list[str]) -> bool:
|
||||
return _DcpCapableMLAMetadataBuilder(
|
||||
merged_spec,
|
||||
layer_names,
|
||||
vllm_config,
|
||||
device,
|
||||
supports_dcp_with_varlen=True,
|
||||
).non_causal_multi_token_decode
|
||||
|
||||
assert not _flag(["target"])
|
||||
assert _flag(["draft"])
|
||||
assert _flag(["target", "draft"])
|
||||
assert not _flag(["target", "indexer"])
|
||||
|
||||
|
||||
def test_merged_group_spec_does_not_mark_a_target_only_builder():
|
||||
# Both builders receive the same merged spec; only the draft layer is
|
||||
# non-causal. Constructing through __init__ would still fail DCP
|
||||
# validation if the builder read the group flag.
|
||||
merged_spec = _merged_mla_spec()
|
||||
static_forward_context = {
|
||||
"target": _mla_layer(non_causal=False),
|
||||
"draft": _mla_layer(non_causal=True),
|
||||
}
|
||||
vllm_config = _dspark_dcp_vllm_config(static_forward_context)
|
||||
device = torch.device("cpu")
|
||||
assert merged_spec.non_causal_multi_token_decode
|
||||
|
||||
target = _CausalOnlyMLAMetadataBuilder(
|
||||
merged_spec,
|
||||
["target"],
|
||||
vllm_config,
|
||||
device,
|
||||
supports_dcp_with_varlen=True,
|
||||
)
|
||||
assert not target.non_causal_multi_token_decode
|
||||
|
||||
with pytest.raises(ValueError, match="non-causal draft"):
|
||||
_DraftMLAMetadataBuilder(
|
||||
merged_spec,
|
||||
["draft"],
|
||||
vllm_config,
|
||||
device,
|
||||
supports_dcp_with_varlen=True,
|
||||
)
|
||||
|
||||
@@ -2384,8 +2384,12 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self.non_causal_multi_token_decode = getattr(
|
||||
kv_cache_spec, "non_causal_multi_token_decode", False
|
||||
# merge() unions this flag across the KV group, so a shared target/draft
|
||||
# group would mark the target builder as non-causal.
|
||||
ctx = self.compilation_config.static_forward_context
|
||||
self.non_causal_multi_token_decode = any(
|
||||
getattr(ctx[name], "non_causal_multi_token_decode", False)
|
||||
for name in layer_names
|
||||
)
|
||||
self._validate_dspark_dcp_support(supports_dcp_with_varlen)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user