[Core] Add MRV2 virtual-batch PCP for MLA (#46570)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Matthew Bonanni
parent
9243e0124e
commit
b6ff8a2f50
@@ -578,6 +578,7 @@ def build_attn_metadata(
|
||||
seq_lens_cpu_upper_bound: torch.Tensor | None = None,
|
||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
is_prefilling: torch.Tensor | None = None,
|
||||
mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None = None,
|
||||
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
|
||||
for_cudagraph_capture: bool = False,
|
||||
@@ -605,6 +606,11 @@ def build_attn_metadata(
|
||||
if model_specific_attn_metadata is not None
|
||||
else {}
|
||||
)
|
||||
# Model-specific metadata (e.g. Mamba hybrid) may supply its own
|
||||
# padding-aware is_prefilling, which takes precedence over the default.
|
||||
group_is_prefilling = common_attn_metadata_extra_kwargs.pop(
|
||||
"is_prefilling", is_prefilling
|
||||
)
|
||||
common_attn_metadata = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc_gpu,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
@@ -619,6 +625,7 @@ def build_attn_metadata(
|
||||
causal=group_causal,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
positions=positions,
|
||||
is_prefilling=group_is_prefilling,
|
||||
mm_req_doc_ranges=mm_req_doc_ranges,
|
||||
rswa_prefix_lens=rswa_prefix_lens,
|
||||
**common_attn_metadata_extra_kwargs,
|
||||
|
||||
Reference in New Issue
Block a user