[Bugfix] Preserve tensor causal metadata for grouped attention (#48135)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
@@ -573,7 +573,7 @@ def build_attn_metadata(
|
||||
mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None = None,
|
||||
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
|
||||
for_cudagraph_capture: bool = False,
|
||||
causal: bool | Mapping[int, bool] = True,
|
||||
causal: bool | torch.Tensor | Mapping[int, bool] = True,
|
||||
rswa_prefix_lens: torch.Tensor | None = None,
|
||||
) -> dict[str, Any]:
|
||||
seq_lens = seq_lens[:num_reqs]
|
||||
@@ -588,7 +588,9 @@ def build_attn_metadata(
|
||||
block_table = block_tables[i]
|
||||
slot_mapping = slot_mappings[i]
|
||||
# Per-group causal for hybrid drafters (mixed SWA/full attention).
|
||||
group_causal = causal if isinstance(causal, bool) else causal.get(i, True)
|
||||
group_causal = (
|
||||
causal if isinstance(causal, (bool, torch.Tensor)) else causal.get(i, True)
|
||||
)
|
||||
|
||||
common_attn_metadata_extra_kwargs = (
|
||||
model_specific_attn_metadata.get_extra_common_attn_kwargs(i, num_reqs)
|
||||
|
||||
Reference in New Issue
Block a user