[Performance] Split FlashAttn attention and cache update (#25954)

Signed-off-by: ElizaWszola <ewszola@redhat.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Luka Govedič <luka.govedic@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Varun Sundar Rabindranath <varunsundar08@gmail.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Co-authored-by: Luka Govedič <luka.govedic@gmail.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Luka Govedič <lgovedic@redhat.com>
This commit is contained in:
ElizaWszola
2026-01-23 17:28:06 -08:00
committed by GitHub
co-authored by mgoin Varun Sundar Rabindranath Matthew Bonanni Luka Govedič Luka Govedič Lucas Wilkinson Luka Govedič
parent 0118cdcc02
commit a28b94e6ef
21 changed files with 458 additions and 68 deletions
+127 -14
View File
@@ -314,6 +314,7 @@ class ExecuteModelState(NamedTuple):
aux_hidden_states: list[torch.Tensor] | None
ec_connector_output: ECConnectorOutput | None
cudagraph_stats: CUDAGraphStat | None
slot_mappings: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None
class GPUModelRunner(
@@ -1595,6 +1596,7 @@ class GPUModelRunner(
for_cudagraph_capture: bool = False,
num_scheduled_tokens: dict[str, int] | None = None,
cascade_attn_prefix_lens: list[list[int]] | None = None,
slot_mappings: dict[int, torch.Tensor] | None = None,
) -> tuple[PerLayerAttnMetadata, CommonAttentionMetadata | None]:
"""
:return: tuple[attn_metadata, spec_decode_common_attn_metadata]
@@ -1628,7 +1630,7 @@ class GPUModelRunner(
kv_cache_groups = self.kv_cache_config.kv_cache_groups
def _get_block_table_and_slot_mapping(kv_cache_gid: int):
def _get_block_table(kv_cache_gid: int):
assert num_reqs_padded is not None and num_tokens_padded is not None
kv_cache_spec = kv_cache_groups[kv_cache_gid].kv_cache_spec
if isinstance(kv_cache_spec, EncoderOnlyAttentionSpec):
@@ -1637,24 +1639,19 @@ class GPUModelRunner(
dtype=torch.int32,
device=self.device,
)
slot_mapping = torch.zeros(
(num_tokens_padded,),
dtype=torch.int64,
device=self.device,
)
else:
blk_table = self.input_batch.block_table[kv_cache_gid]
blk_table_tensor = blk_table.get_device_tensor(num_reqs_padded)
slot_mapping = blk_table.slot_mapping.gpu[:num_tokens_padded]
# Fill unused with -1. Needed for reshape_and_cache in full cuda
# graph mode. `blk_table_tensor` -1 to match mamba PAD_SLOT_ID
slot_mapping[num_tokens:num_tokens_padded].fill_(-1)
blk_table_tensor[num_reqs:num_reqs_padded].fill_(-1)
return blk_table_tensor
return blk_table_tensor, slot_mapping
assert slot_mappings is not None
block_table_gid_0 = _get_block_table(0)
slot_mapping_gid_0 = slot_mappings[0]
block_table_gid_0, slot_mapping_gid_0 = _get_block_table_and_slot_mapping(0)
if self.model_config.enable_return_routed_experts:
self.slot_mapping = slot_mapping_gid_0[:num_tokens].cpu().numpy()
cm_base = CommonAttentionMetadata(
@@ -1779,9 +1776,8 @@ class GPUModelRunner(
num_reqs_padded,
)
if kv_cache_gid > 0:
cm.block_table_tensor, cm.slot_mapping = (
_get_block_table_and_slot_mapping(kv_cache_gid)
)
cm.block_table_tensor = _get_block_table(kv_cache_gid)
cm.slot_mapping = slot_mappings[kv_cache_gid]
if self.speculative_config and spec_decode_common_attn_metadata is None:
if isinstance(self.drafter, EagleProposer):
@@ -3119,6 +3115,80 @@ class GPUModelRunner(
pyt_hooks.register_hooks(self.model, self.model.__class__.__name__)
self.layerwise_nvtx_hooks_registered = True
def _get_slot_mappings(
self,
num_tokens_padded: int,
num_reqs_padded: int,
num_tokens_unpadded: int,
ubatch_slices: "UBatchSlices | None" = None,
) -> tuple[
dict[int, torch.Tensor] | None,
dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None,
]:
"""
Build slot mappings in both formats needed by the system.
Args:
num_tokens_padded: Total number of tokens (padded)
num_reqs_padded: Total number of requests (padded)
num_tokens_unpadded: Actual number of tokens (unpadded)
ubatch_slices: Optional ubatch slicing info for DBO
Returns:
A tuple of:
- slot_mappings_by_gid: dict[int, torch.Tensor] for attention metadata
- slot_mappings_by_layer: dict[str, torch.Tensor] or list for ForwardContext
"""
if not (
hasattr(self, "kv_cache_config")
and self.kv_cache_config is not None
and len(self.kv_cache_config.kv_cache_groups) > 0
):
return None, None
def _get_slot_mapping(kv_cache_gid: int):
assert num_reqs_padded is not None and num_tokens_padded is not None
kv_cache_spec = self.kv_cache_config.kv_cache_groups[
kv_cache_gid
].kv_cache_spec
if isinstance(kv_cache_spec, EncoderOnlyAttentionSpec):
slot_mapping = torch.zeros(
(num_tokens_padded,),
dtype=torch.int64,
device=self.device,
)
else:
blk_table = self.input_batch.block_table[kv_cache_gid]
slot_mapping = blk_table.slot_mapping.gpu[:num_tokens_padded]
# Fill unused with -1. Needed for reshape_and_cache in full cuda
# graph mode. `blk_table_tensor` -1 to match mamba PAD_SLOT_ID
slot_mapping[num_tokens_unpadded:num_tokens_padded].fill_(-1)
return slot_mapping
slot_mappings_by_gid = {
gid: _get_slot_mapping(gid)
for gid, _ in enumerate(self.kv_cache_config.kv_cache_groups)
}
slot_mappings_by_layer: dict[str, torch.Tensor] = {}
for gid, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups):
slot_mapping = slot_mappings_by_gid[gid]
for layer_name in kv_cache_group.layer_names:
slot_mappings_by_layer[layer_name] = slot_mapping
if ubatch_slices is not None:
result: list[dict[str, torch.Tensor]] = []
for ubatch in ubatch_slices:
sliced_mappings: dict[str, torch.Tensor] = {}
for layer_name, slot_mapping in slot_mappings_by_layer.items():
sliced_mappings[layer_name] = slot_mapping[ubatch.token_slice]
result.append(sliced_mappings)
return slot_mappings_by_gid, result
return slot_mappings_by_gid, slot_mappings_by_layer
@torch.inference_mode()
def execute_model(
self,
@@ -3248,6 +3318,17 @@ class GPUModelRunner(
ubatch_slices_padded,
)
# True if any attention backend handles KV cache update separately
# from forward() (i.e., forward_includes_kv_cache_update=False). When true,
# slot_mappings must use padded dimensions to match the key/value tensors.
has_separate_kv_update = not all(
all(
g.backend.forward_includes_kv_cache_update
for g in self.attn_groups[id]
)
for id, spec in enumerate(self.kv_cache_config.kv_cache_groups)
if not isinstance(spec.kv_cache_spec, EncoderOnlyAttentionSpec)
)
pad_attn = cudagraph_mode == CUDAGraphMode.FULL
if self.cache_config.mamba_cache_mode == "align":
@@ -3265,6 +3346,17 @@ class GPUModelRunner(
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
ubatch_slices_attn = ubatch_slices_padded if pad_attn else ubatch_slices
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
num_tokens_padded=num_tokens_padded
if pad_attn or has_separate_kv_update
else num_tokens_unpadded,
num_reqs_padded=(
num_reqs_padded if pad_attn or has_separate_kv_update else num_reqs
),
num_tokens_unpadded=num_tokens_unpadded,
ubatch_slices=ubatch_slices_padded,
)
attn_metadata, spec_decode_common_attn_metadata = (
self._build_attention_metadata(
num_tokens=num_tokens_unpadded,
@@ -3277,6 +3369,7 @@ class GPUModelRunner(
use_spec_decode=use_spec_decode,
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
slot_mappings=slot_mappings_by_group,
)
)
@@ -3317,6 +3410,7 @@ class GPUModelRunner(
cudagraph_runtime_mode=cudagraph_mode,
batch_descriptor=batch_desc,
ubatch_slices=ubatch_slices_padded,
slot_mapping=slot_mappings,
skip_compiled=has_encoder_input,
),
record_function_or_nullcontext("gpu_model_runner: forward"),
@@ -3399,6 +3493,7 @@ class GPUModelRunner(
aux_hidden_states,
ec_connector_output,
cudagraph_stats,
slot_mappings,
)
self.kv_connector_output = kv_connector_output
return None
@@ -3435,6 +3530,7 @@ class GPUModelRunner(
aux_hidden_states,
ec_connector_output,
cudagraph_stats,
slot_mappings,
) = self.execute_model_state
# Clear ephemeral state.
self.execute_model_state = None
@@ -3468,6 +3564,7 @@ class GPUModelRunner(
aux_hidden_states,
spec_decode_metadata,
spec_decode_common_attn_metadata,
slot_mappings,
)
self._copy_draft_token_ids_to_cpu(scheduler_output)
@@ -3676,6 +3773,7 @@ class GPUModelRunner(
aux_hidden_states: list[torch.Tensor] | None,
spec_decode_metadata: SpecDecodeMetadata | None,
common_attn_metadata: CommonAttentionMetadata,
slot_mappings: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None,
) -> list[list[int]] | torch.Tensor:
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
spec_config = self.speculative_config
@@ -3687,11 +3785,14 @@ class GPUModelRunner(
sampled_token_ids,
self.input_batch.num_tokens_no_spec,
self.input_batch.token_ids_cpu,
slot_mappings=slot_mappings,
)
elif spec_config.method == "suffix":
assert isinstance(sampled_token_ids, list)
assert isinstance(self.drafter, SuffixDecodingProposer)
draft_token_ids = self.drafter.propose(self.input_batch, sampled_token_ids)
draft_token_ids = self.drafter.propose(
self.input_batch, sampled_token_ids, slot_mappings=slot_mappings
)
elif spec_config.method == "medusa":
assert isinstance(sampled_token_ids, list)
assert isinstance(self.drafter, MedusaProposer)
@@ -3716,6 +3817,7 @@ class GPUModelRunner(
draft_token_ids = self.drafter.propose(
target_hidden_states=hidden_states,
sampling_metadata=sampling_metadata,
slot_mappings=slot_mappings,
)
elif spec_config.use_eagle() or spec_config.uses_draft_model():
assert isinstance(self.drafter, EagleProposer | DraftModelProposer)
@@ -3826,6 +3928,7 @@ class GPUModelRunner(
common_attn_metadata=common_attn_metadata,
mm_embed_inputs=mm_embed_inputs,
num_rejected_tokens_gpu=num_rejected_tokens_gpu,
slot_mappings=slot_mappings,
)
return draft_token_ids
@@ -4406,6 +4509,13 @@ class GPUModelRunner(
attn_metadata: PerLayerAttnMetadata | None = None
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
num_tokens_padded=num_tokens,
num_reqs_padded=num_reqs_padded,
num_tokens_unpadded=num_tokens_unpadded,
ubatch_slices=ubatch_slices_padded,
)
# If force_attention is True, we always capture attention. Otherwise,
# it only happens for cudagraph_runtime_mode=FULL.
if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL:
@@ -4431,6 +4541,7 @@ class GPUModelRunner(
max_query_len=max_query_len,
ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices,
for_cudagraph_capture=is_graph_capturing,
slot_mappings=slot_mappings_by_group,
)
with self.maybe_dummy_run_with_lora(
@@ -4499,6 +4610,7 @@ class GPUModelRunner(
cudagraph_runtime_mode=cudagraph_runtime_mode,
batch_descriptor=batch_desc,
ubatch_slices=ubatch_slices_padded,
slot_mapping=slot_mappings,
),
):
outputs = self.model(
@@ -4545,6 +4657,7 @@ class GPUModelRunner(
num_tokens,
use_cudagraphs=use_cudagraphs,
is_graph_capturing=is_graph_capturing,
slot_mappings=slot_mappings,
)
# We register layerwise NVTX hooks here after the first dynamo tracing is