[Performance] Split FlashAttn attention and cache update (#25954)
Signed-off-by: ElizaWszola <ewszola@redhat.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Luka Govedič <luka.govedic@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Signed-off-by: Luka Govedič <lgovedic@redhat.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Varun Sundar Rabindranath <varunsundar08@gmail.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Luka Govedič <luka.govedic@gmail.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Luka Govedič <lgovedic@redhat.com>
This commit is contained in:
co-authored by
mgoin
Varun Sundar Rabindranath
Matthew Bonanni
Luka Govedič
Luka Govedič
Lucas Wilkinson
Luka Govedič
parent
0118cdcc02
commit
a28b94e6ef
@@ -314,6 +314,7 @@ class ExecuteModelState(NamedTuple):
|
||||
aux_hidden_states: list[torch.Tensor] | None
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
cudagraph_stats: CUDAGraphStat | None
|
||||
slot_mappings: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None
|
||||
|
||||
|
||||
class GPUModelRunner(
|
||||
@@ -1595,6 +1596,7 @@ class GPUModelRunner(
|
||||
for_cudagraph_capture: bool = False,
|
||||
num_scheduled_tokens: dict[str, int] | None = None,
|
||||
cascade_attn_prefix_lens: list[list[int]] | None = None,
|
||||
slot_mappings: dict[int, torch.Tensor] | None = None,
|
||||
) -> tuple[PerLayerAttnMetadata, CommonAttentionMetadata | None]:
|
||||
"""
|
||||
:return: tuple[attn_metadata, spec_decode_common_attn_metadata]
|
||||
@@ -1628,7 +1630,7 @@ class GPUModelRunner(
|
||||
|
||||
kv_cache_groups = self.kv_cache_config.kv_cache_groups
|
||||
|
||||
def _get_block_table_and_slot_mapping(kv_cache_gid: int):
|
||||
def _get_block_table(kv_cache_gid: int):
|
||||
assert num_reqs_padded is not None and num_tokens_padded is not None
|
||||
kv_cache_spec = kv_cache_groups[kv_cache_gid].kv_cache_spec
|
||||
if isinstance(kv_cache_spec, EncoderOnlyAttentionSpec):
|
||||
@@ -1637,24 +1639,19 @@ class GPUModelRunner(
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
slot_mapping = torch.zeros(
|
||||
(num_tokens_padded,),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
blk_table = self.input_batch.block_table[kv_cache_gid]
|
||||
blk_table_tensor = blk_table.get_device_tensor(num_reqs_padded)
|
||||
slot_mapping = blk_table.slot_mapping.gpu[:num_tokens_padded]
|
||||
|
||||
# Fill unused with -1. Needed for reshape_and_cache in full cuda
|
||||
# graph mode. `blk_table_tensor` -1 to match mamba PAD_SLOT_ID
|
||||
slot_mapping[num_tokens:num_tokens_padded].fill_(-1)
|
||||
blk_table_tensor[num_reqs:num_reqs_padded].fill_(-1)
|
||||
return blk_table_tensor
|
||||
|
||||
return blk_table_tensor, slot_mapping
|
||||
assert slot_mappings is not None
|
||||
block_table_gid_0 = _get_block_table(0)
|
||||
slot_mapping_gid_0 = slot_mappings[0]
|
||||
|
||||
block_table_gid_0, slot_mapping_gid_0 = _get_block_table_and_slot_mapping(0)
|
||||
if self.model_config.enable_return_routed_experts:
|
||||
self.slot_mapping = slot_mapping_gid_0[:num_tokens].cpu().numpy()
|
||||
cm_base = CommonAttentionMetadata(
|
||||
@@ -1779,9 +1776,8 @@ class GPUModelRunner(
|
||||
num_reqs_padded,
|
||||
)
|
||||
if kv_cache_gid > 0:
|
||||
cm.block_table_tensor, cm.slot_mapping = (
|
||||
_get_block_table_and_slot_mapping(kv_cache_gid)
|
||||
)
|
||||
cm.block_table_tensor = _get_block_table(kv_cache_gid)
|
||||
cm.slot_mapping = slot_mappings[kv_cache_gid]
|
||||
|
||||
if self.speculative_config and spec_decode_common_attn_metadata is None:
|
||||
if isinstance(self.drafter, EagleProposer):
|
||||
@@ -3119,6 +3115,80 @@ class GPUModelRunner(
|
||||
pyt_hooks.register_hooks(self.model, self.model.__class__.__name__)
|
||||
self.layerwise_nvtx_hooks_registered = True
|
||||
|
||||
def _get_slot_mappings(
|
||||
self,
|
||||
num_tokens_padded: int,
|
||||
num_reqs_padded: int,
|
||||
num_tokens_unpadded: int,
|
||||
ubatch_slices: "UBatchSlices | None" = None,
|
||||
) -> tuple[
|
||||
dict[int, torch.Tensor] | None,
|
||||
dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None,
|
||||
]:
|
||||
"""
|
||||
Build slot mappings in both formats needed by the system.
|
||||
|
||||
Args:
|
||||
num_tokens_padded: Total number of tokens (padded)
|
||||
num_reqs_padded: Total number of requests (padded)
|
||||
num_tokens_unpadded: Actual number of tokens (unpadded)
|
||||
ubatch_slices: Optional ubatch slicing info for DBO
|
||||
|
||||
Returns:
|
||||
A tuple of:
|
||||
- slot_mappings_by_gid: dict[int, torch.Tensor] for attention metadata
|
||||
- slot_mappings_by_layer: dict[str, torch.Tensor] or list for ForwardContext
|
||||
"""
|
||||
if not (
|
||||
hasattr(self, "kv_cache_config")
|
||||
and self.kv_cache_config is not None
|
||||
and len(self.kv_cache_config.kv_cache_groups) > 0
|
||||
):
|
||||
return None, None
|
||||
|
||||
def _get_slot_mapping(kv_cache_gid: int):
|
||||
assert num_reqs_padded is not None and num_tokens_padded is not None
|
||||
kv_cache_spec = self.kv_cache_config.kv_cache_groups[
|
||||
kv_cache_gid
|
||||
].kv_cache_spec
|
||||
if isinstance(kv_cache_spec, EncoderOnlyAttentionSpec):
|
||||
slot_mapping = torch.zeros(
|
||||
(num_tokens_padded,),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
blk_table = self.input_batch.block_table[kv_cache_gid]
|
||||
slot_mapping = blk_table.slot_mapping.gpu[:num_tokens_padded]
|
||||
|
||||
# Fill unused with -1. Needed for reshape_and_cache in full cuda
|
||||
# graph mode. `blk_table_tensor` -1 to match mamba PAD_SLOT_ID
|
||||
slot_mapping[num_tokens_unpadded:num_tokens_padded].fill_(-1)
|
||||
|
||||
return slot_mapping
|
||||
|
||||
slot_mappings_by_gid = {
|
||||
gid: _get_slot_mapping(gid)
|
||||
for gid, _ in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||
}
|
||||
|
||||
slot_mappings_by_layer: dict[str, torch.Tensor] = {}
|
||||
for gid, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups):
|
||||
slot_mapping = slot_mappings_by_gid[gid]
|
||||
for layer_name in kv_cache_group.layer_names:
|
||||
slot_mappings_by_layer[layer_name] = slot_mapping
|
||||
|
||||
if ubatch_slices is not None:
|
||||
result: list[dict[str, torch.Tensor]] = []
|
||||
for ubatch in ubatch_slices:
|
||||
sliced_mappings: dict[str, torch.Tensor] = {}
|
||||
for layer_name, slot_mapping in slot_mappings_by_layer.items():
|
||||
sliced_mappings[layer_name] = slot_mapping[ubatch.token_slice]
|
||||
result.append(sliced_mappings)
|
||||
return slot_mappings_by_gid, result
|
||||
|
||||
return slot_mappings_by_gid, slot_mappings_by_layer
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
@@ -3248,6 +3318,17 @@ class GPUModelRunner(
|
||||
ubatch_slices_padded,
|
||||
)
|
||||
|
||||
# True if any attention backend handles KV cache update separately
|
||||
# from forward() (i.e., forward_includes_kv_cache_update=False). When true,
|
||||
# slot_mappings must use padded dimensions to match the key/value tensors.
|
||||
has_separate_kv_update = not all(
|
||||
all(
|
||||
g.backend.forward_includes_kv_cache_update
|
||||
for g in self.attn_groups[id]
|
||||
)
|
||||
for id, spec in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||
if not isinstance(spec.kv_cache_spec, EncoderOnlyAttentionSpec)
|
||||
)
|
||||
pad_attn = cudagraph_mode == CUDAGraphMode.FULL
|
||||
|
||||
if self.cache_config.mamba_cache_mode == "align":
|
||||
@@ -3265,6 +3346,17 @@ class GPUModelRunner(
|
||||
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
|
||||
ubatch_slices_attn = ubatch_slices_padded if pad_attn else ubatch_slices
|
||||
|
||||
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
|
||||
num_tokens_padded=num_tokens_padded
|
||||
if pad_attn or has_separate_kv_update
|
||||
else num_tokens_unpadded,
|
||||
num_reqs_padded=(
|
||||
num_reqs_padded if pad_attn or has_separate_kv_update else num_reqs
|
||||
),
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
)
|
||||
|
||||
attn_metadata, spec_decode_common_attn_metadata = (
|
||||
self._build_attention_metadata(
|
||||
num_tokens=num_tokens_unpadded,
|
||||
@@ -3277,6 +3369,7 @@ class GPUModelRunner(
|
||||
use_spec_decode=use_spec_decode,
|
||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3317,6 +3410,7 @@ class GPUModelRunner(
|
||||
cudagraph_runtime_mode=cudagraph_mode,
|
||||
batch_descriptor=batch_desc,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
slot_mapping=slot_mappings,
|
||||
skip_compiled=has_encoder_input,
|
||||
),
|
||||
record_function_or_nullcontext("gpu_model_runner: forward"),
|
||||
@@ -3399,6 +3493,7 @@ class GPUModelRunner(
|
||||
aux_hidden_states,
|
||||
ec_connector_output,
|
||||
cudagraph_stats,
|
||||
slot_mappings,
|
||||
)
|
||||
self.kv_connector_output = kv_connector_output
|
||||
return None
|
||||
@@ -3435,6 +3530,7 @@ class GPUModelRunner(
|
||||
aux_hidden_states,
|
||||
ec_connector_output,
|
||||
cudagraph_stats,
|
||||
slot_mappings,
|
||||
) = self.execute_model_state
|
||||
# Clear ephemeral state.
|
||||
self.execute_model_state = None
|
||||
@@ -3468,6 +3564,7 @@ class GPUModelRunner(
|
||||
aux_hidden_states,
|
||||
spec_decode_metadata,
|
||||
spec_decode_common_attn_metadata,
|
||||
slot_mappings,
|
||||
)
|
||||
self._copy_draft_token_ids_to_cpu(scheduler_output)
|
||||
|
||||
@@ -3676,6 +3773,7 @@ class GPUModelRunner(
|
||||
aux_hidden_states: list[torch.Tensor] | None,
|
||||
spec_decode_metadata: SpecDecodeMetadata | None,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
slot_mappings: dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None,
|
||||
) -> list[list[int]] | torch.Tensor:
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
spec_config = self.speculative_config
|
||||
@@ -3687,11 +3785,14 @@ class GPUModelRunner(
|
||||
sampled_token_ids,
|
||||
self.input_batch.num_tokens_no_spec,
|
||||
self.input_batch.token_ids_cpu,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
elif spec_config.method == "suffix":
|
||||
assert isinstance(sampled_token_ids, list)
|
||||
assert isinstance(self.drafter, SuffixDecodingProposer)
|
||||
draft_token_ids = self.drafter.propose(self.input_batch, sampled_token_ids)
|
||||
draft_token_ids = self.drafter.propose(
|
||||
self.input_batch, sampled_token_ids, slot_mappings=slot_mappings
|
||||
)
|
||||
elif spec_config.method == "medusa":
|
||||
assert isinstance(sampled_token_ids, list)
|
||||
assert isinstance(self.drafter, MedusaProposer)
|
||||
@@ -3716,6 +3817,7 @@ class GPUModelRunner(
|
||||
draft_token_ids = self.drafter.propose(
|
||||
target_hidden_states=hidden_states,
|
||||
sampling_metadata=sampling_metadata,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
elif spec_config.use_eagle() or spec_config.uses_draft_model():
|
||||
assert isinstance(self.drafter, EagleProposer | DraftModelProposer)
|
||||
@@ -3826,6 +3928,7 @@ class GPUModelRunner(
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
mm_embed_inputs=mm_embed_inputs,
|
||||
num_rejected_tokens_gpu=num_rejected_tokens_gpu,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
|
||||
return draft_token_ids
|
||||
@@ -4406,6 +4509,13 @@ class GPUModelRunner(
|
||||
|
||||
attn_metadata: PerLayerAttnMetadata | None = None
|
||||
|
||||
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
|
||||
num_tokens_padded=num_tokens,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
)
|
||||
|
||||
# If force_attention is True, we always capture attention. Otherwise,
|
||||
# it only happens for cudagraph_runtime_mode=FULL.
|
||||
if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL:
|
||||
@@ -4431,6 +4541,7 @@ class GPUModelRunner(
|
||||
max_query_len=max_query_len,
|
||||
ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices,
|
||||
for_cudagraph_capture=is_graph_capturing,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
)
|
||||
|
||||
with self.maybe_dummy_run_with_lora(
|
||||
@@ -4499,6 +4610,7 @@ class GPUModelRunner(
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
batch_descriptor=batch_desc,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
slot_mapping=slot_mappings,
|
||||
),
|
||||
):
|
||||
outputs = self.model(
|
||||
@@ -4545,6 +4657,7 @@ class GPUModelRunner(
|
||||
num_tokens,
|
||||
use_cudagraphs=use_cudagraphs,
|
||||
is_graph_capturing=is_graph_capturing,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
|
||||
# We register layerwise NVTX hooks here after the first dynamo tracing is
|
||||
|
||||
Reference in New Issue
Block a user