[fix] Remove trtllm ragged mla prefills (#36540)
Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
This commit is contained in:
@@ -1181,6 +1181,7 @@ class MLACommonPrefillMetadata:
|
||||
padded_local_cu_seq_lens: torch.Tensor | None = None
|
||||
cu_seq_lens_lst: list[list[int]] | None = None
|
||||
chunk_size: int | None = None
|
||||
prefill_tokens_with_context: int | None = None
|
||||
|
||||
block_table: torch.Tensor
|
||||
query_start_loc: torch.Tensor
|
||||
@@ -1743,6 +1744,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
prefill_query_start_loc = (
|
||||
query_start_loc[reqs_start:] - query_start_loc[reqs_start]
|
||||
)
|
||||
prefill_query_start_loc_cpu = (
|
||||
query_start_loc_cpu[reqs_start:] - query_start_loc_cpu[reqs_start]
|
||||
)
|
||||
|
||||
chunked_context_metadata = None
|
||||
if max_context_len_cpu > 0:
|
||||
@@ -1864,6 +1868,11 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
if self._use_cudnn_prefill
|
||||
else MLACommonPrefillMetadata.ChunkedContextMetadata
|
||||
)
|
||||
prefill_tokens_with_context = None
|
||||
if num_prefills_with_context_cpu > 0:
|
||||
prefill_tokens_with_context = prefill_query_start_loc_cpu[
|
||||
num_prefills_with_context_cpu
|
||||
].item()
|
||||
if self.dcp_world_size > 1:
|
||||
chunked_context_metadata = chunked_context_metadata_cls(
|
||||
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
||||
@@ -1883,6 +1892,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
),
|
||||
cu_seq_lens_lst=cu_seq_lens_cpu.tolist(),
|
||||
chunk_size=padded_local_max_context_chunk_across_ranks,
|
||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||
)
|
||||
else:
|
||||
chunked_context_metadata = chunked_context_metadata_cls(
|
||||
@@ -1896,6 +1906,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
),
|
||||
chunk_total_token=chunk_total_token,
|
||||
workspace=self.chunked_prefill_workspace,
|
||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||
)
|
||||
|
||||
if self._use_cudnn_prefill:
|
||||
@@ -2382,14 +2393,13 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
assert prefill.chunked_context.seq_lens[chunk_idx] is not None
|
||||
assert prefill.workspace_buffer is not None
|
||||
|
||||
out = torch.zeros(
|
||||
out = torch.empty(
|
||||
q.shape[0],
|
||||
q.shape[1],
|
||||
v.shape[2],
|
||||
device=q.device,
|
||||
dtype=prefill.output_dtype,
|
||||
)
|
||||
prefill.workspace_buffer.fill_(0)
|
||||
|
||||
attn_out, lse = trtllm_ragged_attention_deepseek(
|
||||
query=q,
|
||||
@@ -2691,6 +2701,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
)
|
||||
|
||||
if has_context:
|
||||
assert prefill_metadata.chunked_context is not None
|
||||
suffix_output, suffix_lse = output_prefill
|
||||
if self.dcp_world_size > 1:
|
||||
context_output, context_lse = (
|
||||
@@ -2719,6 +2730,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
prefix_lse=context_lse,
|
||||
suffix_output=suffix_output,
|
||||
suffix_lse=suffix_lse,
|
||||
prefill_tokens_with_context=prefill_metadata.chunked_context.prefill_tokens_with_context,
|
||||
)
|
||||
else:
|
||||
output_prefill = output_prefill[..., : v.shape[-1]].flatten(start_dim=-2)
|
||||
|
||||
Reference in New Issue
Block a user