[Bugfix] Fix DeepGEMM context lens contiguity in MLA indexer (#42135)
Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
@@ -288,20 +288,14 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
dtype=torch.int32,
|
dtype=torch.int32,
|
||||||
device=self.device,
|
device=self.device,
|
||||||
)
|
)
|
||||||
if not self.use_flattening and next_n > 1:
|
# Shared workspace for decode seq_lens. Native MTP views this as
|
||||||
# Native MTP: 2D buffer for per-token seq_lens.
|
# (B, max_decode_len) at runtime, keeping context_lens contiguous even
|
||||||
self.decode_seq_lens_buffer = torch.zeros(
|
# when max_decode_len is smaller than next_n.
|
||||||
(scheduler_config.max_num_seqs, next_n),
|
self.decode_seq_lens_buffer = torch.zeros(
|
||||||
dtype=torch.int32,
|
(scheduler_config.max_num_batched_tokens,),
|
||||||
device=self.device,
|
dtype=torch.int32,
|
||||||
)
|
device=self.device,
|
||||||
else:
|
)
|
||||||
# Flattening or no MTP: 1D buffer for expanded per-token seq_lens.
|
|
||||||
self.decode_seq_lens_buffer = torch.zeros(
|
|
||||||
(scheduler_config.max_num_batched_tokens,),
|
|
||||||
dtype=torch.int32,
|
|
||||||
device=self.device,
|
|
||||||
)
|
|
||||||
self.arange_buffer = torch.arange(
|
self.arange_buffer = torch.arange(
|
||||||
max(
|
max(
|
||||||
scheduler_config.max_num_seqs * next_n,
|
scheduler_config.max_num_seqs * next_n,
|
||||||
@@ -373,7 +367,8 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
Plain decode or spec-decode with 2D per-token context lengths.
|
Plain decode or spec-decode with 2D per-token context lengths.
|
||||||
|
|
||||||
Returns (seq_lens, block_table, decode_lens, batch_size, requires_padding).
|
Returns (seq_lens, block_table, decode_lens, batch_size, requires_padding).
|
||||||
seq_lens is 1D (batch_size,) for flatten/plain, 2D (B, next_n) for native MTP.
|
seq_lens is 1D (batch_size,) for flatten/plain, 2D (B, max_decode_len)
|
||||||
|
for native MTP.
|
||||||
"""
|
"""
|
||||||
min_decode_len = int(decode_lens_cpu.min().item())
|
min_decode_len = int(decode_lens_cpu.min().item())
|
||||||
if not use_native and max_decode_len > 1:
|
if not use_native and max_decode_len > 1:
|
||||||
@@ -454,16 +449,19 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
# (requires_padding) instead.
|
# (requires_padding) instead.
|
||||||
requires_padding = min_decode_len != max_decode_len
|
requires_padding = min_decode_len != max_decode_len
|
||||||
if use_native and next_n > 1:
|
if use_native and next_n > 1:
|
||||||
assert self.decode_seq_lens_buffer.dim() == 2
|
assert self.decode_seq_lens_buffer.dim() == 1
|
||||||
# (B, max_decode_len): token j attends to
|
# (B, max_decode_len): token j attends to
|
||||||
# L - max_decode_len + j + 1 KV tokens.
|
# L - max_decode_len + j + 1 KV tokens.
|
||||||
self.decode_seq_lens_buffer[:num_decodes, :max_decode_len] = (
|
seq_lens_buffer = self.decode_seq_lens_buffer[
|
||||||
|
: num_decodes * max_decode_len
|
||||||
|
].view(num_decodes, max_decode_len)
|
||||||
|
seq_lens_buffer[:] = (
|
||||||
seq_lens.unsqueeze(1)
|
seq_lens.unsqueeze(1)
|
||||||
- max_decode_len
|
- max_decode_len
|
||||||
+ 1
|
+ 1
|
||||||
+ self.offsets_buffer[:max_decode_len]
|
+ self.offsets_buffer[:max_decode_len]
|
||||||
)
|
)
|
||||||
seq_lens = self.decode_seq_lens_buffer[:num_decodes, :max_decode_len]
|
seq_lens = seq_lens_buffer
|
||||||
return seq_lens, block_table, decode_lens, num_decodes, requires_padding
|
return seq_lens, block_table, decode_lens, num_decodes, requires_padding
|
||||||
|
|
||||||
def build(
|
def build(
|
||||||
|
|||||||
Reference in New Issue
Block a user