[BugFix] Stop dummy runs from writing mamba state through stale block-table rows (#49757)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: Jeff Ma <jeffjma@umich.edu>
This commit is contained in:
@@ -144,6 +144,11 @@ class BlockTable:
|
||||
block_table_np = self.block_table.np
|
||||
block_table_np[tgt, :num_blocks] = block_table_np[src, :num_blocks]
|
||||
self.num_blocks_per_row[tgt] = num_blocks
|
||||
# Clear the vacated source row: dummy-run batches dereference stale
|
||||
# rows as mamba state slots and write state in place there, possibly
|
||||
# after the blocks have been freed and reallocated.
|
||||
block_table_np[src, :num_blocks] = 0
|
||||
self.num_blocks_per_row[src] = 0
|
||||
|
||||
def swap_row(self, src: int, tgt: int) -> None:
|
||||
src_tgt, tgt_src = [src, tgt], [tgt, src]
|
||||
|
||||
@@ -163,7 +163,13 @@ class BlockTables:
|
||||
# Therefore, this method must return the persistent tensor
|
||||
# with the same memory address as that used during the model's forward pass,
|
||||
# rather than allocating a new tensor.
|
||||
return tuple(block_table[:num_reqs] for block_table in self.input_block_tables)
|
||||
#
|
||||
# Zero the rows so dummy runs write mamba state to the reserved null
|
||||
# block rather than through the previous real step's (stale) block
|
||||
# ids, which may point at blocks since freed and reallocated.
|
||||
return tuple(
|
||||
block_table[:num_reqs].zero_() for block_table in self.input_block_tables
|
||||
)
|
||||
|
||||
def compute_slot_mappings(
|
||||
self,
|
||||
|
||||
@@ -1290,6 +1290,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
# FULL replay reads capture-time metadata buffers. Re-stage them
|
||||
# from the zeroed dummy block tables instead of retaining state
|
||||
# indices from the previous real batch.
|
||||
for_capture=dummy_run and batch_desc.cg_mode == CUDAGraphMode.FULL,
|
||||
)
|
||||
|
||||
input_ids = input_batch.input_ids
|
||||
|
||||
@@ -6064,7 +6064,13 @@ class GPUModelRunner(
|
||||
num_reqs=num_reqs_padded,
|
||||
max_query_len=max_query_len,
|
||||
ubatch_slices=(ubatch_slices_padded if pad_attn else ubatch_slices),
|
||||
for_cudagraph_capture=is_graph_capturing,
|
||||
# FULL replay reads capture-time metadata buffers. Re-stage them
|
||||
# from the zeroed dummy block tables instead of retaining state
|
||||
# indices from the previous real batch.
|
||||
for_cudagraph_capture=(
|
||||
is_graph_capturing
|
||||
or cudagraph_runtime_mode == CUDAGraphMode.FULL
|
||||
),
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
use_spec_decode=self.speculative_config is not None,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user