[Bugfix][Mamba] Store conv_kernel - 1 inputs in Mamba2 prefill checkpoints (#60797)
Signed-off-by: Sungsoo Ha <sungsooh@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
tomeras91
parent
206e45d470
commit
c7a305ec10
@@ -114,23 +114,27 @@ def test_checkpoint_does_not_perturb_the_final_state():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spec_slots", [0, 3])
|
||||
@pytest.mark.parametrize("dim_first", [True, False])
|
||||
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
|
||||
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first, spec_slots):
|
||||
"""The store reads the checkpoint row out of every chunk state.
|
||||
|
||||
Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the
|
||||
program's own row, so the store only passes if ``chunk_idx`` is used.
|
||||
Row 1 declines and must write nothing. The conv input is a column slice of
|
||||
a wider projection, the SD conv layout is a transposed view, and SSM rows
|
||||
are padded pages, matching the strides the mixer passes.
|
||||
are padded pages, matching the strides the mixer passes. Spec decoding
|
||||
widens the conv state, and the store must still fill only the first
|
||||
``state_len`` slots with the newest inputs.
|
||||
"""
|
||||
dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8
|
||||
projection = torch.randn(64, dim + 8, device=DEVICE)
|
||||
conv_input = projection[:, :dim]
|
||||
num_slots = state_len + spec_slots
|
||||
if dim_first:
|
||||
conv_state = torch.zeros(3, dim, state_len, device=DEVICE)
|
||||
conv_state = torch.zeros(3, dim, num_slots, device=DEVICE)
|
||||
else:
|
||||
conv_state = torch.zeros(3, state_len, dim, device=DEVICE).transpose(-1, -2)
|
||||
conv_state = torch.zeros(3, num_slots, dim, device=DEVICE).transpose(-1, -2)
|
||||
varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE)
|
||||
# Blocks are pages holding more than the SSM state, so rows are padded.
|
||||
row = heads * head_dim * d_state
|
||||
@@ -152,12 +156,14 @@ def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE),
|
||||
state_len=state_len,
|
||||
)
|
||||
|
||||
exact = dict(rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
conv_state[2], conv_input[32 - state_len : 32].T, **exact
|
||||
conv_state[2, :, :state_len], conv_input[32 - state_len : 32].T, **exact
|
||||
)
|
||||
assert not conv_state[2, :, state_len:].any(), "wrote into the spec slots"
|
||||
torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact)
|
||||
assert not conv_state[:2].any() and not ssm_storage[:2].any()
|
||||
assert not ssm_storage[:, row:].any(), "wrote into the page padding"
|
||||
|
||||
@@ -890,6 +890,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
query_start_loc_p,
|
||||
state_len=self.conv_kernel_size - 1,
|
||||
)
|
||||
ssm_state[state_indices_tensor_p] = final_states
|
||||
if ring_start is not None and self._updates_replayssm_trackers:
|
||||
|
||||
@@ -69,6 +69,8 @@ def store_prefill_checkpoint(
|
||||
chunk_states: torch.Tensor,
|
||||
ssm_state: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
*,
|
||||
state_len: int,
|
||||
) -> None:
|
||||
"""Write each prefill row's mid-prefill checkpoint into its paged block.
|
||||
|
||||
@@ -78,14 +80,15 @@ def store_prefill_checkpoint(
|
||||
checkpoint: Per-row offsets and destination blocks.
|
||||
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
|
||||
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
|
||||
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
|
||||
conv_state: Paged conv state, ``(num_blocks, dim, state_len + num_spec)``.
|
||||
chunk_states: SSD states at every chunk end.
|
||||
ssm_state: Paged SSM state.
|
||||
query_start_loc: Prefill query start locations.
|
||||
state_len: ``conv_kernel_size - 1``, not the spec-widened conv width.
|
||||
Readers take the first ``state_len`` slots as the newest inputs.
|
||||
|
||||
"""
|
||||
dim = x.size(0)
|
||||
state_len = conv_state.size(-1)
|
||||
ssm_row_size = chunk_states[0].numel()
|
||||
block_size = 1024
|
||||
grid = (
|
||||
|
||||
Reference in New Issue
Block a user