[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)
Signed-off-by: Johnny-Liou <a897111@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude Opus 4.8
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
tomeras91
Cyrus Leung
parent
d02df748bf
commit
866fea2b99
@@ -0,0 +1,244 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Prefill == decode equivalence for the Mamba2 ReplaySSM kernels.
|
||||
|
||||
The SSM recurrence is path-independent: vLLM's chunked prefill (the SSD kernel
|
||||
``mamba_chunk_scan_combined_varlen``) and step-by-step decode must produce the
|
||||
same per-position outputs and final state over a sequence. This file feeds one
|
||||
set of inputs through the production dt flow (raw dt + softplus + a per-head
|
||||
dt_bias, applied inside each kernel) to:
|
||||
|
||||
* the exact fp32 step recurrence (``selective_state_update_ref``) -- the
|
||||
ground truth,
|
||||
* the chunked prefill kernel,
|
||||
* the baseline decode kernel,
|
||||
* the ReplaySSM output_only decode kernel,
|
||||
|
||||
and checks all of them agree. Prefill (a chunked scan) and decode (a step
|
||||
recurrence) are different code paths, so they differ numerically: the chunked
|
||||
scan carries ~2e-2 (fp32) / ~4e-2 (bf16) vs the exact recurrence, far above the
|
||||
near-exact decode. We therefore anchor every path on the exact recurrence at
|
||||
SSD-level tolerances (the same regime as ``test_mamba_ssm_ssd.py``), which the
|
||||
chunked scan sets, keyed off the activation dtype.
|
||||
|
||||
State and activation/buffer precision are swept independently, including the
|
||||
fp32-state + bf16-activation production config.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.mamba.utils import selective_state_update_ref
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_state_update
|
||||
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
|
||||
selective_state_update_replayssm_output_only,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata
|
||||
|
||||
|
||||
def _prefill_tolerances(act_dtype: torch.dtype) -> tuple[float, float]:
|
||||
# The chunked prefill scan, not the decode, sets these: it carries ~2e-2
|
||||
# (fp32) / ~6e-2 (bf16) vs the exact recurrence, while ReplaySSM decode is
|
||||
# near-exact (~2e-6 fp32). Keyed off the activation dtype (the outputs are
|
||||
# in act_dtype). Same regime as test_mamba_ssm_ssd.py.
|
||||
if act_dtype == torch.float32:
|
||||
return 1e-2, 3e-2
|
||||
return 6e-2, 1e-1
|
||||
|
||||
|
||||
def _run_prefill_decode_equivalence(
|
||||
*,
|
||||
state_dtype: torch.dtype,
|
||||
act_dtype: torch.dtype,
|
||||
nheads: int,
|
||||
headdim: int,
|
||||
ngroups: int,
|
||||
dstate: int,
|
||||
seqlen: int,
|
||||
chunk_size: int,
|
||||
max_cache_len: int,
|
||||
seed: int = 0,
|
||||
) -> None:
|
||||
"""Prefill the whole sequence and decode it step by step; check both match
|
||||
the exact fp32 recurrence (and each other). All paths use the production dt
|
||||
flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill
|
||||
and decode apply the softplus/bias preprocessing consistently. ``state_dtype``
|
||||
is the recurrent-state precision; ``act_dtype`` the activation/buffer one."""
|
||||
device = "cuda"
|
||||
rtol, atol = _prefill_tolerances(act_dtype)
|
||||
set_random_seed(seed)
|
||||
|
||||
# Production dt flow: raw dt + a per-head dt_bias (~-4 keeps softplus(dt +
|
||||
# bias) small and well-conditioned). dt_bias is (nheads,) for prefill and
|
||||
# (nheads, headdim) for the decode kernels/reference.
|
||||
A = -torch.exp(torch.rand(nheads, device=device, dtype=act_dtype))
|
||||
dt = torch.randn(seqlen, nheads, device=device, dtype=act_dtype)
|
||||
dt_bias = torch.rand(nheads, device=device, dtype=act_dtype) - 4
|
||||
dt_bias_hd = dt_bias.view(nheads, 1).expand(nheads, headdim)
|
||||
X = torch.randn(seqlen, nheads, headdim, device=device, dtype=act_dtype)
|
||||
B = torch.randn(seqlen, ngroups, dstate, device=device, dtype=act_dtype)
|
||||
C = torch.randn(seqlen, ngroups, dstate, device=device, dtype=act_dtype)
|
||||
A_bcast = A.view(nheads, 1, 1).expand(nheads, headdim, dstate)
|
||||
|
||||
# Chunked prefill over the whole sequence (implicit batch=1, varlen). The
|
||||
# kernel always returns the final state in fp32, so no state_dtype plumbing.
|
||||
cu_seqlens = torch.tensor((0, seqlen), device=device).cumsum(0).to(torch.int32)
|
||||
cu_chunk_seqlens, last_chunk_indices, seq_idx = compute_varlen_chunk_metadata(
|
||||
cu_seqlens, chunk_size
|
||||
)
|
||||
y_prefill = torch.empty(seqlen, nheads, headdim, device=device, dtype=act_dtype)
|
||||
final_state_prefill = mamba_chunk_scan_combined_varlen(
|
||||
X,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
chunk_size,
|
||||
cu_seqlens=cu_seqlens,
|
||||
cu_chunk_seqlens=cu_chunk_seqlens,
|
||||
last_chunk_indices=last_chunk_indices,
|
||||
seq_idx=seq_idx,
|
||||
out=y_prefill,
|
||||
D=None,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=True,
|
||||
)
|
||||
|
||||
# Step paths: exact fp32 recurrence (ground truth), baseline, ReplaySSM.
|
||||
# State follows state_dtype; caches follow act_dtype (dt_cache is fp32).
|
||||
state_ref = torch.zeros(
|
||||
1, nheads, headdim, dstate, device=device, dtype=torch.float32
|
||||
)
|
||||
state_base = torch.zeros(
|
||||
1, nheads, headdim, dstate, device=device, dtype=state_dtype
|
||||
)
|
||||
state_dec = torch.zeros(
|
||||
1, nheads, headdim, dstate, device=device, dtype=state_dtype
|
||||
)
|
||||
x_cache = torch.zeros(
|
||||
1, nheads, max_cache_len, headdim, device=device, dtype=act_dtype
|
||||
)
|
||||
dt_cache = torch.zeros(1, nheads, max_cache_len, device=device, dtype=torch.float32)
|
||||
B_cache = torch.zeros(
|
||||
1, ngroups, max_cache_len, dstate, device=device, dtype=act_dtype
|
||||
)
|
||||
bc_pre = torch.empty(1, ngroups, max_cache_len, device=device, dtype=torch.float32)
|
||||
write_pos = torch.zeros(1, dtype=torch.int32, device=device)
|
||||
# No skip connection (D=0) on any path here; the D!=0 path is covered by the
|
||||
# standard-decode suite. The baseline kernel needs a D tensor, not None.
|
||||
D_zero = torch.zeros(nheads, headdim, device=device)
|
||||
|
||||
y_ref = torch.empty(seqlen, nheads, headdim, device=device, dtype=torch.float32)
|
||||
y_base = torch.empty(seqlen, nheads, headdim, device=device, dtype=act_dtype)
|
||||
y_dec = torch.empty(seqlen, nheads, headdim, device=device, dtype=act_dtype)
|
||||
for t in range(seqlen):
|
||||
dt_t = dt[t].view(1, nheads, 1).expand(1, nheads, headdim)
|
||||
is_flush = write_pos == max_cache_len - 1
|
||||
|
||||
y_ref[t] = selective_state_update_ref(
|
||||
state_ref,
|
||||
X[t : t + 1].float(),
|
||||
dt_t.float(),
|
||||
A_bcast.float(),
|
||||
B[t : t + 1].float(),
|
||||
C[t : t + 1].float(),
|
||||
dt_bias=dt_bias_hd.float(),
|
||||
dt_softplus=True,
|
||||
)[0]
|
||||
|
||||
out_b = torch.empty(1, nheads, headdim, device=device, dtype=act_dtype)
|
||||
selective_state_update(
|
||||
state_base,
|
||||
X[t : t + 1],
|
||||
dt_t,
|
||||
A_bcast,
|
||||
B[t : t + 1],
|
||||
C[t : t + 1],
|
||||
D=D_zero,
|
||||
dt_bias=dt_bias_hd,
|
||||
dt_softplus=True,
|
||||
out=out_b,
|
||||
)
|
||||
y_base[t] = out_b[0]
|
||||
|
||||
out_d = torch.empty(1, nheads, headdim, device=device, dtype=act_dtype)
|
||||
common = dict(
|
||||
dt_bias=dt_bias_hd,
|
||||
dt_softplus=True,
|
||||
x_cache=x_cache,
|
||||
dt_cache=dt_cache,
|
||||
B_cache=B_cache,
|
||||
write_pos=write_pos,
|
||||
is_flush=is_flush,
|
||||
max_cache_len=max_cache_len,
|
||||
out=out_d,
|
||||
)
|
||||
selective_state_update_replayssm_output_only(
|
||||
state_dec,
|
||||
X[t : t + 1],
|
||||
dt_t,
|
||||
A_bcast,
|
||||
B[t : t + 1],
|
||||
C[t : t + 1],
|
||||
bc_pre=bc_pre,
|
||||
**common,
|
||||
)
|
||||
y_dec[t] = out_d[0]
|
||||
|
||||
write_pos = torch.where(is_flush, torch.zeros_like(write_pos), write_pos + 1)
|
||||
|
||||
# Every path computes the same recurrence; anchor each on the fp32 truth.
|
||||
torch.testing.assert_close(y_prefill.float(), y_ref, rtol=rtol, atol=atol)
|
||||
torch.testing.assert_close(y_base.float(), y_ref, rtol=rtol, atol=atol)
|
||||
torch.testing.assert_close(y_dec.float(), y_ref, rtol=rtol, atol=atol)
|
||||
# Headline: ReplaySSM decode matches the chunked prefill directly.
|
||||
torch.testing.assert_close(y_dec.float(), y_prefill.float(), rtol=rtol, atol=atol)
|
||||
# Final state too (the recurrence ends in state_ref after the loop).
|
||||
torch.testing.assert_close(
|
||||
final_state_prefill[0].float(), state_ref[0], rtol=rtol, atol=atol
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
|
||||
@pytest.mark.parametrize(
|
||||
"precision",
|
||||
# fp32 state is the default; bf16/fp16 are reduced-footprint configs. fp16
|
||||
# appears as an activation dtype (s32_afp16, sfp16_afp16) and as a finer-
|
||||
# mantissa state under bf16 activations (sfp16_a16).
|
||||
[
|
||||
pytest.param((torch.float32, torch.float32), id="s32_a32"),
|
||||
pytest.param((torch.float32, torch.bfloat16), id="s32_a16"),
|
||||
pytest.param((torch.bfloat16, torch.bfloat16), id="s16_a16"),
|
||||
pytest.param((torch.float32, torch.float16), id="s32_afp16"),
|
||||
pytest.param((torch.float16, torch.float16), id="sfp16_afp16"),
|
||||
pytest.param((torch.float16, torch.bfloat16), id="sfp16_a16"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"geometry", # (nheads, headdim, dstate, ngroups)
|
||||
[
|
||||
pytest.param((8, 64, 64, 2), id="small"),
|
||||
pytest.param((96, 80, 128, 8), id="nano4b"),
|
||||
],
|
||||
)
|
||||
def test_replayssm_prefill_decode_equivalence(
|
||||
precision: tuple[torch.dtype, torch.dtype],
|
||||
geometry: tuple[int, int, int, int],
|
||||
):
|
||||
state_dtype, act_dtype = precision
|
||||
nheads, headdim, dstate, ngroups = geometry
|
||||
_run_prefill_decode_equivalence(
|
||||
state_dtype=state_dtype,
|
||||
act_dtype=act_dtype,
|
||||
nheads=nheads,
|
||||
headdim=headdim,
|
||||
ngroups=ngroups,
|
||||
dstate=dstate,
|
||||
seqlen=16,
|
||||
chunk_size=8,
|
||||
max_cache_len=4,
|
||||
)
|
||||
@@ -0,0 +1,673 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Standard (autoregressive) decode correctness for the Mamba2 ReplaySSM kernels.
|
||||
|
||||
ReplaySSM caches the recent SSM inputs ``(x, dt, B)`` in a small ring buffer and
|
||||
reconstructs / reads out the recurrent state on the fly, writing the full state
|
||||
back to HBM only when the buffer flushes. This file checks that, over a
|
||||
multi-step decode, the ReplaySSM output_only kernel reproduces the exact SSM
|
||||
recurrence.
|
||||
|
||||
For every step we assert, against trusted oracles driven one token at a time:
|
||||
|
||||
* the ReplaySSM output matches its pure-PyTorch reference, which models the
|
||||
kernel's exact arithmetic (including its bf16 reconstruction), at every
|
||||
precision,
|
||||
* when the state is fp32, the output also matches the baseline decode kernel
|
||||
(``selective_state_update``); at bf16 state the baseline legitimately
|
||||
differs -- it downcasts the fp32 state to bf16 every step, while ReplaySSM
|
||||
accumulates a whole buffer in fp32 and is the more accurate path,
|
||||
* when state and activations are both fp32, the output also matches the exact
|
||||
elementwise ``selective_state_update_ref``,
|
||||
* the cached inputs match the reference cache management,
|
||||
* the checkpoint state matches the reference (and the baseline at fp32 state).
|
||||
|
||||
State and activation/buffer precision are swept independently. Nemotron-3
|
||||
defaults to fp32 SSM state (``mamba_ssm_cache_dtype=float32``), but bf16 state is
|
||||
also supported; the buffer dtype follows the activation dtype and ``dt_cache``
|
||||
is always fp32.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.mamba.utils import (
|
||||
allocate_update_caches,
|
||||
selective_state_update_ref,
|
||||
selective_state_update_replayssm_output_only_ref,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_state_update
|
||||
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
|
||||
selective_state_update_replayssm_output_only,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def _tolerances(dtype: torch.dtype) -> tuple[float, float]:
|
||||
# fp32 demands true fp32 parity. A correct fp32 reconstruction (tl.dot with
|
||||
# input_precision="tf32x3"/"ieee") matches the elementwise baseline to
|
||||
# ~1e-5, while a TF32 reconstruction drifts to ~1e-2. atol=1e-3 sits between,
|
||||
# so it flags TF32 degradation yet passes a correct fp32 kernel. bf16 stays
|
||||
# loose (bf16 rounding dominates and is unaffected by the matmul precision).
|
||||
if dtype == torch.float32:
|
||||
return 1e-4, 1e-3
|
||||
return 6e-2, 2e-1
|
||||
|
||||
|
||||
def _tied_A(nheads: int, headdim: int, dstate: int, device: str) -> torch.Tensor:
|
||||
A = -torch.rand(nheads, device=device) - 1.0
|
||||
return A.view(nheads, 1, 1).expand(nheads, headdim, dstate)
|
||||
|
||||
|
||||
def _tied_dt(
|
||||
batch: int,
|
||||
nheads: int,
|
||||
headdim: int,
|
||||
device: str,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
dt = torch.randn(batch, nheads, device=device, dtype=dtype)
|
||||
return dt.unsqueeze(-1).expand(batch, nheads, headdim)
|
||||
|
||||
|
||||
def _tied_dt_bias(nheads: int, headdim: int, device: str) -> torch.Tensor:
|
||||
dt_bias = torch.rand(nheads, device=device) - 4.0
|
||||
return dt_bias.view(nheads, 1).expand(nheads, headdim)
|
||||
|
||||
|
||||
def _run_standard_decode(
|
||||
*,
|
||||
state_dtype: torch.dtype,
|
||||
act_dtype: torch.dtype,
|
||||
batch: int,
|
||||
nheads: int,
|
||||
headdim: int,
|
||||
ngroups: int,
|
||||
dstate: int,
|
||||
max_cache_len: int,
|
||||
num_steps: int,
|
||||
has_z: bool,
|
||||
dt_softplus: bool,
|
||||
use_dt_bias: bool,
|
||||
desync_write_pos: bool = False,
|
||||
seed: int = 0,
|
||||
) -> None:
|
||||
"""Drive ``num_steps`` decode steps and check every path against the
|
||||
trusted recurrence. ``state_dtype`` is the recurrent-state precision;
|
||||
``act_dtype`` is the activation/buffer precision."""
|
||||
device = "cuda"
|
||||
both_fp32 = state_dtype == torch.float32 and act_dtype == torch.float32
|
||||
rtol, atol = _tolerances(torch.float32 if both_fp32 else torch.bfloat16)
|
||||
set_random_seed(seed)
|
||||
|
||||
# One state copy per path; all start identical at the same precision.
|
||||
state0 = torch.randn(
|
||||
batch, nheads, headdim, dstate, dtype=state_dtype, device=device
|
||||
)
|
||||
state_anchor = state0.clone()
|
||||
state_baseline = state0.clone()
|
||||
state_cached = state0.clone()
|
||||
state_ref = state0.clone()
|
||||
|
||||
A = _tied_A(nheads, headdim, dstate, device)
|
||||
dt_bias = _tied_dt_bias(nheads, headdim, device) if use_dt_bias else None
|
||||
D = torch.randn(nheads, headdim, device=device)
|
||||
|
||||
# Caches follow the activation dtype (dt_cache is forced to fp32 inside).
|
||||
x_cache, dt_cache, B_cache, _ = allocate_update_caches(
|
||||
batch,
|
||||
nheads,
|
||||
ngroups,
|
||||
headdim,
|
||||
dstate,
|
||||
max_cache_len,
|
||||
device,
|
||||
act_dtype,
|
||||
act_dtype,
|
||||
)
|
||||
x_cache_ref, dt_cache_ref, B_cache_ref, _ = allocate_update_caches(
|
||||
batch,
|
||||
nheads,
|
||||
ngroups,
|
||||
headdim,
|
||||
dstate,
|
||||
max_cache_len,
|
||||
device,
|
||||
act_dtype,
|
||||
act_dtype,
|
||||
)
|
||||
bc_pre = torch.empty(
|
||||
batch, ngroups, max_cache_len, device=device, dtype=torch.float32
|
||||
)
|
||||
|
||||
if desync_write_pos:
|
||||
# Rows start at different ring positions so they flush on different
|
||||
# steps, exercising per-row write-position handling.
|
||||
write_pos = (
|
||||
torch.arange(batch, device=device, dtype=torch.int32) % max_cache_len
|
||||
)
|
||||
else:
|
||||
write_pos = torch.zeros(batch, dtype=torch.int32, device=device)
|
||||
|
||||
for _ in range(num_steps):
|
||||
x = torch.randn(batch, nheads, headdim, device=device, dtype=act_dtype)
|
||||
dt = _tied_dt(batch, nheads, headdim, device, act_dtype)
|
||||
B = torch.randn(batch, ngroups, dstate, device=device, dtype=act_dtype)
|
||||
C = torch.randn(batch, ngroups, dstate, device=device, dtype=act_dtype)
|
||||
z = torch.randn_like(x) if has_z else None
|
||||
is_flush = write_pos == max_cache_len - 1
|
||||
|
||||
# Trusted recurrence (mutates state_anchor in place, returns output).
|
||||
out_anchor = selective_state_update_ref(
|
||||
state_anchor,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D=D,
|
||||
z=z,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=dt_softplus,
|
||||
)
|
||||
|
||||
# Upstream baseline decode kernel.
|
||||
out_baseline = torch.empty_like(x)
|
||||
selective_state_update(
|
||||
state_baseline,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D=D,
|
||||
z=z,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=dt_softplus,
|
||||
out=out_baseline,
|
||||
)
|
||||
|
||||
# ReplaySSM kernel under test + its pure-PyTorch reference.
|
||||
out_cached = torch.empty_like(x)
|
||||
common = dict(
|
||||
D=D,
|
||||
z=z,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=dt_softplus,
|
||||
x_cache=x_cache,
|
||||
dt_cache=dt_cache,
|
||||
B_cache=B_cache,
|
||||
write_pos=write_pos,
|
||||
is_flush=is_flush,
|
||||
max_cache_len=max_cache_len,
|
||||
out=out_cached,
|
||||
)
|
||||
selective_state_update_replayssm_output_only(
|
||||
state_cached, x, dt, A, B, C, bc_pre=bc_pre, **common
|
||||
)
|
||||
out_ref = selective_state_update_replayssm_output_only_ref(
|
||||
state_ref,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D=D,
|
||||
z=z,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=dt_softplus,
|
||||
x_cache=x_cache_ref,
|
||||
dt_cache=dt_cache_ref,
|
||||
B_cache=B_cache_ref,
|
||||
write_pos=write_pos,
|
||||
max_cache_len=max_cache_len,
|
||||
)
|
||||
|
||||
# The reference models the kernel's exact arithmetic (including its bf16
|
||||
# reconstruction), so the kernel must match it tightly at every
|
||||
# precision. At fp32 this also flags any TF32 reconstruction drift.
|
||||
torch.testing.assert_close(out_cached, out_ref, rtol=rtol, atol=atol)
|
||||
# When the STATE is fp32 the baseline decode kernel is a valid oracle (it
|
||||
# does not downcast the state per step), so ReplaySSM must match it. At
|
||||
# bf16 state the baseline legitimately differs: it downcasts the fp32
|
||||
# state to bf16 every step, while ReplaySSM accumulates a whole buffer in
|
||||
# fp32 and is the MORE accurate path -- so it is not a tight oracle there.
|
||||
if state_dtype == torch.float32:
|
||||
torch.testing.assert_close(out_cached, out_baseline, rtol=rtol, atol=atol)
|
||||
# The exact elementwise reference is valid only when state AND
|
||||
# activations are fp32; otherwise it downcasts the state (at readout for
|
||||
# bf16 activations, or per step for bf16 state).
|
||||
if both_fp32:
|
||||
torch.testing.assert_close(out_cached, out_anchor, rtol=rtol, atol=atol)
|
||||
|
||||
# Cached inputs match the reference cache management.
|
||||
torch.testing.assert_close(x_cache, x_cache_ref, rtol=rtol, atol=atol)
|
||||
torch.testing.assert_close(dt_cache, dt_cache_ref, rtol=rtol, atol=atol)
|
||||
torch.testing.assert_close(B_cache, B_cache_ref, rtol=rtol, atol=atol)
|
||||
|
||||
# Checkpoint state at flush matches the reference (and, when the state is
|
||||
# fp32, the baseline kernel).
|
||||
if bool(is_flush.any()):
|
||||
torch.testing.assert_close(
|
||||
state_cached[is_flush], state_ref[is_flush], rtol=rtol, atol=atol
|
||||
)
|
||||
if state_dtype == torch.float32:
|
||||
torch.testing.assert_close(
|
||||
state_cached[is_flush],
|
||||
state_baseline[is_flush],
|
||||
rtol=rtol,
|
||||
atol=atol,
|
||||
)
|
||||
|
||||
write_pos = torch.where(is_flush, torch.zeros_like(write_pos), write_pos + 1)
|
||||
|
||||
|
||||
# State/activation precisions. fp32 state is the default; bf16/fp16 are the
|
||||
# reduced-footprint configs. fp16 appears both as an activation dtype (fully-fp16
|
||||
# model sfp16_afp16, or fp16 act over fp32 state s32_afp16) and as a state dtype
|
||||
# under bf16 activations (sfp16_a16): fp16 has a finer mantissa than bf16 at the
|
||||
# same 2 bytes, so it is a more accurate state at no extra footprint. We still
|
||||
# skip fp16 state under fp32 activations (the unused low-state/high-act mix).
|
||||
_PRECISIONS = [
|
||||
pytest.param((torch.float32, torch.float32), id="s32_a32"),
|
||||
pytest.param((torch.float32, torch.bfloat16), id="s32_a16"),
|
||||
pytest.param((torch.bfloat16, torch.bfloat16), id="s16_a16"),
|
||||
pytest.param((torch.float32, torch.float16), id="s32_afp16"),
|
||||
pytest.param((torch.float16, torch.float16), id="sfp16_afp16"),
|
||||
pytest.param((torch.float16, torch.bfloat16), id="sfp16_a16"),
|
||||
]
|
||||
# Small synthetic shapes for the full axis sweep (compile fast).
|
||||
_SMALL_GEOMETRIES = [
|
||||
pytest.param((8, 64, 64, 4), id="small"),
|
||||
pytest.param((4, 64, 16, 1), id="tiny"),
|
||||
]
|
||||
# Production Mamba2 shapes (nheads, headdim, dstate, ngroups), TP=1.
|
||||
_REAL_GEOMETRIES = [
|
||||
pytest.param((96, 80, 128, 8), id="nano4b"),
|
||||
pytest.param((128, 64, 128, 8), id="super120b"),
|
||||
pytest.param((256, 64, 128, 8), id="ultra550b"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
|
||||
@pytest.mark.parametrize("precision", _PRECISIONS)
|
||||
@pytest.mark.parametrize("max_cache_len", [1, 4, 16])
|
||||
@pytest.mark.parametrize("geometry", _SMALL_GEOMETRIES)
|
||||
@pytest.mark.parametrize("has_z", [False, True])
|
||||
def test_replayssm_standard_decode_matches_reference(
|
||||
precision: tuple[torch.dtype, torch.dtype],
|
||||
max_cache_len: int,
|
||||
geometry: tuple[int, int, int, int],
|
||||
has_z: bool,
|
||||
):
|
||||
state_dtype, act_dtype = precision
|
||||
nheads, headdim, dstate, ngroups = geometry
|
||||
_run_standard_decode(
|
||||
state_dtype=state_dtype,
|
||||
act_dtype=act_dtype,
|
||||
batch=4,
|
||||
nheads=nheads,
|
||||
headdim=headdim,
|
||||
ngroups=ngroups,
|
||||
dstate=dstate,
|
||||
max_cache_len=max_cache_len,
|
||||
num_steps=2 * max_cache_len + 1,
|
||||
has_z=has_z,
|
||||
dt_softplus=True,
|
||||
use_dt_bias=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
|
||||
@pytest.mark.parametrize("precision", _PRECISIONS)
|
||||
@pytest.mark.parametrize("geometry", _REAL_GEOMETRIES)
|
||||
def test_replayssm_standard_decode_real_geometry(
|
||||
precision: tuple[torch.dtype, torch.dtype],
|
||||
geometry: tuple[int, int, int, int],
|
||||
):
|
||||
# Production Mamba2 shapes for the Nemotron-3 family (Nano-4B / Super-120B /
|
||||
# Ultra-550B), at the production buffer length (8). All three precisions,
|
||||
# including the bf16 state case.
|
||||
state_dtype, act_dtype = precision
|
||||
nheads, headdim, dstate, ngroups = geometry
|
||||
_run_standard_decode(
|
||||
state_dtype=state_dtype,
|
||||
act_dtype=act_dtype,
|
||||
batch=4,
|
||||
nheads=nheads,
|
||||
headdim=headdim,
|
||||
ngroups=ngroups,
|
||||
dstate=dstate,
|
||||
max_cache_len=8,
|
||||
num_steps=17,
|
||||
has_z=True,
|
||||
dt_softplus=True,
|
||||
use_dt_bias=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
|
||||
@pytest.mark.parametrize(
|
||||
"precision",
|
||||
[
|
||||
pytest.param((torch.float32, torch.float32), id="s32_a32"),
|
||||
pytest.param((torch.float32, torch.bfloat16), id="s32_a16"),
|
||||
pytest.param((torch.float32, torch.float16), id="s32_afp16"),
|
||||
],
|
||||
)
|
||||
def test_replayssm_standard_decode_desync_write_pos(
|
||||
precision: tuple[torch.dtype, torch.dtype],
|
||||
):
|
||||
# Rows start at staggered ring positions, so they flush on different steps
|
||||
# and hold genuinely different cached histories.
|
||||
state_dtype, act_dtype = precision
|
||||
_run_standard_decode(
|
||||
state_dtype=state_dtype,
|
||||
act_dtype=act_dtype,
|
||||
batch=4,
|
||||
nheads=8,
|
||||
headdim=64,
|
||||
ngroups=4,
|
||||
dstate=64,
|
||||
max_cache_len=4,
|
||||
num_steps=12,
|
||||
has_z=True,
|
||||
dt_softplus=True,
|
||||
use_dt_bias=True,
|
||||
desync_write_pos=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
|
||||
@pytest.mark.parametrize("precision", _PRECISIONS)
|
||||
@pytest.mark.parametrize("with_padding", [False, True])
|
||||
def test_replayssm_standard_decode_with_batch_indices(
|
||||
precision: tuple[torch.dtype, torch.dtype],
|
||||
with_padding: bool,
|
||||
):
|
||||
# Sparse state allocation via state_batch_indices, with NULL_BLOCK_ID
|
||||
# padding rows. The pure-PyTorch references do not model the sparse
|
||||
# gather, so the anchor here is the upstream baseline decode kernel.
|
||||
state_dtype, act_dtype = precision
|
||||
device = "cuda"
|
||||
both_fp32 = state_dtype == torch.float32 and act_dtype == torch.float32
|
||||
rtol, atol = _tolerances(torch.float32 if both_fp32 else torch.bfloat16)
|
||||
set_random_seed(0)
|
||||
|
||||
batch = 3
|
||||
padding = 2 if with_padding else 0
|
||||
padded_batch = batch + padding
|
||||
total_state_slots = 16
|
||||
nheads = 4
|
||||
ngroups = 2
|
||||
headdim = 64
|
||||
dstate = 16
|
||||
max_cache_len = 4
|
||||
num_steps = 2 * max_cache_len
|
||||
|
||||
state = torch.randn(
|
||||
total_state_slots, nheads, headdim, dstate, dtype=state_dtype, device=device
|
||||
)
|
||||
state_baseline = state.clone()
|
||||
state_cached = state.clone()
|
||||
state_before = state.clone()
|
||||
|
||||
state_indices = (
|
||||
torch.randperm(total_state_slots - 1, device=device)[:batch] + 1
|
||||
).to(torch.int32)
|
||||
state_batch_indices = torch.cat(
|
||||
[
|
||||
state_indices,
|
||||
torch.full((padding,), NULL_BLOCK_ID, dtype=torch.int32, device=device),
|
||||
]
|
||||
)
|
||||
unused_states = torch.ones(total_state_slots, dtype=torch.bool, device=device)
|
||||
unused_states[state_indices] = False
|
||||
|
||||
A = _tied_A(nheads, headdim, dstate, device)
|
||||
dt_bias = _tied_dt_bias(nheads, headdim, device)
|
||||
D = torch.randn(nheads, headdim, device=device)
|
||||
x_cache = torch.zeros(
|
||||
total_state_slots,
|
||||
nheads,
|
||||
max_cache_len,
|
||||
headdim,
|
||||
device=device,
|
||||
dtype=act_dtype,
|
||||
)
|
||||
dt_cache = torch.zeros(
|
||||
total_state_slots, nheads, max_cache_len, device=device, dtype=torch.float32
|
||||
)
|
||||
B_cache = torch.zeros(
|
||||
total_state_slots,
|
||||
ngroups,
|
||||
max_cache_len,
|
||||
dstate,
|
||||
device=device,
|
||||
dtype=act_dtype,
|
||||
)
|
||||
bc_pre = torch.empty(
|
||||
padded_batch, ngroups, max_cache_len, device=device, dtype=torch.float32
|
||||
)
|
||||
write_pos = torch.zeros(padded_batch, dtype=torch.int32, device=device)
|
||||
|
||||
for _ in range(num_steps):
|
||||
x = torch.randn(padded_batch, nheads, headdim, device=device, dtype=act_dtype)
|
||||
dt = _tied_dt(padded_batch, nheads, headdim, device, act_dtype)
|
||||
B = torch.randn(padded_batch, ngroups, dstate, device=device, dtype=act_dtype)
|
||||
C = torch.randn(padded_batch, ngroups, dstate, device=device, dtype=act_dtype)
|
||||
z = torch.randn_like(x)
|
||||
is_flush = write_pos == max_cache_len - 1
|
||||
|
||||
out_baseline = torch.empty_like(x)
|
||||
selective_state_update(
|
||||
state_baseline,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D=D,
|
||||
z=z,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_batch_indices,
|
||||
out=out_baseline,
|
||||
)
|
||||
|
||||
out_cached = torch.full_like(x, 42)
|
||||
common = dict(
|
||||
D=D,
|
||||
z=z,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=True,
|
||||
x_cache=x_cache,
|
||||
dt_cache=dt_cache,
|
||||
B_cache=B_cache,
|
||||
write_pos=write_pos,
|
||||
is_flush=is_flush,
|
||||
max_cache_len=max_cache_len,
|
||||
state_batch_indices=state_batch_indices,
|
||||
out=out_cached,
|
||||
)
|
||||
selective_state_update_replayssm_output_only(
|
||||
state_cached, x, dt, A, B, C, bc_pre=bc_pre, **common
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
out_cached[:batch], out_baseline[:batch], rtol=rtol, atol=atol
|
||||
)
|
||||
if with_padding:
|
||||
assert torch.equal(
|
||||
out_cached[batch:], torch.full_like(out_cached[batch:], 42)
|
||||
)
|
||||
|
||||
if bool(is_flush[:batch].all()):
|
||||
torch.testing.assert_close(
|
||||
state_cached[state_indices],
|
||||
state_baseline[state_indices],
|
||||
rtol=rtol,
|
||||
atol=atol,
|
||||
)
|
||||
|
||||
write_pos = torch.where(is_flush, torch.zeros_like(write_pos), write_pos + 1)
|
||||
|
||||
assert torch.equal(state_cached[unused_states], state_before[unused_states])
|
||||
assert torch.equal(state_baseline[unused_states], state_before[unused_states])
|
||||
|
||||
|
||||
# Geometries with (nheads, ngroups) both divisible by the tp below.
|
||||
_TP_GEOMETRIES = [
|
||||
pytest.param((8, 64, 64, 4), id="small"),
|
||||
pytest.param((96, 80, 128, 8), id="nano4b"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
|
||||
@pytest.mark.parametrize("precision", _PRECISIONS)
|
||||
@pytest.mark.parametrize("geometry", _TP_GEOMETRIES)
|
||||
@pytest.mark.parametrize("tp", [2])
|
||||
def test_replayssm_standard_decode_tp_head_shard_equivalence(
|
||||
precision: tuple[torch.dtype, torch.dtype],
|
||||
geometry: tuple[int, int, int, int],
|
||||
tp: int,
|
||||
):
|
||||
"""Tensor-parallel correctness at the kernel boundary (single GPU).
|
||||
|
||||
The kernel has no cross-rank communication, so head sharding must be exactly
|
||||
separable: one run over all heads must equal concatenating ``tp`` independent
|
||||
per-rank runs on ``nheads // tp`` heads and ``ngroups // tp`` groups. This
|
||||
guards the per-rank divisors and group->head mapping the state-shape wiring
|
||||
relies on. The real TP1==TP2 engine check lives in the v1/e2e suite."""
|
||||
state_dtype, act_dtype = precision
|
||||
nheads, headdim, dstate, ngroups = geometry
|
||||
assert nheads % tp == 0 and ngroups % tp == 0
|
||||
device = "cuda"
|
||||
both_fp32 = state_dtype == torch.float32 and act_dtype == torch.float32
|
||||
rtol, atol = _tolerances(torch.float32 if both_fp32 else torch.bfloat16)
|
||||
set_random_seed(0)
|
||||
|
||||
batch = 4
|
||||
max_cache_len = 4
|
||||
num_steps = 2 * max_cache_len + 1
|
||||
nh_s = nheads // tp
|
||||
ng_s = ngroups // tp
|
||||
|
||||
# Shards slice the tied params; never .contiguous() -- that would drop the
|
||||
# stride-0 broadcast the kernel's TIE_HDIM asserts require.
|
||||
A = _tied_A(nheads, headdim, dstate, device)
|
||||
dt_bias = _tied_dt_bias(nheads, headdim, device)
|
||||
D = torch.randn(nheads, headdim, device=device)
|
||||
|
||||
state_full = torch.randn(
|
||||
batch, nheads, headdim, dstate, dtype=state_dtype, device=device
|
||||
)
|
||||
x_cache, dt_cache, B_cache, _ = allocate_update_caches(
|
||||
batch,
|
||||
nheads,
|
||||
ngroups,
|
||||
headdim,
|
||||
dstate,
|
||||
max_cache_len,
|
||||
device,
|
||||
act_dtype,
|
||||
act_dtype,
|
||||
)
|
||||
bc_pre = torch.empty(
|
||||
batch, ngroups, max_cache_len, device=device, dtype=torch.float32
|
||||
)
|
||||
|
||||
# Per-rank shards, each seeded from the matching head slice so all start equal.
|
||||
shard_state = []
|
||||
shard_caches = []
|
||||
for r in range(tp):
|
||||
h0 = r * nh_s
|
||||
shard_state.append(state_full[:, h0 : h0 + nh_s].contiguous())
|
||||
xc, dtc, bc, _ = allocate_update_caches(
|
||||
batch,
|
||||
nh_s,
|
||||
ng_s,
|
||||
headdim,
|
||||
dstate,
|
||||
max_cache_len,
|
||||
device,
|
||||
act_dtype,
|
||||
act_dtype,
|
||||
)
|
||||
bcp = torch.empty(
|
||||
batch, ng_s, max_cache_len, device=device, dtype=torch.float32
|
||||
)
|
||||
shard_caches.append((xc, dtc, bc, bcp))
|
||||
|
||||
write_pos = torch.zeros(batch, dtype=torch.int32, device=device)
|
||||
for _ in range(num_steps):
|
||||
x = torch.randn(batch, nheads, headdim, device=device, dtype=act_dtype)
|
||||
dt = _tied_dt(batch, nheads, headdim, device, act_dtype)
|
||||
B = torch.randn(batch, ngroups, dstate, device=device, dtype=act_dtype)
|
||||
C = torch.randn(batch, ngroups, dstate, device=device, dtype=act_dtype)
|
||||
z = torch.randn_like(x)
|
||||
is_flush = write_pos == max_cache_len - 1
|
||||
|
||||
out_full = torch.empty_like(x)
|
||||
selective_state_update_replayssm_output_only(
|
||||
state_full,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D=D,
|
||||
z=z,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=True,
|
||||
x_cache=x_cache,
|
||||
dt_cache=dt_cache,
|
||||
B_cache=B_cache,
|
||||
bc_pre=bc_pre,
|
||||
write_pos=write_pos,
|
||||
is_flush=is_flush,
|
||||
max_cache_len=max_cache_len,
|
||||
out=out_full,
|
||||
)
|
||||
|
||||
for r in range(tp):
|
||||
h0 = r * nh_s
|
||||
g0 = r * ng_s
|
||||
xc, dtc, bc, bcp = shard_caches[r]
|
||||
out_shard = torch.empty(
|
||||
batch, nh_s, headdim, device=device, dtype=act_dtype
|
||||
)
|
||||
selective_state_update_replayssm_output_only(
|
||||
shard_state[r],
|
||||
x[:, h0 : h0 + nh_s].contiguous(),
|
||||
dt[:, h0 : h0 + nh_s],
|
||||
A[h0 : h0 + nh_s],
|
||||
B[:, g0 : g0 + ng_s].contiguous(),
|
||||
C[:, g0 : g0 + ng_s].contiguous(),
|
||||
D=D[h0 : h0 + nh_s].contiguous(),
|
||||
z=z[:, h0 : h0 + nh_s].contiguous(),
|
||||
dt_bias=dt_bias[h0 : h0 + nh_s],
|
||||
dt_softplus=True,
|
||||
x_cache=xc,
|
||||
dt_cache=dtc,
|
||||
B_cache=bc,
|
||||
bc_pre=bcp,
|
||||
write_pos=write_pos,
|
||||
is_flush=is_flush,
|
||||
max_cache_len=max_cache_len,
|
||||
out=out_shard,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
out_full[:, h0 : h0 + nh_s], out_shard, rtol=rtol, atol=atol
|
||||
)
|
||||
if bool(is_flush.any()):
|
||||
torch.testing.assert_close(
|
||||
state_full[:, h0 : h0 + nh_s][is_flush],
|
||||
shard_state[r][is_flush],
|
||||
rtol=rtol,
|
||||
atol=atol,
|
||||
)
|
||||
|
||||
write_pos = torch.where(is_flush, torch.zeros_like(write_pos), write_pos + 1)
|
||||
@@ -76,3 +76,163 @@ def selective_state_update_ref(
|
||||
if not has_heads:
|
||||
out = out.squeeze(1)
|
||||
return out
|
||||
|
||||
|
||||
def selective_state_update_replayssm_output_only_ref(
|
||||
state: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
dt: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
D: torch.Tensor | None = None,
|
||||
z: torch.Tensor | None = None,
|
||||
dt_bias: torch.Tensor | None = None,
|
||||
dt_softplus: bool = False,
|
||||
x_cache: torch.Tensor | None = None,
|
||||
dt_cache: torch.Tensor | None = None,
|
||||
B_cache: torch.Tensor | None = None,
|
||||
write_pos: torch.Tensor | None = None,
|
||||
max_cache_len: int = 16,
|
||||
) -> torch.Tensor:
|
||||
"""Pure-PyTorch cached-bc reference for validation."""
|
||||
has_heads = state.dim() > 3
|
||||
if state.dim() == 3:
|
||||
state = state.unsqueeze(1)
|
||||
if x.dim() == 2:
|
||||
x = x.unsqueeze(1)
|
||||
if dt.dim() == 2:
|
||||
dt = dt.unsqueeze(1)
|
||||
if A.dim() == 2:
|
||||
A = A.unsqueeze(0)
|
||||
if B.dim() == 2:
|
||||
B = B.unsqueeze(1)
|
||||
if C.dim() == 2:
|
||||
C = C.unsqueeze(1)
|
||||
if D is not None and D.dim() == 1:
|
||||
D = D.unsqueeze(0)
|
||||
if z is not None and z.dim() == 2:
|
||||
z = z.unsqueeze(1)
|
||||
if dt_bias is not None and dt_bias.dim() == 1:
|
||||
dt_bias = dt_bias.unsqueeze(0)
|
||||
|
||||
batch, nheads, dim, dstate = state.shape
|
||||
assert x.shape == (batch, nheads, dim)
|
||||
assert dt.shape == x.shape
|
||||
assert A.shape == (nheads, dim, dstate)
|
||||
ngroups = B.shape[1]
|
||||
assert nheads % ngroups == 0, "nheads must be divisible by ngroups"
|
||||
assert B.shape == (batch, ngroups, dstate)
|
||||
assert C.shape == B.shape
|
||||
|
||||
ratio = nheads // ngroups
|
||||
|
||||
dt_val = dt[:, :, 0].float()
|
||||
if dt_bias is not None:
|
||||
dt_val = dt_val + dt_bias[:, 0].float()
|
||||
if dt_softplus:
|
||||
dt_val = F.softplus(dt_val)
|
||||
A_val = A[:, 0, 0].float()
|
||||
C_heads = C.repeat_interleave(ratio, dim=1)
|
||||
out = torch.empty(batch, nheads, dim, device=x.device, dtype=torch.float32)
|
||||
|
||||
assert x_cache is not None
|
||||
assert dt_cache is not None
|
||||
assert B_cache is not None
|
||||
assert write_pos is not None
|
||||
|
||||
for b in range(batch):
|
||||
cache_len = int(write_pos[b].item())
|
||||
is_flush = cache_len == max_cache_len - 1
|
||||
n_steps = cache_len + 1
|
||||
|
||||
dt_all = torch.zeros(nheads, n_steps, device=x.device, dtype=torch.float32)
|
||||
if cache_len > 0:
|
||||
dt_all[:, :cache_len] = dt_cache[b, :, :cache_len]
|
||||
dt_all[:, cache_len] = dt_val[b]
|
||||
|
||||
cumsum = torch.cumsum(dt_all, dim=-1)
|
||||
total = cumsum[:, -1]
|
||||
dA_cumsum = A_val[:, None] * cumsum
|
||||
dA_total = A_val * total
|
||||
total_decay = torch.exp(dA_total)
|
||||
scale = dt_all * torch.exp(dA_total[:, None] - dA_cumsum)
|
||||
|
||||
x_all = torch.zeros(nheads, dim, n_steps, device=x.device, dtype=x.dtype)
|
||||
if cache_len > 0:
|
||||
x_all[..., :cache_len] = x_cache[b, :, :cache_len, :].permute(0, 2, 1)
|
||||
x_all[..., cache_len] = x[b]
|
||||
|
||||
B_all = torch.zeros(ngroups, n_steps, dstate, device=B.device, dtype=B.dtype)
|
||||
if cache_len > 0:
|
||||
B_all[:, :cache_len, :] = B_cache[b, :, :cache_len, :]
|
||||
B_all[:, cache_len, :] = B[b]
|
||||
|
||||
B_heads = B_all.repeat_interleave(ratio, dim=0)
|
||||
C_heads_b = C_heads[b]
|
||||
|
||||
if is_flush:
|
||||
B_scaled = (B_heads.float() * scale[:, :, None]).to(B_heads.dtype)
|
||||
delta = torch.einsum("hdk,hkn->hdn", x_all.float(), B_scaled.float())
|
||||
state_new = state[b].float() * total_decay[:, None, None] + delta
|
||||
state[b].copy_(state_new.to(state.dtype))
|
||||
out[b] = torch.einsum("hdn,hn->hd", state_new, C_heads_b.float())
|
||||
else:
|
||||
checkpoint_out = torch.einsum(
|
||||
"hdn,hn->hd", state[b].float(), C_heads_b.float()
|
||||
)
|
||||
checkpoint_out = checkpoint_out * total_decay[:, None]
|
||||
BC = torch.einsum("hkn,hn->hk", B_heads.float(), C_heads_b.float())
|
||||
cache_out = torch.einsum("hdk,hk->hd", x_all.float(), scale * BC)
|
||||
out[b] = checkpoint_out + cache_out
|
||||
x_cache[b, :, cache_len, :] = x[b]
|
||||
dt_cache[b, :, cache_len] = dt_val[b]
|
||||
B_cache[b, :, cache_len, :] = B[b]
|
||||
|
||||
if D is not None:
|
||||
out = out + (x.float() * D[None]).to(out.dtype)
|
||||
if z is not None:
|
||||
out = out * F.silu(z.float())
|
||||
out = out.to(x.dtype)
|
||||
if not has_heads:
|
||||
out = out.squeeze(1)
|
||||
return out
|
||||
|
||||
|
||||
def allocate_update_caches(
|
||||
batch: int,
|
||||
nheads: int,
|
||||
ngroups: int,
|
||||
dim: int,
|
||||
dstate: int,
|
||||
max_cache_len: int,
|
||||
device: torch.device,
|
||||
x_dtype: torch.dtype,
|
||||
B_dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Allocate dense reference caches for standalone validation."""
|
||||
x_cache = torch.zeros(
|
||||
batch,
|
||||
nheads,
|
||||
max_cache_len,
|
||||
dim,
|
||||
device=device,
|
||||
dtype=x_dtype,
|
||||
)
|
||||
dt_cache = torch.zeros(
|
||||
batch,
|
||||
nheads,
|
||||
max_cache_len,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
B_cache = torch.zeros(
|
||||
batch,
|
||||
ngroups,
|
||||
max_cache_len,
|
||||
dstate,
|
||||
device=device,
|
||||
dtype=B_dtype,
|
||||
)
|
||||
write_pos = torch.zeros(batch, dtype=torch.int32, device=device)
|
||||
return x_cache, dt_cache, B_cache, write_pos
|
||||
|
||||
@@ -134,6 +134,22 @@ def test_registry_is_pp(model_arch, is_pp, init_cuda):
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,supported",
|
||||
[
|
||||
# ReplaySSM is opt-in per model; only Nemotron-H sets the flag today.
|
||||
("NemotronHForCausalLM", True),
|
||||
("Mamba2ForCausalLM", False),
|
||||
("Zamba2ForCausalLM", False),
|
||||
],
|
||||
)
|
||||
def test_registry_supports_replayssm(model_arch, supported):
|
||||
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
|
||||
assert model_info is not None
|
||||
assert model_info.supports_replayssm is supported
|
||||
|
||||
|
||||
def test_lazy_modelinfo_package_hash_includes_submodules(tmp_path):
|
||||
package_dir = tmp_path / "model_package"
|
||||
package_dir.mkdir()
|
||||
|
||||
@@ -42,6 +42,8 @@ def _make_vllm_config(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=block_size,
|
||||
mamba_cache_mode="all",
|
||||
use_replayssm=False,
|
||||
replayssm_buffer_len=16,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mamba2 ReplaySSM decode write-position derivation in
|
||||
BaseMambaAttentionMetadataBuilder: write_pos and is_flush computed from the
|
||||
per-request ring origin (replayssm_decode_base) and num_computed.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
MockMambaBuilder,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
DEVICE = torch.device("cpu")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReplaySSMBuildCase:
|
||||
"""A decode batch and its expected per-row write_pos / is_flush.
|
||||
|
||||
num_computed = seq_len - query_len; write_pos =
|
||||
(num_computed - decode_base) % buffer_len; is_flush = write_pos ==
|
||||
buffer_len - 1 (or a forced one-token flush when num_computed < decode_base).
|
||||
"""
|
||||
|
||||
seq_lens: list[int]
|
||||
query_lens: list[int]
|
||||
is_prefilling: list[bool]
|
||||
decode_base: list[int]
|
||||
buffer_len: int
|
||||
expected_write_pos: list[int]
|
||||
expected_is_flush: list[int]
|
||||
mamba_cache_mode: str = "none"
|
||||
|
||||
|
||||
REPLAYSSM_BUILD_CASES = {
|
||||
# decode_base == num_prompt (fresh request).
|
||||
"fresh_decode": ReplaySSMBuildCase(
|
||||
seq_lens=[106],
|
||||
query_lens=[1],
|
||||
is_prefilling=[False],
|
||||
decode_base=[100],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[5],
|
||||
expected_is_flush=[0],
|
||||
),
|
||||
# decode_base > num_prompt anchors write_pos at the resume point.
|
||||
"resumed_reanchors_to_zero": ReplaySSMBuildCase(
|
||||
seq_lens=[106],
|
||||
query_lens=[1],
|
||||
is_prefilling=[False],
|
||||
decode_base=[105],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[0],
|
||||
expected_is_flush=[0],
|
||||
),
|
||||
# write_pos == buffer_len - 1 flushes.
|
||||
"flush_boundary": ReplaySSMBuildCase(
|
||||
seq_lens=[116],
|
||||
query_lens=[1],
|
||||
is_prefilling=[False],
|
||||
decode_base=[100],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[15],
|
||||
expected_is_flush=[1],
|
||||
),
|
||||
# Resumed request landing on a flush boundary.
|
||||
"resumed_flush_boundary": ReplaySSMBuildCase(
|
||||
seq_lens=[121],
|
||||
query_lens=[1],
|
||||
is_prefilling=[False],
|
||||
decode_base=[105],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[15],
|
||||
expected_is_flush=[1],
|
||||
),
|
||||
# Per-row write_pos / is_flush are independent.
|
||||
"mixed_rows": ReplaySSMBuildCase(
|
||||
seq_lens=[104, 106, 216],
|
||||
query_lens=[1, 1, 1],
|
||||
is_prefilling=[False, False, False],
|
||||
decode_base=[100, 105, 200],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[3, 0, 15],
|
||||
expected_is_flush=[0, 0, 1],
|
||||
),
|
||||
# write_pos wraps within the buffer (6 % 4 == 2).
|
||||
"small_buffer_wrap": ReplaySSMBuildCase(
|
||||
seq_lens=[112],
|
||||
query_lens=[1],
|
||||
is_prefilling=[False],
|
||||
decode_base=[105],
|
||||
buffer_len=4,
|
||||
expected_write_pos=[2],
|
||||
expected_is_flush=[0],
|
||||
),
|
||||
# Single-token prefill-as-decode still in the prompt (num_computed <
|
||||
# decode_base): forced one-token flush.
|
||||
"leftover_prompt_one_token_flush": ReplaySSMBuildCase(
|
||||
seq_lens=[100],
|
||||
query_lens=[1],
|
||||
is_prefilling=[True],
|
||||
decode_base=[100],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[0],
|
||||
expected_is_flush=[1],
|
||||
),
|
||||
# Align mode (block_size 16). Past the first boundary the ring re-anchors at
|
||||
# the block start: num_computed 117 -> block_start 112, write_pos 5 (vs 1 in
|
||||
# none mode).
|
||||
"align_reanchor_past_boundary": ReplaySSMBuildCase(
|
||||
seq_lens=[118],
|
||||
query_lens=[1],
|
||||
is_prefilling=[False],
|
||||
decode_base=[100],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[5],
|
||||
expected_is_flush=[0],
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
# First-block boundary (num_computed+1 == 112) forces a flush even though
|
||||
# write_pos (11) != buffer_len - 1.
|
||||
"align_first_block_boundary_flush": ReplaySSMBuildCase(
|
||||
seq_lens=[112],
|
||||
query_lens=[1],
|
||||
is_prefilling=[False],
|
||||
decode_base=[100],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[11],
|
||||
expected_is_flush=[1],
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
# First step of a new block re-anchors write_pos to 0.
|
||||
"align_new_block_start_zero": ReplaySSMBuildCase(
|
||||
seq_lens=[113],
|
||||
query_lens=[1],
|
||||
is_prefilling=[False],
|
||||
decode_base=[100],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[0],
|
||||
expected_is_flush=[0],
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
# block_size % buffer_len == 0: a later boundary lands on write_pos ==
|
||||
# buffer_len - 1, so the boundary flush coincides with the natural flush.
|
||||
"align_boundary_coincides_natural_flush": ReplaySSMBuildCase(
|
||||
seq_lens=[128],
|
||||
query_lens=[1],
|
||||
is_prefilling=[False],
|
||||
decode_base=[100],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[15],
|
||||
expected_is_flush=[1],
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
# block_size % buffer_len != 0 (buffer_len 6): the boundary step still flushes
|
||||
# although write_pos (3) != buffer_len - 1.
|
||||
"align_unaligned_buffer_forces_flush": ReplaySSMBuildCase(
|
||||
seq_lens=[128],
|
||||
query_lens=[1],
|
||||
is_prefilling=[False],
|
||||
decode_base=[100],
|
||||
buffer_len=6,
|
||||
expected_write_pos=[3],
|
||||
expected_is_flush=[1],
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
# Per-row independence in align mode: partial-block / new-block / boundary.
|
||||
"align_mixed_rows": ReplaySSMBuildCase(
|
||||
seq_lens=[105, 113, 112],
|
||||
query_lens=[1, 1, 1],
|
||||
is_prefilling=[False, False, False],
|
||||
decode_base=[100, 100, 100],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[4, 0, 11],
|
||||
expected_is_flush=[0, 0, 1],
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _make_mamba_spec(buffer_len: int) -> MambaSpec:
|
||||
# Five-tensor ReplaySSM page; the builder only reads shapes[4][0] (bc groups).
|
||||
return MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=(
|
||||
(1, 1),
|
||||
(1, 1, 1),
|
||||
(1, buffer_len, 1),
|
||||
(1, buffer_len),
|
||||
(1, buffer_len, 1),
|
||||
),
|
||||
dtypes=(torch.float32,),
|
||||
)
|
||||
|
||||
|
||||
def _create_replayssm_builder(
|
||||
buffer_len: int, mamba_cache_mode: str = "none"
|
||||
) -> MockMambaBuilder:
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE
|
||||
)
|
||||
# Set the flags after construction to skip validate_mamba_cached_kernel
|
||||
# (it requires a Triton backend) on the mock model.
|
||||
vllm_config.cache_config.use_replayssm = True
|
||||
vllm_config.cache_config.replayssm_buffer_len = buffer_len
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
return MockMambaBuilder(
|
||||
_make_mamba_spec(buffer_len), ["layer0"], vllm_config, DEVICE
|
||||
)
|
||||
|
||||
|
||||
def _build(builder: MockMambaBuilder, case: ReplaySSMBuildCase):
|
||||
batch = BatchSpec(seq_lens=case.seq_lens, query_lens=case.query_lens)
|
||||
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
|
||||
is_prefilling=torch.tensor(case.is_prefilling, dtype=torch.bool),
|
||||
replayssm_decode_base_cpu=torch.tensor(case.decode_base, dtype=torch.int32),
|
||||
)
|
||||
return builder.build(0, common)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", REPLAYSSM_BUILD_CASES.values(), ids=REPLAYSSM_BUILD_CASES.keys()
|
||||
)
|
||||
def test_replayssm_write_pos(case: ReplaySSMBuildCase):
|
||||
builder = _create_replayssm_builder(case.buffer_len, case.mamba_cache_mode)
|
||||
meta = _build(builder, case)
|
||||
|
||||
assert meta.write_pos_d is not None
|
||||
assert meta.is_flush_d is not None
|
||||
n = len(case.expected_write_pos)
|
||||
assert meta.write_pos_d[:n].tolist() == case.expected_write_pos
|
||||
assert meta.is_flush_d[:n].tolist() == case.expected_is_flush
|
||||
|
||||
|
||||
def test_resumed_request_differs_from_fresh():
|
||||
"""Same token count, different decode_base: fresh (base 100) -> write_pos 5,
|
||||
resumed (base 105) -> write_pos 0."""
|
||||
builder = _create_replayssm_builder(16)
|
||||
batch = BatchSpec(seq_lens=[106, 106], query_lens=[1, 1])
|
||||
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
|
||||
is_prefilling=torch.tensor([False, False]),
|
||||
replayssm_decode_base_cpu=torch.tensor([100, 105], dtype=torch.int32),
|
||||
)
|
||||
meta = builder.build(0, common)
|
||||
|
||||
assert meta.write_pos_d.tolist()[:2] == [5, 0]
|
||||
assert meta.is_flush_d.tolist()[:2] == [0, 0]
|
||||
@@ -0,0 +1,138 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Engine-level parity: ReplaySSM standard decode vs the baseline SSM kernel."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.v1.metrics.reader import Counter
|
||||
|
||||
from ...models.utils import check_logprobs_close
|
||||
from ...utils import large_gpu_mark, multi_gpu_test
|
||||
|
||||
# Mamba2 (Nemotron-3) hybrid.
|
||||
MAMBA2_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16"
|
||||
MODELS = [
|
||||
pytest.param(MAMBA2_MODEL, marks=large_gpu_mark(min_gb=40)),
|
||||
]
|
||||
|
||||
PROMPTS = [
|
||||
"The capital of France is",
|
||||
"Once upon a time, in a small village,",
|
||||
]
|
||||
|
||||
|
||||
def _check_replayssm_parity(vllm_runner, model_name, *, tensor_parallel_size=1):
|
||||
# Compare logprobs, not greedy ids: ReplaySSM's fp arithmetic can flip a
|
||||
# near-tie. Baseline and ReplaySSM run at the same TP, so TP numerics are
|
||||
# common-mode and only ReplaySSM varies.
|
||||
common = dict(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
)
|
||||
with vllm_runner(model_name, **common) as llm:
|
||||
baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
|
||||
with vllm_runner(
|
||||
model_name, use_replayssm=True, replayssm_buffer_len=16, **common
|
||||
) as llm:
|
||||
replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline,
|
||||
outputs_1_lst=replay,
|
||||
name_0="baseline",
|
||||
name_1="replayssm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_replayssm_decode_matches_baseline(vllm_runner, model_name):
|
||||
_check_replayssm_parity(vllm_runner, model_name)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("model_name", [MAMBA2_MODEL])
|
||||
def test_replayssm_decode_matches_baseline_tp2(vllm_runner, model_name):
|
||||
# Tensor-parallel correctness: ReplaySSM's caches and checkpoint state are
|
||||
# sharded per rank, so TP2 decode must still match the baseline at TP2.
|
||||
_check_replayssm_parity(vllm_runner, model_name, tensor_parallel_size=2)
|
||||
|
||||
|
||||
# Prefix spans several mamba blocks; prefix caching only reuses full blocks.
|
||||
_PC_SENTENCE = (
|
||||
"In a detailed survey of state space models, the authors compared many "
|
||||
"architectures across a wide range of long-context language tasks and "
|
||||
"measured their throughput, memory use, and accuracy in careful detail. "
|
||||
)
|
||||
_PC_PREFIX = _PC_SENTENCE * 120
|
||||
PREFIX_CACHING_PROMPTS = [
|
||||
_PC_PREFIX + "The most important conclusion was that",
|
||||
_PC_PREFIX + "Surprisingly, the experiments showed that",
|
||||
_PC_PREFIX + "The most important conclusion was that",
|
||||
]
|
||||
|
||||
|
||||
def _prefix_cache_hits(llm) -> int:
|
||||
return sum(
|
||||
m.value
|
||||
for m in llm.llm.get_metrics()
|
||||
if isinstance(m, Counter) and m.name == "vllm:prefix_cache_hits"
|
||||
)
|
||||
|
||||
|
||||
def _check_replayssm_prefix_caching_parity(
|
||||
vllm_runner, model_name, *, tensor_parallel_size=1
|
||||
):
|
||||
# align mode materializes the exact SSM state at each block boundary, so
|
||||
# ReplaySSM's cached prefixes must match the always-materialized baseline.
|
||||
common = dict(
|
||||
max_model_len=8192,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=True,
|
||||
enable_chunked_prefill=True,
|
||||
mamba_cache_mode="align",
|
||||
disable_log_stats=False, # required for llm.get_metrics()
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
)
|
||||
with vllm_runner(model_name, **common) as llm:
|
||||
baseline = llm.generate_greedy_logprobs(
|
||||
PREFIX_CACHING_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
with vllm_runner(
|
||||
model_name, use_replayssm=True, replayssm_buffer_len=16, **common
|
||||
) as llm:
|
||||
# Prime the cache, then measure, so cache hits are deterministic.
|
||||
llm.generate_greedy_logprobs(
|
||||
PREFIX_CACHING_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
replay = llm.generate_greedy_logprobs(
|
||||
PREFIX_CACHING_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
replay_hits = _prefix_cache_hits(llm)
|
||||
|
||||
# Without real cache hits the cached path is never exercised.
|
||||
assert replay_hits > 0, (
|
||||
"ReplaySSM align-mode run produced no prefix-cache hits; the shared "
|
||||
"prefix may be shorter than one mamba block, so prefix caching is inert"
|
||||
)
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline,
|
||||
outputs_1_lst=replay,
|
||||
name_0="baseline_align_pc",
|
||||
name_1="replayssm_align_pc",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_replayssm_prefix_caching_matches_baseline(vllm_runner, model_name):
|
||||
_check_replayssm_prefix_caching_parity(vllm_runner, model_name)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("model_name", [MAMBA2_MODEL])
|
||||
def test_replayssm_prefix_caching_matches_baseline_tp2(vllm_runner, model_name):
|
||||
_check_replayssm_prefix_caching_parity(
|
||||
vllm_runner, model_name, tensor_parallel_size=2
|
||||
)
|
||||
Reference in New Issue
Block a user