Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
+31 -10
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass
from dataclasses import dataclass, replace
import torch
@@ -58,10 +58,33 @@ def compute_mamba_prefill_checkpoints(
return offsets, cols
def _gather_checkpoint_state_indices(
block_table: torch.Tensor, request_rows: torch.Tensor, block_cols: torch.Tensor
) -> torch.Tensor:
return torch.where(
block_cols >= 0, block_table[request_rows, block_cols], NULL_BLOCK_ID
)
@dataclass
class MambaPrefillCheckpointMetadata:
checkpoint_offsets: torch.Tensor
state_indices: torch.Tensor
# Gather indices into the block table, kept so that another KV cache group
# with the same spec can re-derive its own ``state_indices``.
request_rows: torch.Tensor | None = None
block_cols: torch.Tensor | None = None
def regather_state_indices(
self, block_table: torch.Tensor
) -> "MambaPrefillCheckpointMetadata":
assert self.request_rows is not None and self.block_cols is not None
return replace(
self,
state_indices=_gather_checkpoint_state_indices(
block_table, self.request_rows, self.block_cols
),
)
class MambaPrefillCheckpointBuilder:
@@ -76,6 +99,8 @@ class MambaPrefillCheckpointBuilder:
m: CommonAttentionMetadata,
request_rows: list[int],
) -> MambaPrefillCheckpointMetadata | None:
if self.vllm_config.cache_config.mamba_cache_mode != "align":
return None
if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0:
return None
assert m.seq_lens_cpu_upper_bound is not None
@@ -109,17 +134,13 @@ class MambaPrefillCheckpointBuilder:
checkpoint_cols_tensor = async_tensor_h2d(
checkpoint_cols, dtype=torch.int64, device=m.query_start_loc.device
)
checkpoint_state_indices = m.block_table_tensor[
request_rows_tensor, checkpoint_cols_tensor
]
checkpoint_state_indices = torch.where(
checkpoint_cols_tensor >= 0,
checkpoint_state_indices,
NULL_BLOCK_ID,
)
return MambaPrefillCheckpointMetadata(
checkpoint_offsets_tensor,
checkpoint_state_indices,
_gather_checkpoint_state_indices(
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
),
request_rows=request_rows_tensor,
block_cols=checkpoint_cols_tensor,
)
@@ -260,8 +260,6 @@ class MambaMixer(MambaBase, PluggableLayer):
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
@@ -297,8 +295,6 @@ class MambaMixer(MambaBase, PluggableLayer):
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
num_decode_tokens = attn_metadata.num_decode_tokens
num_prefills = attn_metadata.num_prefills # request count
num_decodes = attn_metadata.num_decode_tokens # token count (=request)
has_prefill = num_prefill_tokens > 0
has_decode = num_decode_tokens > 0
num_actual_tokens = num_prefill_tokens + num_decode_tokens
@@ -314,34 +310,6 @@ class MambaMixer(MambaBase, PluggableLayer):
gate_p = prefill_decode_split.gate_p
gate_d = prefill_decode_split.gate_d
if is_mamba_cache_all:
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
torch.split(
attn_metadata.block_idx_last_computed_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
torch.split(
attn_metadata.block_idx_last_scheduled_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_first_scheduled_token_p = (
attn_metadata.block_idx_first_scheduled_token_p
)
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
else:
block_idx_last_computed_token_d = None
block_idx_last_computed_token_p = None
block_idx_last_scheduled_token_d = None
block_idx_last_scheduled_token_p = None
block_idx_first_scheduled_token_p = None
num_computed_tokens_p = None
ssm_outputs = []
if has_prefill:
@@ -355,11 +323,6 @@ class MambaMixer(MambaBase, PluggableLayer):
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
query_start_loc=query_start_loc_p,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
)
# 3. State Space Model sequence transformations.
@@ -383,10 +346,6 @@ class MambaMixer(MambaBase, PluggableLayer):
cache_indices=state_indices_tensor_p,
has_initial_state=has_initial_states_p,
query_start_loc=query_start_loc_p,
block_size=mamba_block_size,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
cu_chunk_seqlen=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
)
@@ -396,16 +355,6 @@ class MambaMixer(MambaBase, PluggableLayer):
# state_indices_tensor_d is assigned when attn_metadata is not None,
# and has_decode is only True when attn_metadata is not None
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, block_idx_last_computed_token_d.unsqueeze(1)
).squeeze(1)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, block_idx_last_scheduled_token_d.unsqueeze(1)
).squeeze(1)
else:
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
# 2. Convolution sequence transformation
conv_out_d = causal_conv1d_update(
hidden_states_BC_d.transpose(0, 1),
@@ -414,8 +363,6 @@ class MambaMixer(MambaBase, PluggableLayer):
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
initial_state_idx=block_idx_last_computed_token_d,
).transpose(0, 1)
# 3. State Space Model sequence transformation.
@@ -437,8 +384,8 @@ class MambaMixer(MambaBase, PluggableLayer):
time_proj_bias,
z=gate_d.transpose(0, 1),
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=scan_outputs_d,
)
scan_outputs_d = scan_outputs_d.transpose(0, 1)
+126 -198
View File
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
commit_replayssm_ring_trackers,
reset_replayssm_ring_trackers,
selective_state_update,
selective_state_update_replayssm_flashinfer,
@@ -69,6 +70,13 @@ from vllm.v1.kv_cache_interface import KVCacheGroupSpec
logger = init_logger(__name__)
def _view_mtp_decode_tensor(
tensor: torch.Tensor, decode_batch: int, spec_query_len: int
) -> torch.Tensor:
return tensor.view(decode_batch, spec_query_len, *tensor.shape[1:])
# Added by the IBM Team, 2024
@@ -232,13 +240,14 @@ def mamba_v2_sharded_weight_loader(
# - the ignore is for a mundane mypy error as it does not
# seem to handle slices well.
# https://github.com/python/mypy/issues/2410
target_slice = param.data[boundary : (boundary + take), ...]
param.data[
boundary : (boundary + take), ... # type: ignore[misc]
] = loaded_weight[
loaded_start_idx : (
loaded_start_idx + take
) # type: ignore[misc]
] # type: ignore[misc]
].view_as(target_slice)
# move indexing boundaries
boundary += shard_size
@@ -440,17 +449,18 @@ class MambaMixer2(MambaBase, PluggableLayer):
tp_rank,
)
# Apply the custom weight loader to in_proj.weight
# Apply the custom weight loader to in_proj.weight and its scales.
# Works for both non-quantized (Parameter) and quantized
# (ModelWeightParameter which extends BasevLLMParameter)
if isinstance(self.in_proj.weight, BasevLLMParameter):
# For BasevLLMParameter subclasses (quantized layers like FP8)
# These have a weight_loader property that can be directly set
self.in_proj.weight.weight_loader = mamba_loader
else:
# For standard Parameter (non-quantized layers)
delattr(self.in_proj.weight, "weight_loader")
set_weight_attrs(self.in_proj.weight, {"weight_loader": mamba_loader})
for attr_name in ("weight", "weight_scale", "input_scale"):
param = getattr(self.in_proj, attr_name, None)
if param is not None:
if isinstance(param, BasevLLMParameter):
param.weight_loader = mamba_loader
else:
if hasattr(param, "weight_loader"):
delattr(param, "weight_loader")
set_weight_attrs(param, {"weight_loader": mamba_loader})
# unsqueeze to fit conv1d weights shape into the linear weights shape.
# Can't do this in `weight_loader` since it already exists in
@@ -534,15 +544,11 @@ class MambaMixer2(MambaBase, PluggableLayer):
self.kv_cache = tuple(torch.tensor([]) for _ in range(_n_state))
self._replayssm_ring_start = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_query_len = torch.empty(0, dtype=torch.int32)
self._commits_replayssm_trackers = True
self._updates_replayssm_trackers = True
self.num_spec = vllm_config.num_speculative_tokens
if self.num_spec > 0:
self.register_buffer(
"_decode_state_offsets",
torch.arange(1 + self.num_spec, dtype=torch.int32).unsqueeze(0),
persistent=False,
)
# Pre-compute sizes for forward pass
self.tped_intermediate_size = self.intermediate_size // self.tp_size
@@ -720,9 +726,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
ring_start = prev_num_accepted = None
use_spec_decode = self.num_spec > 0
ring_start = prev_num_accepted = prev_query_len = None
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
@@ -744,6 +749,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
if self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
ring_start = self._replayssm_ring_start
prev_num_accepted = self._replayssm_prev_num_accepted
prev_query_len = self._replayssm_prev_query_len
else:
x_cache = dt_cache = B_cache = None
has_initial_states_p = attn_metadata.has_initial_states_p
@@ -755,6 +761,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
last_chunk_indices_p = attn_metadata.last_chunk_indices_p
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
replayssm_state_indices_d = attn_metadata.replayssm_state_indices_d
num_accepted_tokens = attn_metadata.num_accepted_tokens
query_start_loc_d = attn_metadata.query_start_loc_d
num_decodes = attn_metadata.num_decodes
@@ -788,45 +795,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
dim=0,
)
if is_mamba_cache_all:
# If prefix caching is enabled, retrieve the relevant variables
# for prefill and decode
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
torch.split(
attn_metadata.block_idx_last_computed_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
torch.split(
attn_metadata.block_idx_last_scheduled_token,
[num_decodes, num_prefills],
dim=0,
)
)
if attn_metadata.block_idx_last_scheduled_token_prev_step is not None:
block_idx_last_scheduled_token_prev_step_d, _ = torch.split(
attn_metadata.block_idx_last_scheduled_token_prev_step,
[num_decodes, num_prefills],
dim=0,
)
else:
block_idx_last_scheduled_token_prev_step_d = None
# Prefill-only variables:
block_idx_first_scheduled_token_p = (
attn_metadata.block_idx_first_scheduled_token_p
)
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
else:
block_idx_last_computed_token_p = None
block_idx_last_scheduled_token_p = None
block_idx_first_scheduled_token_p = None
block_idx_last_scheduled_token_d = None
block_idx_last_computed_token_d = None
block_idx_last_scheduled_token_prev_step_d = None
num_computed_tokens_p = None
preallocated_ssm_out_d, preallocated_ssm_out_p = torch.split(
output[:num_actual_tokens],
[num_decode_tokens, num_prefill_tokens],
@@ -843,11 +811,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
# to by "state_indices_tensor_p".
# In particular, it will always write the state at the
# sequence end.
# In addition, "block_idx_first_scheduled_token_p" and
# "block_idx_last_scheduled_token_p"
# are provided (which are pointers into
# "state_indices_tensor_p"), it will write additional cache
# states aligned at "block_size_to_align".
x = hidden_states_B_C_p.transpose(
0, 1
) # this is the form that causal-conv see
@@ -859,11 +822,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
conv_states=conv_state,
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
query_start_loc=query_start_loc_p,
).transpose(0, 1)[:num_prefill_tokens]
@@ -876,14 +834,9 @@ class MambaMixer2(MambaBase, PluggableLayer):
initial_states = None
if has_initial_states_p is not None and prep_initial_states:
assert state_indices_tensor_p is not None
kernel_ssm_indices = state_indices_tensor_p
if is_mamba_cache_all:
kernel_ssm_indices = state_indices_tensor_p.gather(
1, block_idx_last_computed_token_p.unsqueeze(1)
).squeeze(1)
initial_states = torch.where(
has_initial_states_p[:, None, None, None],
ssm_state[kernel_ssm_indices],
ssm_state[state_indices_tensor_p],
0,
)
@@ -906,128 +859,56 @@ class MambaMixer2(MambaBase, PluggableLayer):
cu_chunk_seqlens=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
initial_states=initial_states,
return_intermediate_states=is_mamba_cache_all,
dt_softplus=True,
dt_limit=(0.0, float("inf")),
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
state_dtype=ssm_state.dtype,
)
if is_mamba_cache_all:
assert mamba_block_size is not None
assert state_indices_tensor_p is not None
assert block_idx_first_scheduled_token_p is not None
assert block_idx_last_scheduled_token_p is not None
assert last_chunk_indices_p is not None
assert num_computed_tokens_p is not None
# The chunk_stride is the number of chunks per mamba block
# e.g., if mamba_block_size = 512 and chunk_size = 256,
# then chunk_stride = 2
chunk_stride = mamba_block_size // chunk_size
# Save state for sequences with more than just final state,
# batched over sequences so that the number of kernel launches
# does not scale with the number of prefills.
n_blocks_to_fill = (
block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p
).clamp(min=0)
total_fill = attn_metadata.num_state_writes_p
if total_fill > 0:
first_chunk = torch.zeros_like(last_chunk_indices_p)
first_chunk[1:] = last_chunk_indices_p[:-1] + 1
# The chunk that completes this sequence's first mamba block.
# A block spans chunk_stride chunks. When the sequence resumed
# mid-block, the chunks before this step are not in varlen_states,
# so subtract them: the boundary sits earlier by their count.
num_unaligned_computed_tokens = (
num_computed_tokens_p % mamba_block_size
)
unaligned_chunk_shift = num_unaligned_computed_tokens // chunk_size
first_aligned_chunk = (
first_chunk + (chunk_stride - 1) - unaligned_chunk_shift
)
# Flatten all writes into one list: fill_counts=[2,1,2]
# -> seq_ids=[0,0,1,2,2], write_offsets=[0,1,0,0,1]
fill_counts = n_blocks_to_fill.long()
# output_size, or repeat_interleave reads the size off the device
seq_ids = torch.repeat_interleave(
torch.arange(num_prefills, device=ssm_state.device),
fill_counts,
output_size=total_fill,
)
write_starts = torch.cumsum(fill_counts, dim=0) - fill_counts
write_offsets = torch.arange(
total_fill, device=ssm_state.device
) - torch.repeat_interleave(
write_starts, fill_counts, output_size=total_fill
)
cache_blocks_to_fill = state_indices_tensor_p[
seq_ids,
block_idx_first_scheduled_token_p[seq_ids] + write_offsets,
]
# Write the states. Each row goes to its own cache block,
# so the indexed copy has no duplicate destinations.
ssm_state[cache_blocks_to_fill] = varlen_states[
first_aligned_chunk[seq_ids] + write_offsets * chunk_stride
]
# For all seqs, store the last state (note: might be partial):
assert state_indices_tensor_p is not None
ssm_state[
state_indices_tensor_p.gather(
1, block_idx_last_scheduled_token_p.unsqueeze(1)
).squeeze(1)
] = varlen_states[last_chunk_indices_p]
else:
# update ssm states
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
state_indices_tensor_p,
)
# update ssm states
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_indices_tensor_p,
)
# Process decode requests
if has_decode:
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
if self.num_spec > 0:
assert block_idx_last_scheduled_token_prev_step_d is not None
input_indices = (
block_idx_last_scheduled_token_prev_step_d.unsqueeze(1)
+ self._decode_state_offsets
)
output_indices = (
block_idx_last_scheduled_token_d.unsqueeze(1)
+ self._decode_state_offsets
)
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, input_indices
)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, output_indices
)
else:
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, block_idx_last_computed_token_d.unsqueeze(1)
).squeeze(1)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, block_idx_last_scheduled_token_d.unsqueeze(1)
).squeeze(1)
else:
# Without caching, read and write in-place to the same blocks:
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
if (
self.use_replayssm
and self.mamba_config.backend == MambaBackendEnum.FLASHINFER
and use_spec_decode
and self._commits_replayssm_trackers
):
assert ring_start is not None
assert prev_num_accepted is not None
assert prev_query_len is not None
assert replayssm_state_indices_d is not None
assert num_accepted_tokens is not None
assert query_start_loc_d is not None
assert x_cache is not None
assert self.replayssm_buffer_len is not None
# The previous MTP query's accepted prefix is known only now.
# Commit it once, before this cache group's layers evaluate the
# current speculative query with the shared ring positions.
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
replayssm_state_indices_d,
num_accepted_tokens,
query_start_loc_d,
logical_window=self.replayssm_buffer_len,
ring_buffer_len=x_cache.size(2),
)
# 2. Convolution sequence transformation
hidden_states_B_C_d = causal_conv1d_update(
@@ -1037,11 +918,15 @@ class MambaMixer2(MambaBase, PluggableLayer):
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
initial_state_idx=block_idx_last_computed_token_d,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc_d,
max_query_len=state_indices_tensor_d.size(-1),
# ReplaySSM keeps one physical state block while a speculative
# decode call still processes the full target + draft window.
max_query_len=(
1 + self.num_spec
if self.use_replayssm and use_spec_decode
else state_indices_tensor_d.size(-1)
),
)
hidden_states_d, B_d, C_d = self.split_hidden_states_B_C_fn(
@@ -1077,7 +962,35 @@ class MambaMixer2(MambaBase, PluggableLayer):
if self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
assert ring_start is not None
assert prev_num_accepted is not None
assert prev_query_len is not None
assert attn_metadata.replayssm_scratch is not None
fi_cu_seqlens = query_start_loc_d
fi_max_seqlen = None
if use_spec_decode:
spec_query_len = 1 + self.num_spec
fi_max_seqlen = spec_query_len
assert replayssm_state_indices_d is not None
decode_batch = replayssm_state_indices_d.size(0)
if num_decode_tokens == decode_batch * spec_query_len:
hidden_states_d = _view_mtp_decode_tensor(
hidden_states_d, decode_batch, spec_query_len
)
dt_d = _view_mtp_decode_tensor(
dt_d, decode_batch, spec_query_len
)
B_d = _view_mtp_decode_tensor(
B_d, decode_batch, spec_query_len
)
C_d = _view_mtp_decode_tensor(
C_d, decode_batch, spec_query_len
)
preallocated_ssm_out_d = _view_mtp_decode_tensor(
preallocated_ssm_out_d,
decode_batch,
spec_query_len,
)
fi_cu_seqlens = None
fi_max_seqlen = None
selective_state_update_replayssm_flashinfer(
ssm_state,
hidden_states_d,
@@ -1091,19 +1004,25 @@ class MambaMixer2(MambaBase, PluggableLayer):
dt_cache,
ring_start,
prev_num_accepted,
prev_query_len,
logical_window=self.replayssm_buffer_len,
D=D_d,
dt_bias=dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
state_batch_indices=replayssm_state_indices_d,
scratch=attn_metadata.replayssm_scratch,
update_trackers=self._updates_replayssm_trackers,
update_trackers=(
self._updates_replayssm_trackers and not use_spec_decode
),
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
),
stochastic_rounding_philox_rounds=(
self.mamba_config.stochastic_rounding_philox_rounds
),
cu_seqlens=fi_cu_seqlens,
max_seqlen=fi_max_seqlen,
enable_pdl=False,
)
else:
selective_state_update_replayssm_output_only(
@@ -1123,7 +1042,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
write_pos=attn_metadata.write_pos_d,
is_flush=attn_metadata.is_flush_d,
max_cache_len=self.replayssm_buffer_len,
state_batch_indices=state_indices_tensor_d_input,
state_batch_indices=state_indices_tensor_d,
out=preallocated_ssm_out_d,
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
@@ -1143,8 +1062,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
D_d,
dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=preallocated_ssm_out_d,
num_accepted_tokens=num_accepted_tokens,
cu_seqlens=query_start_loc_d,
@@ -1185,6 +1104,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
tp_world_size=tp_world_size,
logical_window=self.replayssm_buffer_len,
backend=self.mamba_config.backend,
num_speculative_tokens=self.num_spec,
)
return base_shape
@@ -1202,9 +1122,10 @@ def share_replayssm_ring_trackers(
Layers backed by one KV-cache group use the same physical block indices and
can therefore share cursors. Different KV-cache groups may assign different
block indices to the same request and must keep separate cursor tensors.
The final local layer in each group advances its cursors after every layer
in that group has consumed the previous values.
block indices to the same request and must keep separate cursor tensors. For
speculative decode, the first local layer commits the preceding acceptance;
for standard decode, the final local layer advances after all layers consume
the previous values.
"""
replayssm_mixers: dict[str, MambaMixer2] = {}
for layer_name in ordered_layer_names:
@@ -1230,6 +1151,7 @@ def share_replayssm_ring_trackers(
groups_by_namespace.setdefault(namespace, []).append(layer_name)
for group_layer_names in groups_by_namespace.values():
first_layer_name = group_layer_names[0]
last_layer_name = group_layer_names[-1]
first_mixer = replayssm_mixers[group_layer_names[0]]
@@ -1244,11 +1166,17 @@ def share_replayssm_ring_trackers(
ring_start = torch.zeros(num_blocks, dtype=torch.int32, device=device)
prev_num_accepted = torch.zeros_like(ring_start)
prev_query_len = torch.zeros_like(ring_start)
for layer_name in group_layer_names:
mixer = replayssm_mixers[layer_name]
mixer._replayssm_ring_start = ring_start
mixer._replayssm_prev_num_accepted = prev_num_accepted
mixer._updates_replayssm_trackers = layer_name == last_layer_name
mixer._replayssm_prev_query_len = prev_query_len
mixer._commits_replayssm_trackers = False
mixer._updates_replayssm_trackers = False
replayssm_mixers[first_layer_name]._commits_replayssm_trackers = True
replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True
def mamba_mixer2(
@@ -227,6 +227,7 @@ class MambaStateShapeCalculator:
tp_world_size: int,
logical_window: int,
backend: MambaBackendEnum,
num_speculative_tokens: int = 0,
) -> tuple[
tuple[int, int],
tuple[int, int, int],
@@ -241,8 +242,8 @@ class MambaStateShapeCalculator:
"""
ring_buffer_len = logical_window
if backend == MambaBackendEnum.FLASHINFER:
# FlashInfer keeps the live window and appended token together.
ring_buffer_len += 1
# FlashInfer keeps the live window and current verify window together.
ring_buffer_len += 1 + num_speculative_tokens
local_nheads, head_dim, state_size = base_shapes[1]
local_ngroups = divide(n_groups, tp_world_size)
return (
@@ -14,7 +14,6 @@ from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
)
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_cpu,
causal_conv1d_update_torch,
)
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import (
@@ -56,7 +55,7 @@ def _causal_conv1d_fwd_cpu(
cache_indices: torch.Tensor,
has_initial_state: torch.Tensor,
silu_activation: bool,
is_vnni: bool,
is_weight_packed: bool,
) -> torch.Tensor:
if not is_conv_state_dim_first():
return ops.causal_conv1d_fwd_cpu(
@@ -68,7 +67,7 @@ def _causal_conv1d_fwd_cpu(
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=silu_activation,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
scratch, identity_indices = _stage_ds_conv_states(conv_states, cache_indices)
@@ -81,7 +80,7 @@ def _causal_conv1d_fwd_cpu(
cache_indices=identity_indices,
has_initial_state=has_initial_state,
silu_activation=silu_activation,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
_scatter_ds_conv_states(conv_states, cache_indices, scratch)
return out
@@ -95,7 +94,7 @@ def _causal_conv1d_update_cpu(
bias: torch.Tensor | None,
silu_activation: bool,
conv_state_indices: torch.Tensor,
is_vnni: bool,
is_weight_packed: bool,
num_accepted_tokens: torch.Tensor | None = None,
) -> torch.Tensor:
if not is_conv_state_dim_first():
@@ -106,7 +105,7 @@ def _causal_conv1d_update_cpu(
bias=bias,
silu_activation=silu_activation,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
num_accepted_tokens=num_accepted_tokens,
)
@@ -118,13 +117,20 @@ def _causal_conv1d_update_cpu(
bias=bias,
silu_activation=silu_activation,
conv_state_indices=identity_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
num_accepted_tokens=num_accepted_tokens,
)
_scatter_ds_conv_states(conv_states, conv_state_indices, scratch)
return out
def is_arm_bf16() -> bool:
return (
current_platform.get_cpu_architecture() == CpuArchEnum.ARM
and torch.cpu.get_capabilities().get("bf16", False)
)
def cpu_gdn_attention_core(
mixed_qkv: torch.Tensor,
b: torch.Tensor,
@@ -201,9 +207,10 @@ def _cpu_gdn_attention_nonspec(
assert state_indices_tensor is not None
assert query_start_loc is not None
# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any
# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time.
use_cpp_conv = torch.cpu._is_avx512_bf16_supported()
# C++ conv (conv.cpp) runs on aarch64 with bf16 support and on x86 uses VDPBF16PS,
# not AMX tiles, so it runs on any AVX-512BF16 CPU. The weights are packed on this
# same predicate at load time.
use_cpp_conv = torch.cpu._is_avx512_bf16_supported() or is_arm_bf16()
conv_state = layer.kv_cache[0]
if use_cpp_conv:
@@ -244,28 +251,17 @@ def _cpu_gdn_attention_nonspec(
bias=layer.conv1d.bias,
silu_activation=(layer.activation == "silu"),
conv_state_indices=decode_state_indices,
is_vnni=True,
is_weight_packed=True,
)
else:
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
decode_conv_state = conv_state[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state[decode_state_indices] = decode_conv_state
else:
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
@@ -315,7 +311,7 @@ def _cpu_gdn_attention_nonspec(
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
is_weight_packed=True,
).transpose(0, 1)
else:
prefill_mixed_qkv = causal_conv1d_torch(
@@ -383,9 +379,9 @@ def _ssm_state_view(layer) -> torch.Tensor:
def _unpacked_conv_weight(layer) -> torch.Tensor:
"""Return the plain (dim, width) conv weight.
On AMX the conv1d weight is VNNI-packed in place at load time (only usable
by the AMX C++ kernel), so the torch spec-decode path relies on the
un-packed copy stashed by ``dispatch_cpu_unquantized_gemm``.
On AMX and aarch64+bf16 the conv1d weight is packed in place at load time,
so the torch spec-decode path relies on the un-packed copy stashed by
``dispatch_cpu_unquantized_gemm``.
"""
w = getattr(layer.conv1d, "_cpu_unpacked_conv_weight", None)
if w is not None:
@@ -505,13 +501,15 @@ def _spec_forward(
seq_starts = spec_qsl_cpu[:-1]
seq_lens = spec_qsl_cpu[1:] - spec_qsl_cpu[:-1]
is_amx = torch.cpu._is_amx_tile_supported()
# ---- 1. Convolution (per-sequence rolling buffer) ----
dim = mixed_qkv_spec.size(-1)
bias = layer.conv1d.bias
silu = layer.activation == "silu"
can_use_native_conv = (
torch.cpu._is_amx_tile_supported()
(is_amx or is_arm_bf16())
and width == 4
and num_spec_decodes > 0
and bool(torch.all(seq_lens == seq_lens[0]).item())
@@ -528,7 +526,7 @@ def _spec_forward(
conv_state_indices=spec_state_indices[:num_spec_decodes, 0]
.to("cpu", torch.int32)
.contiguous(),
is_vnni=True,
is_weight_packed=True,
num_accepted_tokens=num_accepted[:num_spec_decodes].to("cpu", torch.int32),
).view_as(mixed_qkv_spec)
else:
@@ -603,9 +601,9 @@ def _spec_aware_nonspec(
assert query_start_loc is not None
state_indices_tensor = state_indices_tensor.contiguous()
is_amx = torch.cpu._is_amx_tile_supported()
can_use_native_conv = torch.cpu._is_amx_tile_supported() or is_arm_bf16()
if not is_amx:
if not can_use_native_conv:
conv_weights = _unpacked_conv_weight(layer)
num_decodes = attn_metadata_i.num_decodes
@@ -618,7 +616,8 @@ def _spec_aware_nonspec(
decode_b = b[:num_decode_tokens]
decode_a = a[:num_decode_tokens]
decode_state_indices = state_indices_tensor[:num_decodes]
if is_amx:
if can_use_native_conv:
decode_mixed_qkv = _causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_states=conv_buf,
@@ -626,30 +625,19 @@ def _spec_aware_nonspec(
bias=layer.conv1d.bias,
silu_activation=layer.activation == "silu",
conv_state_indices=decode_state_indices,
is_vnni=True,
is_weight_packed=True,
)
else:
# Only the first ``width-1`` columns hold the real conv state.
conv_state_view = conv_buf[:, :, : width - 1]
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
decode_conv_state = conv_state_view[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state_view[decode_state_indices] = decode_conv_state
else:
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state_view,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state_view,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
attn_out = ops.fused_sigmoid_gating_delta_rule_update_cpu(
@@ -685,7 +673,8 @@ def _spec_aware_nonspec(
prefill_has_initial_state = has_initial_state[
num_decodes : num_decodes + num_prefills
]
if is_amx:
if can_use_native_conv:
prefill_mixed_qkv = _causal_conv1d_fwd_cpu(
x=prefill_mixed_qkv.transpose(0, 1),
weight=layer.conv1d.weight,
@@ -695,7 +684,7 @@ def _spec_aware_nonspec(
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
is_weight_packed=True,
).transpose(0, 1)
else:
prefill_mixed_qkv = causal_conv1d_torch(
@@ -757,8 +746,9 @@ def _spec_aware_nonspec_subset(
prefill_state_indices = prefill_state_indices.contiguous()
is_amx = torch.cpu._is_amx_tile_supported()
can_use_native_conv = is_amx or is_arm_bf16()
if is_amx:
if can_use_native_conv:
conv_out = _causal_conv1d_fwd_cpu(
x=mixed_qkv.transpose(0, 1),
weight=layer.conv1d.weight,
@@ -768,7 +758,7 @@ def _spec_aware_nonspec_subset(
cache_indices=prefill_state_indices,
has_initial_state=has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
is_weight_packed=True,
).transpose(0, 1)
else:
conv_weights = _unpacked_conv_weight(layer)
@@ -31,6 +31,7 @@ logger = init_logger(__name__)
def _update_replayssm_ring_trackers_kernel(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
state_batch_indices_stride,
n_slots,
@@ -52,6 +53,9 @@ def _update_replayssm_ring_trackers_kernel(
if RESET:
tl.store(ring_start + slots, 0, mask=valid)
tl.store(prev_num_accepted + slots, 0, mask=valid)
# MTP alone reads prev_query_len. Standard decode deliberately leaves
# it unchanged; reset clears the pending MTP window after prefill.
tl.store(prev_query_len + slots, 0, mask=valid)
else:
prev = tl.load(prev_num_accepted + slots, mask=valid, other=0)
start = tl.load(ring_start + slots, mask=valid, other=0)
@@ -66,9 +70,70 @@ def _update_replayssm_ring_trackers_kernel(
tl.store(prev_num_accepted + slots, next_prev, mask=valid)
@triton.jit(
do_not_specialize=["n_slots", "state_batch_indices_stride"],
do_not_specialize_on_alignment=["state_batch_indices"],
)
def _commit_replayssm_ring_trackers_kernel(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
num_accepted_tokens,
query_start_loc,
state_batch_indices_stride,
n_slots,
num_states,
logical_window: tl.constexpr,
ring_buffer_len: tl.constexpr,
pad_slot_id: tl.constexpr,
BLOCK: tl.constexpr,
) -> None:
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < n_slots
slots = tl.load(
state_batch_indices + offsets * state_batch_indices_stride,
mask=mask,
other=pad_slot_id,
)
valid = mask & (slots != pad_slot_id) & (slots >= 0) & (slots < num_states)
accepted_since_checkpoint = tl.load(prev_num_accepted + slots, mask=valid, other=0)
current_ring_start = tl.load(ring_start + slots, mask=valid, other=0)
previous_speculative_query_len = tl.load(
prev_query_len + slots, mask=valid, other=0
)
accepted_from_previous_query = tl.load(
num_accepted_tokens + offsets, mask=mask, other=0
)
must_checkpoint = (previous_speculative_query_len > 0) & (
accepted_since_checkpoint + previous_speculative_query_len > logical_window
)
next_start = tl.where(
must_checkpoint,
(current_ring_start + accepted_since_checkpoint) % ring_buffer_len,
current_ring_start,
)
next_prev = tl.where(
previous_speculative_query_len == 0,
0,
tl.where(
must_checkpoint,
accepted_from_previous_query,
accepted_since_checkpoint + accepted_from_previous_query,
),
)
current_speculative_query_len = tl.load(
query_start_loc + offsets + 1, mask=mask, other=0
) - tl.load(query_start_loc + offsets, mask=mask, other=0)
tl.store(ring_start + slots, next_start, mask=valid)
tl.store(prev_num_accepted + slots, next_prev, mask=valid)
tl.store(prev_query_len + slots, current_speculative_query_len, mask=valid)
def update_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
logical_window: int | None = None,
ring_buffer_len: int | None = None,
@@ -90,10 +155,15 @@ def update_replayssm_ring_trackers(
_update_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)](
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
state_batch_indices.stride(0),
n_slots,
min(ring_start.numel(), prev_num_accepted.numel()),
min(
ring_start.numel(),
prev_num_accepted.numel(),
prev_query_len.numel(),
),
logical_window,
ring_buffer_len,
pad_slot_id,
@@ -105,6 +175,7 @@ def update_replayssm_ring_trackers(
def reset_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
@@ -112,11 +183,62 @@ def reset_replayssm_ring_trackers(
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
pad_slot_id=pad_slot_id,
)
def commit_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
num_accepted_tokens: torch.Tensor,
query_start_loc: torch.Tensor,
logical_window: int,
ring_buffer_len: int,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
"""Commit the preceding speculative window and record the current one.
MTP evaluates a target token and its draft tokens together, but the number
accepted from that query is available only on the next forward pass. This
function then advances the per-request ring by the accepted prefix and
records the current query length for the following pass. A zero
``prev_query_len`` means that reset/prefill left no prior MTP query to
commit; slot validity is handled independently by the kernel mask.
Standard single-token decode needs no delayed commit: its ReplaySSM kernel
advances the shared ring trackers directly after every token.
"""
if state_batch_indices.dim() > 1:
state_batch_indices = state_batch_indices[:, 0]
n_slots = state_batch_indices.numel()
if n_slots == 0:
return
block = 128
_commit_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)](
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
num_accepted_tokens,
query_start_loc,
state_batch_indices.stride(0),
n_slots,
min(
ring_start.numel(),
prev_num_accepted.numel(),
prev_query_len.numel(),
),
logical_window,
ring_buffer_len,
pad_slot_id,
BLOCK=block,
)
class MambaSSUBackend(ABC):
"""Abstract base class for Mamba SSU backends."""
@@ -385,6 +507,7 @@ def selective_state_update_replayssm_flashinfer(
dt_cache: torch.Tensor,
ring_start: torch.Tensor,
prev_num_accepted_tokens: torch.Tensor,
prev_query_len: torch.Tensor,
logical_window: int,
D: torch.Tensor | None = None,
dt_bias: torch.Tensor | None = None,
@@ -395,6 +518,9 @@ def selective_state_update_replayssm_flashinfer(
update_trackers: bool = True,
enable_stochastic_rounding: bool = False,
stochastic_rounding_philox_rounds: int = 0,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: int | None = None,
enable_pdl: bool = False,
) -> torch.Tensor:
"""Run FlashInfer checkpointing SSU and optionally advance shared trackers."""
if _flashinfer_replayssm_kernel is None:
@@ -404,11 +530,12 @@ def selective_state_update_replayssm_flashinfer(
)
if x.dim() == 3:
x = x.unsqueeze(1)
dt = dt.unsqueeze(1)
B = B.unsqueeze(1)
C = C.unsqueeze(1)
out = out.unsqueeze(1)
dim = 0 if cu_seqlens is not None else 1
x = x.unsqueeze(dim)
dt = dt.unsqueeze(dim)
B = B.unsqueeze(dim)
C = C.unsqueeze(dim)
out = out.unsqueeze(dim)
indices = state_batch_indices
if indices is not None and indices.dim() > 1:
@@ -443,6 +570,9 @@ def selective_state_update_replayssm_flashinfer(
pad_slot_id=null_block_id,
rand_seed=rand_seed,
philox_rounds=stochastic_rounding_philox_rounds or 10,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
enable_pdl=enable_pdl,
cb_scaled=cb_scaled,
cumAdt_vec=cumAdt_vec,
cb_old=cb_old,
@@ -451,6 +581,7 @@ def selective_state_update_replayssm_flashinfer(
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted_tokens,
prev_query_len,
indices,
logical_window=logical_window,
ring_buffer_len=x_cache.size(2),