[Core] Remove deprecated mamba_cache_mode "all" (#58997)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Thomas Parnell <tpa@zurich.ibm.com>
This commit is contained in:
Jiangyun Zhu
2026-10-01 02:59:37 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Thomas Parnell
parent 08d77cad7b
commit cb6458c5ce
45 changed files with 201 additions and 1476 deletions
@@ -260,8 +260,6 @@ class MambaMixer(MambaBase, PluggableLayer):
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
@@ -297,8 +295,6 @@ class MambaMixer(MambaBase, PluggableLayer):
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
num_decode_tokens = attn_metadata.num_decode_tokens
num_prefills = attn_metadata.num_prefills # request count
num_decodes = attn_metadata.num_decode_tokens # token count (=request)
has_prefill = num_prefill_tokens > 0
has_decode = num_decode_tokens > 0
num_actual_tokens = num_prefill_tokens + num_decode_tokens
@@ -314,34 +310,6 @@ class MambaMixer(MambaBase, PluggableLayer):
gate_p = prefill_decode_split.gate_p
gate_d = prefill_decode_split.gate_d
if is_mamba_cache_all:
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
torch.split(
attn_metadata.block_idx_last_computed_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
torch.split(
attn_metadata.block_idx_last_scheduled_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_first_scheduled_token_p = (
attn_metadata.block_idx_first_scheduled_token_p
)
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
else:
block_idx_last_computed_token_d = None
block_idx_last_computed_token_p = None
block_idx_last_scheduled_token_d = None
block_idx_last_scheduled_token_p = None
block_idx_first_scheduled_token_p = None
num_computed_tokens_p = None
ssm_outputs = []
if has_prefill:
@@ -355,11 +323,6 @@ class MambaMixer(MambaBase, PluggableLayer):
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
query_start_loc=query_start_loc_p,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
)
# 3. State Space Model sequence transformations.
@@ -383,10 +346,6 @@ class MambaMixer(MambaBase, PluggableLayer):
cache_indices=state_indices_tensor_p,
has_initial_state=has_initial_states_p,
query_start_loc=query_start_loc_p,
block_size=mamba_block_size,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
cu_chunk_seqlen=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
)
@@ -396,16 +355,6 @@ class MambaMixer(MambaBase, PluggableLayer):
# state_indices_tensor_d is assigned when attn_metadata is not None,
# and has_decode is only True when attn_metadata is not None
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, block_idx_last_computed_token_d.unsqueeze(1)
).squeeze(1)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, block_idx_last_scheduled_token_d.unsqueeze(1)
).squeeze(1)
else:
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
# 2. Convolution sequence transformation
conv_out_d = causal_conv1d_update(
hidden_states_BC_d.transpose(0, 1),
@@ -414,8 +363,6 @@ class MambaMixer(MambaBase, PluggableLayer):
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
initial_state_idx=block_idx_last_computed_token_d,
).transpose(0, 1)
# 3. State Space Model sequence transformation.
@@ -437,8 +384,8 @@ class MambaMixer(MambaBase, PluggableLayer):
time_proj_bias,
z=gate_d.transpose(0, 1),
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=scan_outputs_d,
)
scan_outputs_d = scan_outputs_d.transpose(0, 1)
+17 -182
View File
@@ -549,12 +549,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
self._updates_replayssm_trackers = True
self.num_spec = vllm_config.num_speculative_tokens
if self.num_spec > 0:
self.register_buffer(
"_decode_state_offsets",
torch.arange(1 + self.num_spec, dtype=torch.int32).unsqueeze(0),
persistent=False,
)
# Pre-compute sizes for forward pass
self.tped_intermediate_size = self.intermediate_size // self.tp_size
@@ -732,8 +726,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
use_spec_decode = self.num_spec > 0
ring_start = prev_num_accepted = prev_query_len = None
@@ -803,45 +795,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
dim=0,
)
if is_mamba_cache_all:
# If prefix caching is enabled, retrieve the relevant variables
# for prefill and decode
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
torch.split(
attn_metadata.block_idx_last_computed_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
torch.split(
attn_metadata.block_idx_last_scheduled_token,
[num_decodes, num_prefills],
dim=0,
)
)
if attn_metadata.block_idx_last_scheduled_token_prev_step is not None:
block_idx_last_scheduled_token_prev_step_d, _ = torch.split(
attn_metadata.block_idx_last_scheduled_token_prev_step,
[num_decodes, num_prefills],
dim=0,
)
else:
block_idx_last_scheduled_token_prev_step_d = None
# Prefill-only variables:
block_idx_first_scheduled_token_p = (
attn_metadata.block_idx_first_scheduled_token_p
)
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
else:
block_idx_last_computed_token_p = None
block_idx_last_scheduled_token_p = None
block_idx_first_scheduled_token_p = None
block_idx_last_scheduled_token_d = None
block_idx_last_computed_token_d = None
block_idx_last_scheduled_token_prev_step_d = None
num_computed_tokens_p = None
preallocated_ssm_out_d, preallocated_ssm_out_p = torch.split(
output[:num_actual_tokens],
[num_decode_tokens, num_prefill_tokens],
@@ -858,11 +811,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
# to by "state_indices_tensor_p".
# In particular, it will always write the state at the
# sequence end.
# In addition, "block_idx_first_scheduled_token_p" and
# "block_idx_last_scheduled_token_p"
# are provided (which are pointers into
# "state_indices_tensor_p"), it will write additional cache
# states aligned at "block_size_to_align".
x = hidden_states_B_C_p.transpose(
0, 1
) # this is the form that causal-conv see
@@ -874,11 +822,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
conv_states=conv_state,
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
query_start_loc=query_start_loc_p,
).transpose(0, 1)[:num_prefill_tokens]
@@ -891,14 +834,9 @@ class MambaMixer2(MambaBase, PluggableLayer):
initial_states = None
if has_initial_states_p is not None and prep_initial_states:
assert state_indices_tensor_p is not None
kernel_ssm_indices = state_indices_tensor_p
if is_mamba_cache_all:
kernel_ssm_indices = state_indices_tensor_p.gather(
1, block_idx_last_computed_token_p.unsqueeze(1)
).squeeze(1)
initial_states = torch.where(
has_initial_states_p[:, None, None, None],
ssm_state[kernel_ssm_indices],
ssm_state[state_indices_tensor_p],
0,
)
@@ -921,130 +859,29 @@ class MambaMixer2(MambaBase, PluggableLayer):
cu_chunk_seqlens=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
initial_states=initial_states,
return_intermediate_states=is_mamba_cache_all,
dt_softplus=True,
dt_limit=(0.0, float("inf")),
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
state_dtype=ssm_state.dtype,
)
if is_mamba_cache_all:
assert mamba_block_size is not None
assert state_indices_tensor_p is not None
assert block_idx_first_scheduled_token_p is not None
assert block_idx_last_scheduled_token_p is not None
assert last_chunk_indices_p is not None
assert num_computed_tokens_p is not None
# The chunk_stride is the number of chunks per mamba block
# e.g., if mamba_block_size = 512 and chunk_size = 256,
# then chunk_stride = 2
chunk_stride = mamba_block_size // chunk_size
# Save state for sequences with more than just final state,
# batched over sequences so that the number of kernel launches
# does not scale with the number of prefills.
n_blocks_to_fill = (
block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p
).clamp(min=0)
total_fill = attn_metadata.num_state_writes_p
if total_fill > 0:
first_chunk = torch.zeros_like(last_chunk_indices_p)
first_chunk[1:] = last_chunk_indices_p[:-1] + 1
# The chunk that completes this sequence's first mamba block.
# A block spans chunk_stride chunks. When the sequence resumed
# mid-block, the chunks before this step are not in varlen_states,
# so subtract them: the boundary sits earlier by their count.
num_unaligned_computed_tokens = (
num_computed_tokens_p % mamba_block_size
)
unaligned_chunk_shift = num_unaligned_computed_tokens // chunk_size
first_aligned_chunk = (
first_chunk + (chunk_stride - 1) - unaligned_chunk_shift
)
# Flatten all writes into one list: fill_counts=[2,1,2]
# -> seq_ids=[0,0,1,2,2], write_offsets=[0,1,0,0,1]
fill_counts = n_blocks_to_fill.long()
# output_size, or repeat_interleave reads the size off the device
seq_ids = torch.repeat_interleave(
torch.arange(num_prefills, device=ssm_state.device),
fill_counts,
output_size=total_fill,
)
write_starts = torch.cumsum(fill_counts, dim=0) - fill_counts
write_offsets = torch.arange(
total_fill, device=ssm_state.device
) - torch.repeat_interleave(
write_starts, fill_counts, output_size=total_fill
)
cache_blocks_to_fill = state_indices_tensor_p[
seq_ids,
block_idx_first_scheduled_token_p[seq_ids] + write_offsets,
]
# Write the states. Each row goes to its own cache block,
# so the indexed copy has no duplicate destinations.
ssm_state[cache_blocks_to_fill] = varlen_states[
first_aligned_chunk[seq_ids] + write_offsets * chunk_stride
]
# For all seqs, store the last state (note: might be partial):
assert state_indices_tensor_p is not None
ssm_state[
state_indices_tensor_p.gather(
1, block_idx_last_scheduled_token_p.unsqueeze(1)
).squeeze(1)
] = varlen_states[last_chunk_indices_p]
else:
# update ssm states
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_indices_tensor_p,
)
# update ssm states
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_indices_tensor_p,
)
# Process decode requests
if has_decode:
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
if use_spec_decode:
assert block_idx_last_scheduled_token_prev_step_d is not None
input_indices = (
block_idx_last_scheduled_token_prev_step_d.unsqueeze(1)
+ self._decode_state_offsets
)
output_indices = (
block_idx_last_scheduled_token_d.unsqueeze(1)
+ self._decode_state_offsets
)
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, input_indices
)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, output_indices
)
else:
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, block_idx_last_computed_token_d.unsqueeze(1)
).squeeze(1)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, block_idx_last_scheduled_token_d.unsqueeze(1)
).squeeze(1)
else:
# Without caching, read and write in-place to the same blocks:
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
if (
self.use_replayssm
and self.mamba_config.backend == MambaBackendEnum.FLASHINFER
@@ -1081,8 +918,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
initial_state_idx=block_idx_last_computed_token_d,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc_d,
# ReplaySSM keeps one physical state block while a speculative
@@ -1207,7 +1042,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
write_pos=attn_metadata.write_pos_d,
is_flush=attn_metadata.is_flush_d,
max_cache_len=self.replayssm_buffer_len,
state_batch_indices=state_indices_tensor_d_input,
state_batch_indices=state_indices_tensor_d,
out=preallocated_ssm_out_d,
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
@@ -1227,8 +1062,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
D_d,
dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=preallocated_ssm_out_d,
num_accepted_tokens=num_accepted_tokens,
cu_seqlens=query_start_loc_d,
+3 -14
View File
@@ -646,20 +646,9 @@ class MambaModelConfig(VerifyAndUpdateConfig):
cache_config.mamba_cache_mode,
model_config.architecture,
)
if (
cache_config.mamba_cache_mode == "all"
and not model_config.supports_mamba_prefix_caching
):
cache_config.mamba_cache_mode = "align"
logger.warning(
"Hybrid or mamba-based model detected without support "
"for prefix caching with Mamba cache 'all' mode: "
"falling back to 'align' mode."
)
if cache_config.mamba_cache_mode == "align":
assert vllm_config.scheduler_config.enable_chunked_prefill, (
"Chunked prefill is required for mamba cache mode 'align'."
)
assert vllm_config.scheduler_config.enable_chunked_prefill, (
"Chunked prefill is required for mamba cache mode 'align'."
)
# By default, mamba block size will be set to max_model_len (see
# below). When enabling prefix caching, we align mamba block size
# to the block size as the basic granularity for prefix caching.
@@ -418,12 +418,6 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLMBase):
self.quant_config = vllm_config.quant_config
self.scheduler_config = vllm_config.scheduler_config
if vllm_config.cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Intern-S2-Mobius does not support 'all' prefix caching; "
"use '--mamba-cache-mode=align' instead."
)
nn.Module.__init__(self)
self.config = config
self.model = InternS2MobiusModel(
-7
View File
@@ -458,13 +458,6 @@ class Lfm2ForCausalLM(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Lfm2 currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
super().__init__()
self.config = config
self.model = Lfm2Model(
-7
View File
@@ -573,13 +573,6 @@ class Lfm2MoeForCausalLM(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Lfm2Moe currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
super().__init__()
self.config = config
-6
View File
@@ -329,14 +329,8 @@ class Qwen3_5ForCausalLMBase(
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
scheduler_config = vllm_config.scheduler_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3.5 currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
@@ -227,13 +227,6 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3_5MTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
-6
View File
@@ -828,14 +828,8 @@ class Qwen3NextForCausalLM(
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
scheduler_config = vllm_config.scheduler_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3Next currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
@@ -187,13 +187,6 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
config = vllm_config.model_config.hf_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3NextMTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()