Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 17:41:13 +00:00
1122 changed files with 60114 additions and 20956 deletions
+13 -2
View File
@@ -25,6 +25,7 @@ def compute_mamba_prefill_checkpoints(
mamba_block_size: int,
checkpoint_alignment: int | None,
drop_eagle_block: bool,
cache_hit_alignment_tokens: int,
) -> tuple[list[int], list[int]]:
"""Per-row internal prefill checkpoint offsets and cache block columns.
@@ -43,7 +44,7 @@ def compute_mamba_prefill_checkpoints(
for seq_len, query_len in zip(seq_lens, query_lens):
query_start = seq_len - query_len
position = get_mamba_prefill_checkpoint_position(
seq_len, hash_block_size, drop_eagle_block=drop_eagle_block
seq_len, cache_hit_alignment_tokens, drop_eagle_block=drop_eagle_block
)
valid = is_mamba_prefill_checkpoint_valid(
query_start=query_start,
@@ -70,6 +71,9 @@ def _gather_checkpoint_state_indices(
class MambaPrefillCheckpointMetadata:
checkpoint_offsets: torch.Tensor
state_indices: torch.Tensor
# Host copy of ``checkpoint_offsets``, for backends that lay out the
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
offsets: list[int] | None = None
# Gather indices into the block table, kept so that another KV cache group
# with the same spec can re-derive its own ``state_indices``.
request_rows: torch.Tensor | None = None
@@ -103,12 +107,17 @@ class MambaPrefillCheckpointBuilder:
return None
if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0:
return None
cache_config = self.vllm_config.cache_config
hash_block_size = cache_config.hash_block_size
cache_hit_alignment_tokens = cache_config.cache_hit_alignment_tokens
if hash_block_size is None or cache_hit_alignment_tokens is None:
# Unresolved only for the minimal KV cache built for profiling.
return None
assert m.seq_lens_cpu_upper_bound is not None
all_query_lens = m.query_start_loc_cpu.diff().tolist()
query_lens = [all_query_lens[row] for row in request_rows]
seq_lens = m.seq_lens_cpu_upper_bound.tolist()
block_size = self.kv_cache_spec.block_size
hash_block_size = self.vllm_config.cache_config.prefix_match_unit or block_size
speculative_config = self.vllm_config.speculative_config
drop_eagle_block = (
speculative_config is not None and speculative_config.use_eagle_block_drop()
@@ -120,6 +129,7 @@ class MambaPrefillCheckpointBuilder:
mamba_block_size=block_size,
checkpoint_alignment=self.kv_cache_spec.prefill_checkpoint_alignment,
drop_eagle_block=drop_eagle_block,
cache_hit_alignment_tokens=cache_hit_alignment_tokens,
)
if not any(checkpoint_offsets):
return None
@@ -139,6 +149,7 @@ class MambaPrefillCheckpointBuilder:
_gather_checkpoint_state_indices(
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
),
offsets=checkpoint_offsets,
request_rows=request_rows_tensor,
block_cols=checkpoint_cols_tensor,
)
@@ -8,6 +8,7 @@ from typing import Literal
import torch
from einops import rearrange
from torch import nn
from transformers import Qwen3NextConfig
from vllm import _custom_ops as ops
from vllm import envs
@@ -58,7 +59,6 @@ from vllm.third_party.flash_linear_attention.ops import (
)
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
@@ -92,7 +92,7 @@ FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
def _resolve_gdn_prefill_backend(
vllm_config: VllmConfig,
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl"]]:
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
"""Resolve GDN prefill backend.
FlashInfer's GDN prefill kernel is chosen when:
@@ -106,6 +106,11 @@ def _resolve_gdn_prefill_backend(
In-tree CuteDSL GDN prefill kernel is chosen when:
* "cutedsl" is requested; (opt-in only)
* Blackwell (SM10.x) with ``head_k_dim == 128``;
AITER FlyDSL GDN prefill kernels are chosen when:
* "aiter_flydsl" is requested; (opt-in only)
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
"""
additional_config = vllm_config.additional_config
backend_cfg = (
@@ -115,12 +120,50 @@ def _resolve_gdn_prefill_backend(
)
backend = str(backend_cfg).strip().lower()
if not current_platform.is_cuda():
return backend, "triton"
head_k_dim = getattr(
vllm_config.model_config.hf_text_config, "linear_key_head_dim", None
)
head_v_dim = getattr(
vllm_config.model_config.hf_text_config, "linear_value_head_dim", None
)
if current_platform.is_rocm():
if backend == "aiter_flydsl":
if not rocm_aiter_ops.is_gdn_flydsl_prefill_available():
raise RuntimeError(
"GDN prefill backend 'aiter_flydsl' was requested but the "
"AITER FlyDSL prefill kernels are not importable. Check "
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
"newer GPU, and that the installed AITER exports them."
)
if head_k_dim != 128 or head_v_dim != 128:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
"Falling back to Triton/FLA.",
head_k_dim,
head_v_dim,
)
return backend, "triton"
if vllm_config.model_config.dtype != torch.bfloat16:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"model dtype is %s; FlyDSL requires bfloat16. "
"Falling back to Triton/FLA.",
vllm_config.model_config.dtype,
)
return backend, "triton"
return backend, "aiter_flydsl"
return backend, "triton"
if backend == "aiter_flydsl":
raise RuntimeError(
"GDN prefill backend 'aiter_flydsl' was requested but it is a ROCm "
f"AITER backend and the current platform is {current_platform.device_name}."
)
if not current_platform.is_cuda():
return backend, "triton"
supports_flashinfer = False
supports_cutedsl = False
@@ -172,6 +215,7 @@ def _log_gdn_backend_decision(
chosen = {
"flashinfer": "FlashInfer",
"cutedsl": "CuteDSL",
"aiter_flydsl": "AITER FlyDSL",
"triton": "Triton/FLA",
}[active_backend]
logger.info_once(
@@ -259,6 +303,8 @@ class ChunkGatedDeltaRule(CustomOp):
self._forward_method = self.forward_cuda
elif active_backend == "cutedsl":
self._forward_method = self.forward_cutedsl
elif active_backend == "aiter_flydsl":
self._forward_method = self.forward_aiter_flydsl
else:
self._forward_method = self.forward_native
@@ -276,6 +322,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
o, final_state = fi_chunk_gated_delta_rule(
q=q,
@@ -308,6 +355,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
return fla_chunk_gated_delta_rule(
q=q,
@@ -338,6 +386,7 @@ class ChunkGatedDeltaRule(CustomOp):
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
chunk_gated_delta_rule_cutedsl,
@@ -367,6 +416,39 @@ class ChunkGatedDeltaRule(CustomOp):
final_state = None
return o, final_state
def forward_aiter_flydsl(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
initial_state: torch.Tensor,
output_final_state: bool,
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
core_attn_out: torch.Tensor | None = None,
aiter_prefill_metadata: object | None = None,
):
o, final_state = rocm_aiter_ops.gdn_flydsl_prefill(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=output_final_state,
cu_seqlens=cu_seqlens,
aiter_prefill_metadata=aiter_prefill_metadata,
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
)
if core_attn_out is not None:
o_flat = o.reshape(-1)
core_attn_out.reshape(-1)[: o_flat.numel()].copy_(o_flat)
return o, final_state
@PluggableLayer.register("qwen_gated_delta_net_attention")
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
@@ -1193,12 +1275,18 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
# CuteDSL kernels require metadata
chunk_indices = None
chunk_offsets = None
aiter_prefill_metadata = None
if self.gdn_prefill_backend == "cutedsl":
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
prepare_metadata_cutedsl,
)
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, T)
elif self.gdn_prefill_backend == "aiter_flydsl":
aiter_prefill_metadata = rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
[T],
cu_seqlens=cu_seqlens,
)
try:
self.chunk_gated_delta_rule(
@@ -1213,6 +1301,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=aiter_prefill_metadata,
)
except Exception:
logger.warning(
@@ -1243,6 +1332,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
cu_seqlens,
chunk_indices,
chunk_offsets,
aiter_prefill_metadata,
)
torch.accelerator.empty_cache()
@@ -1634,6 +1724,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
)
# Init cache
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
@@ -3,11 +3,12 @@
from collections.abc import Sequence
from dataclasses import replace
import torch
from torch import nn
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.config.mamba import MambaBackendEnum
from vllm.distributed import (
divide,
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
selective_state_update_replayssm_output_only,
)
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import store_prefill_checkpoint
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
@@ -66,7 +68,7 @@ from vllm.utils.torch_utils import (
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec, MambaSpec
logger = init_logger(__name__)
@@ -766,6 +768,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
query_start_loc_d = attn_metadata.query_start_loc_d
num_decodes = attn_metadata.num_decodes
num_decode_tokens = attn_metadata.num_decode_tokens
checkpoint_chunk_idx = attn_metadata.checkpoint_chunk_idx
checkpoint_meta = attn_metadata.checkpoint_meta
if attn_metadata is None:
# V1 profile run -- warm up SSD kernels so that autotuning
@@ -842,6 +846,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
# NOTE: final output is an in-place update of out tensor
assert preallocated_ssm_out_p is not None
has_checkpoints = checkpoint_chunk_idx is not None
varlen_states = mamba_chunk_scan_combined_varlen(
hidden_states_p.view(
num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim
@@ -859,6 +864,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
cu_chunk_seqlens=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
initial_states=initial_states,
return_intermediate_states=has_checkpoints,
dt_softplus=True,
dt_limit=(0.0, float("inf")),
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
@@ -869,7 +875,24 @@ class MambaMixer2(MambaBase, PluggableLayer):
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
final_states = (
varlen_states[last_chunk_indices_p]
if has_checkpoints
else varlen_states
)
if has_checkpoints:
assert checkpoint_meta is not None
assert query_start_loc_p is not None
store_prefill_checkpoint(
checkpoint_meta,
checkpoint_chunk_idx,
x,
conv_state,
varlen_states,
ssm_state,
query_start_loc_p,
)
ssm_state[state_indices_tensor_p] = final_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
@@ -1070,6 +1093,18 @@ class MambaMixer2(MambaBase, PluggableLayer):
is_blackwell=self.is_blackwell,
)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
spec = super().get_kv_cache_spec(vllm_config)
if not isinstance(spec, MambaSpec):
return spec
# The block-boundary state can be exported from inside one forward
# pass, so the scheduler need not split the prefill; every reader
# acts on this only in align mode. The metadata builder puts a chunk
# end exactly on the checkpoint, so any token position works.
return replace(
spec, num_prefill_checkpoint_blocks=1, prefill_checkpoint_alignment=1
)
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
assert self.model_config is not None
assert self.cache_config is not None
@@ -0,0 +1,51 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,51 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"32": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"1024": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"16": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"32": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"64": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"128": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"256": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"8192": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"16384": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"32768": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 8,
"num_warps": 8
},
"16": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"32": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"64": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"256": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"512": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"1024": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"8192": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"16384": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,47 @@
{
"triton_version": "3.8.0",
"48": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"96": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"384": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"768": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"1536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"3072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"6144": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"12288": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"24576": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"49152": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,39 @@
{
"triton_version": "3.8.0",
"48": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"384": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"768": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"1536": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"3072": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"6144": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"12288": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"24576": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"49152": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"16": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"32": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"256": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,47 @@
{
"triton_version": "3.8.0",
"80": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"640": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"1280": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"2560": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"5120": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"10240": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"20480": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"40960": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"81920": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"122880": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"163840": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
}
}
@@ -0,0 +1,47 @@
{
"triton_version": "3.8.0",
"80": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"640": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"1280": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"2560": {
"BLOCK_SIZE_M": 8,
"num_warps": 4
},
"5120": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"10240": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"20480": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"40960": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"81920": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"122880": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"163840": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
}
}
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.triton_utils import tl, triton
@triton.jit
def _store_prefill_checkpoint_kernel(
x_ptr,
conv_state_ptr,
chunk_states_ptr,
ssm_state_ptr,
query_start_loc_ptr,
checkpoint_offsets_ptr,
state_indices_ptr,
chunk_idx_ptr,
stride_x_dim,
stride_x_token,
stride_conv_block,
stride_conv_dim,
stride_conv_token,
stride_chunk_states,
stride_ssm_block,
DIM: tl.constexpr,
STATE_LEN: tl.constexpr,
SSM_ROW_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
checkpoint_offset = tl.load(checkpoint_offsets_ptr + row)
valid = checkpoint_offset > 0
dst = tl.load(state_indices_ptr + row).to(tl.int64)
conv_mask = valid & (cols < DIM * STATE_LEN)
dim_idx = cols // STATE_LEN
history_idx = cols % STATE_LEN
token_idx = (
tl.load(query_start_loc_ptr + row) + checkpoint_offset - STATE_LEN + history_idx
).to(tl.int64)
conv = tl.load(
x_ptr + dim_idx * stride_x_dim + token_idx * stride_x_token, mask=conv_mask
)
tl.store(
conv_state_ptr
+ dst * stride_conv_block
+ dim_idx * stride_conv_dim
+ history_idx * stride_conv_token,
conv,
mask=conv_mask,
)
ssm_mask = valid & (cols < SSM_ROW_SIZE)
src = tl.load(chunk_idx_ptr + row).to(tl.int64)
ssm = tl.load(chunk_states_ptr + src * stride_chunk_states + cols, mask=ssm_mask)
tl.store(ssm_state_ptr + dst * stride_ssm_block + cols, ssm, mask=ssm_mask)
def store_prefill_checkpoint(
checkpoint: MambaPrefillCheckpointMetadata,
chunk_idx: torch.Tensor,
x: torch.Tensor,
conv_state: torch.Tensor,
chunk_states: torch.Tensor,
ssm_state: torch.Tensor,
query_start_loc: torch.Tensor,
) -> None:
"""Write each prefill row's mid-prefill checkpoint into its paged block.
Rows without a checkpoint (offset 0) write nothing.
Args:
checkpoint: Per-row offsets and destination blocks.
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
chunk_states: SSD states at every chunk end.
ssm_state: Paged SSM state.
query_start_loc: Prefill query start locations.
"""
dim = x.size(0)
state_len = conv_state.size(-1)
ssm_row_size = chunk_states[0].numel()
block_size = 1024
grid = (
chunk_idx.numel(),
triton.cdiv(max(dim * state_len, ssm_row_size), block_size),
)
_store_prefill_checkpoint_kernel[grid](
x,
conv_state,
chunk_states,
ssm_state,
query_start_loc,
checkpoint.checkpoint_offsets,
checkpoint.state_indices,
chunk_idx,
x.stride(0),
x.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
chunk_states.stride(0),
ssm_state.stride(0),
DIM=dim,
STATE_LEN=state_len,
SSM_ROW_SIZE=ssm_row_size,
BLOCK_SIZE=block_size,
)