Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -25,6 +25,7 @@ def compute_mamba_prefill_checkpoints(
|
||||
mamba_block_size: int,
|
||||
checkpoint_alignment: int | None,
|
||||
drop_eagle_block: bool,
|
||||
cache_hit_alignment_tokens: int,
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Per-row internal prefill checkpoint offsets and cache block columns.
|
||||
|
||||
@@ -43,7 +44,7 @@ def compute_mamba_prefill_checkpoints(
|
||||
for seq_len, query_len in zip(seq_lens, query_lens):
|
||||
query_start = seq_len - query_len
|
||||
position = get_mamba_prefill_checkpoint_position(
|
||||
seq_len, hash_block_size, drop_eagle_block=drop_eagle_block
|
||||
seq_len, cache_hit_alignment_tokens, drop_eagle_block=drop_eagle_block
|
||||
)
|
||||
valid = is_mamba_prefill_checkpoint_valid(
|
||||
query_start=query_start,
|
||||
@@ -70,6 +71,9 @@ def _gather_checkpoint_state_indices(
|
||||
class MambaPrefillCheckpointMetadata:
|
||||
checkpoint_offsets: torch.Tensor
|
||||
state_indices: torch.Tensor
|
||||
# Host copy of ``checkpoint_offsets``, for backends that lay out the
|
||||
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
|
||||
offsets: list[int] | None = None
|
||||
# Gather indices into the block table, kept so that another KV cache group
|
||||
# with the same spec can re-derive its own ``state_indices``.
|
||||
request_rows: torch.Tensor | None = None
|
||||
@@ -103,12 +107,17 @@ class MambaPrefillCheckpointBuilder:
|
||||
return None
|
||||
if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0:
|
||||
return None
|
||||
cache_config = self.vllm_config.cache_config
|
||||
hash_block_size = cache_config.hash_block_size
|
||||
cache_hit_alignment_tokens = cache_config.cache_hit_alignment_tokens
|
||||
if hash_block_size is None or cache_hit_alignment_tokens is None:
|
||||
# Unresolved only for the minimal KV cache built for profiling.
|
||||
return None
|
||||
assert m.seq_lens_cpu_upper_bound is not None
|
||||
all_query_lens = m.query_start_loc_cpu.diff().tolist()
|
||||
query_lens = [all_query_lens[row] for row in request_rows]
|
||||
seq_lens = m.seq_lens_cpu_upper_bound.tolist()
|
||||
block_size = self.kv_cache_spec.block_size
|
||||
hash_block_size = self.vllm_config.cache_config.prefix_match_unit or block_size
|
||||
speculative_config = self.vllm_config.speculative_config
|
||||
drop_eagle_block = (
|
||||
speculative_config is not None and speculative_config.use_eagle_block_drop()
|
||||
@@ -120,6 +129,7 @@ class MambaPrefillCheckpointBuilder:
|
||||
mamba_block_size=block_size,
|
||||
checkpoint_alignment=self.kv_cache_spec.prefill_checkpoint_alignment,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
cache_hit_alignment_tokens=cache_hit_alignment_tokens,
|
||||
)
|
||||
if not any(checkpoint_offsets):
|
||||
return None
|
||||
@@ -139,6 +149,7 @@ class MambaPrefillCheckpointBuilder:
|
||||
_gather_checkpoint_state_indices(
|
||||
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
|
||||
),
|
||||
offsets=checkpoint_offsets,
|
||||
request_rows=request_rows_tensor,
|
||||
block_cols=checkpoint_cols_tensor,
|
||||
)
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Literal
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
from transformers import Qwen3NextConfig
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
@@ -58,7 +59,6 @@ from vllm.third_party.flash_linear_attention.ops import (
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
@@ -92,7 +92,7 @@ FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
|
||||
|
||||
def _resolve_gdn_prefill_backend(
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl"]]:
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
|
||||
"""Resolve GDN prefill backend.
|
||||
|
||||
FlashInfer's GDN prefill kernel is chosen when:
|
||||
@@ -106,6 +106,11 @@ def _resolve_gdn_prefill_backend(
|
||||
In-tree CuteDSL GDN prefill kernel is chosen when:
|
||||
* "cutedsl" is requested; (opt-in only)
|
||||
* Blackwell (SM10.x) with ``head_k_dim == 128``;
|
||||
|
||||
AITER FlyDSL GDN prefill kernels are chosen when:
|
||||
* "aiter_flydsl" is requested; (opt-in only)
|
||||
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
|
||||
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
|
||||
"""
|
||||
additional_config = vllm_config.additional_config
|
||||
backend_cfg = (
|
||||
@@ -115,12 +120,50 @@ def _resolve_gdn_prefill_backend(
|
||||
)
|
||||
backend = str(backend_cfg).strip().lower()
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
return backend, "triton"
|
||||
|
||||
head_k_dim = getattr(
|
||||
vllm_config.model_config.hf_text_config, "linear_key_head_dim", None
|
||||
)
|
||||
head_v_dim = getattr(
|
||||
vllm_config.model_config.hf_text_config, "linear_value_head_dim", None
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
if backend == "aiter_flydsl":
|
||||
if not rocm_aiter_ops.is_gdn_flydsl_prefill_available():
|
||||
raise RuntimeError(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but the "
|
||||
"AITER FlyDSL prefill kernels are not importable. Check "
|
||||
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
|
||||
"newer GPU, and that the installed AITER exports them."
|
||||
)
|
||||
if head_k_dim != 128 or head_v_dim != 128:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
|
||||
"Falling back to Triton/FLA.",
|
||||
head_k_dim,
|
||||
head_v_dim,
|
||||
)
|
||||
return backend, "triton"
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"model dtype is %s; FlyDSL requires bfloat16. "
|
||||
"Falling back to Triton/FLA.",
|
||||
vllm_config.model_config.dtype,
|
||||
)
|
||||
return backend, "triton"
|
||||
return backend, "aiter_flydsl"
|
||||
return backend, "triton"
|
||||
|
||||
if backend == "aiter_flydsl":
|
||||
raise RuntimeError(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but it is a ROCm "
|
||||
f"AITER backend and the current platform is {current_platform.device_name}."
|
||||
)
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
return backend, "triton"
|
||||
|
||||
supports_flashinfer = False
|
||||
supports_cutedsl = False
|
||||
@@ -172,6 +215,7 @@ def _log_gdn_backend_decision(
|
||||
chosen = {
|
||||
"flashinfer": "FlashInfer",
|
||||
"cutedsl": "CuteDSL",
|
||||
"aiter_flydsl": "AITER FlyDSL",
|
||||
"triton": "Triton/FLA",
|
||||
}[active_backend]
|
||||
logger.info_once(
|
||||
@@ -259,6 +303,8 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
self._forward_method = self.forward_cuda
|
||||
elif active_backend == "cutedsl":
|
||||
self._forward_method = self.forward_cutedsl
|
||||
elif active_backend == "aiter_flydsl":
|
||||
self._forward_method = self.forward_aiter_flydsl
|
||||
else:
|
||||
self._forward_method = self.forward_native
|
||||
|
||||
@@ -276,6 +322,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
o, final_state = fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
@@ -308,6 +355,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
return fla_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
@@ -338,6 +386,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
|
||||
chunk_gated_delta_rule_cutedsl,
|
||||
@@ -367,6 +416,39 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
final_state = None
|
||||
return o, final_state
|
||||
|
||||
def forward_aiter_flydsl(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
o, final_state = rocm_aiter_ops.gdn_flydsl_prefill(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
)
|
||||
if core_attn_out is not None:
|
||||
o_flat = o.reshape(-1)
|
||||
core_attn_out.reshape(-1)[: o_flat.numel()].copy_(o_flat)
|
||||
return o, final_state
|
||||
|
||||
|
||||
@PluggableLayer.register("qwen_gated_delta_net_attention")
|
||||
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
@@ -1193,12 +1275,18 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
# CuteDSL kernels require metadata
|
||||
chunk_indices = None
|
||||
chunk_offsets = None
|
||||
aiter_prefill_metadata = None
|
||||
if self.gdn_prefill_backend == "cutedsl":
|
||||
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
|
||||
prepare_metadata_cutedsl,
|
||||
)
|
||||
|
||||
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, T)
|
||||
elif self.gdn_prefill_backend == "aiter_flydsl":
|
||||
aiter_prefill_metadata = rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
|
||||
[T],
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
|
||||
try:
|
||||
self.chunk_gated_delta_rule(
|
||||
@@ -1213,6 +1301,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
@@ -1243,6 +1332,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
chunk_offsets,
|
||||
aiter_prefill_metadata,
|
||||
)
|
||||
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1634,6 +1724,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
chunk_indices=attn_metadata.chunk_indices,
|
||||
chunk_offsets=attn_metadata.chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
|
||||
)
|
||||
# Init cache
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
|
||||
|
||||
@@ -3,11 +3,12 @@
|
||||
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
|
||||
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
|
||||
selective_state_update_replayssm_output_only,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import store_prefill_checkpoint
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
@@ -66,7 +68,7 @@ from vllm.utils.torch_utils import (
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec, MambaSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -766,6 +768,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
query_start_loc_d = attn_metadata.query_start_loc_d
|
||||
num_decodes = attn_metadata.num_decodes
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
checkpoint_chunk_idx = attn_metadata.checkpoint_chunk_idx
|
||||
checkpoint_meta = attn_metadata.checkpoint_meta
|
||||
|
||||
if attn_metadata is None:
|
||||
# V1 profile run -- warm up SSD kernels so that autotuning
|
||||
@@ -842,6 +846,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
|
||||
# NOTE: final output is an in-place update of out tensor
|
||||
assert preallocated_ssm_out_p is not None
|
||||
has_checkpoints = checkpoint_chunk_idx is not None
|
||||
varlen_states = mamba_chunk_scan_combined_varlen(
|
||||
hidden_states_p.view(
|
||||
num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim
|
||||
@@ -859,6 +864,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
cu_chunk_seqlens=cu_chunk_seqlen_p,
|
||||
last_chunk_indices=last_chunk_indices_p,
|
||||
initial_states=initial_states,
|
||||
return_intermediate_states=has_checkpoints,
|
||||
dt_softplus=True,
|
||||
dt_limit=(0.0, float("inf")),
|
||||
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
|
||||
@@ -869,7 +875,24 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
# - varlen state is a (num_prefills, nheads, headdim, dstate)
|
||||
# tensor
|
||||
assert state_indices_tensor_p is not None
|
||||
ssm_state[state_indices_tensor_p] = varlen_states
|
||||
final_states = (
|
||||
varlen_states[last_chunk_indices_p]
|
||||
if has_checkpoints
|
||||
else varlen_states
|
||||
)
|
||||
if has_checkpoints:
|
||||
assert checkpoint_meta is not None
|
||||
assert query_start_loc_p is not None
|
||||
store_prefill_checkpoint(
|
||||
checkpoint_meta,
|
||||
checkpoint_chunk_idx,
|
||||
x,
|
||||
conv_state,
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
query_start_loc_p,
|
||||
)
|
||||
ssm_state[state_indices_tensor_p] = final_states
|
||||
if ring_start is not None and self._updates_replayssm_trackers:
|
||||
assert prev_num_accepted is not None
|
||||
reset_replayssm_ring_trackers(
|
||||
@@ -1070,6 +1093,18 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
is_blackwell=self.is_blackwell,
|
||||
)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
if not isinstance(spec, MambaSpec):
|
||||
return spec
|
||||
# The block-boundary state can be exported from inside one forward
|
||||
# pass, so the scheduler need not split the prefill; every reader
|
||||
# acts on this only in align mode. The metadata builder puts a chunk
|
||||
# end exactly on the checkpoint, so any token position works.
|
||||
return replace(
|
||||
spec, num_prefill_checkpoint_blocks=1, prefill_checkpoint_alignment=1
|
||||
)
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"384": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"768": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"6144": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"12288": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"24576": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"49152": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"384": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"768": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"6144": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"12288": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"24576": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"49152": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"80": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"640": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"1280": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"2560": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"10240": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"20480": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"40960": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"81920": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"122880": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"163840": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"80": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"640": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"1280": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"2560": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 4
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 8
|
||||
},
|
||||
"10240": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 8
|
||||
},
|
||||
"20480": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 8
|
||||
},
|
||||
"40960": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"81920": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"122880": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"163840": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_prefill_checkpoint_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
chunk_states_ptr,
|
||||
ssm_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
state_indices_ptr,
|
||||
chunk_idx_ptr,
|
||||
stride_x_dim,
|
||||
stride_x_token,
|
||||
stride_conv_block,
|
||||
stride_conv_dim,
|
||||
stride_conv_token,
|
||||
stride_chunk_states,
|
||||
stride_ssm_block,
|
||||
DIM: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
SSM_ROW_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
checkpoint_offset = tl.load(checkpoint_offsets_ptr + row)
|
||||
valid = checkpoint_offset > 0
|
||||
dst = tl.load(state_indices_ptr + row).to(tl.int64)
|
||||
|
||||
conv_mask = valid & (cols < DIM * STATE_LEN)
|
||||
dim_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
token_idx = (
|
||||
tl.load(query_start_loc_ptr + row) + checkpoint_offset - STATE_LEN + history_idx
|
||||
).to(tl.int64)
|
||||
conv = tl.load(
|
||||
x_ptr + dim_idx * stride_x_dim + token_idx * stride_x_token, mask=conv_mask
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ dst * stride_conv_block
|
||||
+ dim_idx * stride_conv_dim
|
||||
+ history_idx * stride_conv_token,
|
||||
conv,
|
||||
mask=conv_mask,
|
||||
)
|
||||
|
||||
ssm_mask = valid & (cols < SSM_ROW_SIZE)
|
||||
src = tl.load(chunk_idx_ptr + row).to(tl.int64)
|
||||
ssm = tl.load(chunk_states_ptr + src * stride_chunk_states + cols, mask=ssm_mask)
|
||||
tl.store(ssm_state_ptr + dst * stride_ssm_block + cols, ssm, mask=ssm_mask)
|
||||
|
||||
|
||||
def store_prefill_checkpoint(
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
chunk_idx: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
chunk_states: torch.Tensor,
|
||||
ssm_state: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
) -> None:
|
||||
"""Write each prefill row's mid-prefill checkpoint into its paged block.
|
||||
|
||||
Rows without a checkpoint (offset 0) write nothing.
|
||||
|
||||
Args:
|
||||
checkpoint: Per-row offsets and destination blocks.
|
||||
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
|
||||
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
|
||||
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
|
||||
chunk_states: SSD states at every chunk end.
|
||||
ssm_state: Paged SSM state.
|
||||
query_start_loc: Prefill query start locations.
|
||||
|
||||
"""
|
||||
dim = x.size(0)
|
||||
state_len = conv_state.size(-1)
|
||||
ssm_row_size = chunk_states[0].numel()
|
||||
block_size = 1024
|
||||
grid = (
|
||||
chunk_idx.numel(),
|
||||
triton.cdiv(max(dim * state_len, ssm_row_size), block_size),
|
||||
)
|
||||
_store_prefill_checkpoint_kernel[grid](
|
||||
x,
|
||||
conv_state,
|
||||
chunk_states,
|
||||
ssm_state,
|
||||
query_start_loc,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
chunk_idx,
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
chunk_states.stride(0),
|
||||
ssm_state.stride(0),
|
||||
DIM=dim,
|
||||
STATE_LEN=state_len,
|
||||
SSM_ROW_SIZE=ssm_row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
Reference in New Issue
Block a user