[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,14 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
)
|
||||
|
||||
|
||||
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
|
||||
return 16 if backend == "flashkda" else None
|
||||
|
||||
|
||||
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
|
||||
"""Store FlashKDA recurrent and convolution checkpoint states."""
|
||||
@@ -1,704 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
|
||||
from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import (
|
||||
MiniMaxText01LinearAttention,
|
||||
MiniMaxText01LinearKernel,
|
||||
linear_attention_decode,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
|
||||
RMSNormGated,
|
||||
layernorm_fn,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
|
||||
|
||||
|
||||
def _build_rope_parameters(config: PreTrainedConfig) -> dict | None:
|
||||
rope_parameters = copy.deepcopy(getattr(config, "rope_parameters", None)) or {}
|
||||
if "rope_theta" not in rope_parameters and hasattr(config, "rope_theta"):
|
||||
rope_parameters["rope_theta"] = config.rope_theta
|
||||
if "partial_rotary_factor" not in rope_parameters and hasattr(
|
||||
config, "partial_rotary_factor"
|
||||
):
|
||||
rope_parameters["partial_rotary_factor"] = config.partial_rotary_factor
|
||||
|
||||
rope_scaling = getattr(config, "rope_scaling", None)
|
||||
if isinstance(rope_scaling, dict):
|
||||
rope_scaling = copy.deepcopy(rope_scaling)
|
||||
if "type" in rope_scaling and "rope_type" not in rope_scaling:
|
||||
rope_scaling["rope_type"] = rope_scaling.pop("type")
|
||||
rope_parameters.update(rope_scaling)
|
||||
|
||||
return rope_parameters or None
|
||||
|
||||
|
||||
def clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
attn_metadata: LinearAttentionMetadata,
|
||||
) -> None:
|
||||
num_prefills = getattr(attn_metadata, "num_prefills", 0)
|
||||
if num_prefills <= 0:
|
||||
return
|
||||
|
||||
num_decodes = getattr(attn_metadata, "num_decodes", 0)
|
||||
prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None)
|
||||
for prefill_idx in range(num_prefills):
|
||||
if num_decodes + prefill_idx + 1 >= len(attn_metadata.query_start_loc):
|
||||
break
|
||||
q_start = attn_metadata.query_start_loc[num_decodes + prefill_idx]
|
||||
q_end = attn_metadata.query_start_loc[num_decodes + prefill_idx + 1]
|
||||
query_len = q_end - q_start
|
||||
context_len = attn_metadata.seq_lens[num_decodes + prefill_idx] - query_len
|
||||
if context_len == 0:
|
||||
if prefill_state_indices is not None:
|
||||
block_to_clear = prefill_state_indices[prefill_idx]
|
||||
else:
|
||||
block_to_clear = state_indices_tensor[num_decodes + prefill_idx]
|
||||
kv_cache[block_to_clear, ...] = 0
|
||||
|
||||
|
||||
def linear_attention_prefill_and_mix(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
attn_metadata: LinearAttentionMetadata,
|
||||
slope_rate: torch.Tensor,
|
||||
block_size: int,
|
||||
decode_fn,
|
||||
prefix_fn,
|
||||
layer_idx: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
hidden = []
|
||||
req_offset = getattr(attn_metadata, "num_decodes", 0)
|
||||
prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None)
|
||||
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
|
||||
if req_offset + _prefill_idx + 1 >= len(attn_metadata.query_start_loc):
|
||||
break
|
||||
if prefill_state_indices is not None and _prefill_idx >= len(
|
||||
prefill_state_indices
|
||||
):
|
||||
break
|
||||
if prefill_state_indices is None and _prefill_idx >= len(state_indices_tensor):
|
||||
break
|
||||
_start = attn_metadata.query_start_loc[req_offset + _prefill_idx]
|
||||
_end = attn_metadata.query_start_loc[req_offset + _prefill_idx + 1]
|
||||
if prefill_state_indices is not None:
|
||||
slot_id = prefill_state_indices[_prefill_idx]
|
||||
else:
|
||||
slot_id = state_indices_tensor[req_offset + _prefill_idx]
|
||||
qs = q[_start:_end].transpose(0, 1).contiguous()
|
||||
ks = k[_start:_end].transpose(0, 1).contiguous()
|
||||
vs = v[_start:_end].transpose(0, 1).contiguous()
|
||||
slice_layer_cache = kv_cache[slot_id, ...]
|
||||
out_slice = prefix_fn(
|
||||
qs,
|
||||
ks,
|
||||
vs,
|
||||
slice_layer_cache,
|
||||
slope_rate,
|
||||
block_size,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
hidden.append(out_slice.contiguous())
|
||||
|
||||
if attn_metadata.num_decode_tokens > 0:
|
||||
hidden_decode = decode_fn(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
hidden.insert(0, hidden_decode)
|
||||
|
||||
if not hidden:
|
||||
return torch.empty((0, q.size(1) * q.size(2)), device=q.device, dtype=q.dtype)
|
||||
|
||||
hidden = torch.concat(hidden, dim=0).contiguous()
|
||||
return hidden
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _bailing_linear_attn_decode_spec_step_kernel(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
v_ptr,
|
||||
kv_cache_ptr,
|
||||
slope_rate_ptr,
|
||||
state_indices_ptr,
|
||||
query_start_loc_ptr,
|
||||
num_accepted_tokens_ptr,
|
||||
output_ptr,
|
||||
q_start: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
q_b_stride,
|
||||
q_h_stride,
|
||||
q_d_stride,
|
||||
k_b_stride,
|
||||
k_h_stride,
|
||||
k_d_stride,
|
||||
v_b_stride,
|
||||
v_h_stride,
|
||||
v_d_stride,
|
||||
cache_b_stride,
|
||||
cache_h_stride,
|
||||
cache_d0_stride,
|
||||
cache_d1_stride,
|
||||
state_indices_b_stride,
|
||||
state_indices_t_stride,
|
||||
output_b_stride,
|
||||
output_d_stride,
|
||||
DRAFT_IDX: tl.constexpr,
|
||||
STATE_WIDTH: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_id = tl.program_id(0)
|
||||
head_id = tl.program_id(1)
|
||||
block_id = tl.program_id(2)
|
||||
|
||||
req_start = tl.load(query_start_loc_ptr + req_id).to(tl.int64)
|
||||
req_end = tl.load(query_start_loc_ptr + req_id + 1).to(tl.int64)
|
||||
query_len = req_end - req_start
|
||||
if query_len <= DRAFT_IDX:
|
||||
return
|
||||
|
||||
dst_slot = tl.load(
|
||||
state_indices_ptr
|
||||
+ req_id * state_indices_b_stride
|
||||
+ DRAFT_IDX * state_indices_t_stride
|
||||
).to(tl.int64)
|
||||
if dst_slot == -1:
|
||||
return
|
||||
|
||||
if DRAFT_IDX == 0:
|
||||
accepted_offset = tl.load(num_accepted_tokens_ptr + req_id).to(tl.int64) - 1
|
||||
accepted_offset = tl.maximum(accepted_offset, 0)
|
||||
accepted_offset = tl.minimum(accepted_offset, STATE_WIDTH - 1)
|
||||
src_slot = tl.load(
|
||||
state_indices_ptr
|
||||
+ req_id * state_indices_b_stride
|
||||
+ accepted_offset * state_indices_t_stride
|
||||
).to(tl.int64)
|
||||
else:
|
||||
src_slot = tl.load(
|
||||
state_indices_ptr
|
||||
+ req_id * state_indices_b_stride
|
||||
+ (DRAFT_IDX - 1) * state_indices_t_stride
|
||||
).to(tl.int64)
|
||||
if src_slot == -1:
|
||||
return
|
||||
|
||||
token_idx = req_start - q_start + DRAFT_IDX
|
||||
qk_offsets = tl.arange(0, D)
|
||||
v_offsets = tl.arange(0, BLOCK_SIZE) + block_id * BLOCK_SIZE
|
||||
qk_mask = qk_offsets < D
|
||||
v_mask = v_offsets < D
|
||||
kv_mask = qk_mask[:, None] & v_mask[None, :]
|
||||
|
||||
q = tl.load(
|
||||
q_ptr + token_idx * q_b_stride + head_id * q_h_stride + qk_offsets * q_d_stride,
|
||||
mask=qk_mask,
|
||||
other=0.0,
|
||||
)
|
||||
k = tl.load(
|
||||
k_ptr + token_idx * k_b_stride + head_id * k_h_stride + qk_offsets * k_d_stride,
|
||||
mask=qk_mask,
|
||||
other=0.0,
|
||||
)
|
||||
v = tl.load(
|
||||
v_ptr + token_idx * v_b_stride + head_id * v_h_stride + v_offsets * v_d_stride,
|
||||
mask=v_mask,
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
cache_offsets = (
|
||||
qk_offsets[:, None] * cache_d0_stride + v_offsets[None, :] * cache_d1_stride
|
||||
)
|
||||
src_cache_ptr = (
|
||||
kv_cache_ptr
|
||||
+ src_slot * cache_b_stride
|
||||
+ head_id * cache_h_stride
|
||||
+ cache_offsets
|
||||
)
|
||||
dst_cache_ptr = (
|
||||
kv_cache_ptr
|
||||
+ dst_slot * cache_b_stride
|
||||
+ head_id * cache_h_stride
|
||||
+ cache_offsets
|
||||
)
|
||||
|
||||
slope = tl.load(slope_rate_ptr + head_id)
|
||||
decay = tl.exp(-slope)
|
||||
kv_old = tl.load(src_cache_ptr, mask=kv_mask, other=0.0)
|
||||
kv_new = k[:, None] * v[None, :] + decay * kv_old
|
||||
|
||||
output = tl.sum(q[:, None].to(tl.float32) * kv_new, axis=0)
|
||||
tl.store(dst_cache_ptr, kv_new, mask=kv_mask)
|
||||
tl.store(
|
||||
output_ptr
|
||||
+ token_idx * output_b_stride
|
||||
+ (head_id * D + v_offsets) * output_d_stride,
|
||||
output,
|
||||
mask=v_mask,
|
||||
)
|
||||
|
||||
|
||||
def bailing_linear_attention_decode_spec(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slope_rate: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
q_start: int,
|
||||
q_end: int | None,
|
||||
slot_start: int,
|
||||
slot_end: int | None,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
q_decode = q[q_start:q_end]
|
||||
k_decode = k[q_start:q_end]
|
||||
v_decode = v[q_start:q_end]
|
||||
hidden = torch.empty(
|
||||
(q_decode.shape[0], q.shape[1] * q.shape[2]),
|
||||
device=q.device,
|
||||
dtype=q.dtype,
|
||||
)
|
||||
hidden.zero_()
|
||||
|
||||
state_indices_tensor = state_indices_tensor[slot_start:slot_end]
|
||||
query_start_loc = query_start_loc.to(device=q.device)
|
||||
|
||||
batch_size = state_indices_tensor.shape[0]
|
||||
num_heads = q_decode.shape[1]
|
||||
head_dim = q_decode.shape[2]
|
||||
assert k_decode.shape == (q_decode.shape[0], num_heads, head_dim)
|
||||
assert v_decode.shape == (q_decode.shape[0], num_heads, head_dim)
|
||||
state_width = state_indices_tensor.shape[1]
|
||||
|
||||
grid = (batch_size, num_heads, triton.cdiv(head_dim, block_size))
|
||||
for draft_idx in range(state_width):
|
||||
_bailing_linear_attn_decode_spec_step_kernel[grid](
|
||||
q_decode,
|
||||
k_decode,
|
||||
v_decode,
|
||||
kv_cache,
|
||||
slope_rate,
|
||||
state_indices_tensor,
|
||||
query_start_loc,
|
||||
num_accepted_tokens[:batch_size],
|
||||
hidden,
|
||||
q_start,
|
||||
head_dim,
|
||||
q_decode.stride(0),
|
||||
q_decode.stride(1),
|
||||
q_decode.stride(2),
|
||||
k_decode.stride(0),
|
||||
k_decode.stride(1),
|
||||
k_decode.stride(2),
|
||||
v_decode.stride(0),
|
||||
v_decode.stride(1),
|
||||
v_decode.stride(2),
|
||||
kv_cache.stride(0),
|
||||
kv_cache.stride(1),
|
||||
kv_cache.stride(2),
|
||||
kv_cache.stride(3),
|
||||
state_indices_tensor.stride(0),
|
||||
state_indices_tensor.stride(1),
|
||||
hidden.stride(0),
|
||||
hidden.stride(1),
|
||||
DRAFT_IDX=draft_idx,
|
||||
STATE_WIDTH=state_width,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
|
||||
return hidden
|
||||
|
||||
|
||||
class BailingGroupRMSNormGate(RMSNormGated):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size,
|
||||
eps=1e-5,
|
||||
group_size=None,
|
||||
norm_before_gate=True,
|
||||
device=None,
|
||||
dtype=None,
|
||||
):
|
||||
super().__init__(
|
||||
hidden_size,
|
||||
eps=eps,
|
||||
group_size=group_size,
|
||||
norm_before_gate=norm_before_gate,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
activation="sigmoid",
|
||||
)
|
||||
# Add custom weight loader for TP sharding
|
||||
self.weight.weight_loader = self._weight_loader
|
||||
|
||||
@staticmethod
|
||||
def _weight_loader(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
|
||||
"""Load weight with TP sharding."""
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
shard_size = loaded_weight.shape[0] // tp_size
|
||||
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
|
||||
param.data.copy_(loaded_weight[shard].contiguous())
|
||||
|
||||
|
||||
# --8<-- [start:bailing_moe_linear_attention]
|
||||
@PluggableLayer.register("bailing_moe_linear_attention")
|
||||
class BailingMoELinearAttention(LinearAttention):
|
||||
"""Pluggable Bailing MoE Linear Attention layer which allows OOT backends
|
||||
to add custom implementations.
|
||||
|
||||
This implements the linear attention mechanism from sglang, adapted for
|
||||
vLLM's v1 engine with MambaBase interface support.
|
||||
"""
|
||||
|
||||
# --8<-- [end:bailing_moe_linear_attention]
|
||||
def __init__(
|
||||
self,
|
||||
config: PreTrainedConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "linear_attn",
|
||||
):
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
self.scaling = self.head_dim**-0.5
|
||||
|
||||
self.tp_heads = self.num_heads // self.tp_size
|
||||
|
||||
self.max_position_embeddings = config.max_position_embeddings
|
||||
self.rope_theta = getattr(config, "rope_theta", 600000)
|
||||
|
||||
self.tp_kv_heads = self.num_heads // self.tp_size
|
||||
self.q_size_per_rank = self.head_dim * self.tp_heads
|
||||
self.kv_size_per_rank = self.head_dim * self.tp_kv_heads
|
||||
|
||||
self.use_qk_norm = getattr(config, "use_qk_norm", False)
|
||||
self.linear_backend = "minimax"
|
||||
self.linear_scale = self.linear_backend == "minimax"
|
||||
self.linear_rope = getattr(config, "linear_rope", True)
|
||||
if hasattr(config, "use_linear_silu"):
|
||||
self.linear_silu = config.use_linear_silu
|
||||
elif hasattr(config, "linear_silu"):
|
||||
self.linear_silu = config.linear_silu
|
||||
else:
|
||||
self.linear_silu = False
|
||||
|
||||
self.query_key_value = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
self.num_heads, # MHA: kv_heads = num_heads
|
||||
bias=(config.use_bias or config.use_qkv_bias),
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.query_key_value",
|
||||
)
|
||||
|
||||
if self.use_qk_norm:
|
||||
self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.g_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_inner_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_proj",
|
||||
)
|
||||
self.dense = RowParallelLinear(
|
||||
self.hidden_inner_size,
|
||||
self.hidden_size,
|
||||
bias=config.use_bias,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.dense",
|
||||
reduce_results=True,
|
||||
)
|
||||
|
||||
self.group_norm_size = getattr(config, "group_norm_size", 1)
|
||||
self.rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
|
||||
assert self.tp_size <= self.group_norm_size, (
|
||||
"tp_size must be <= group_norm_size for local rms norm"
|
||||
)
|
||||
assert self.group_norm_size % self.tp_size == 0, (
|
||||
"group_norm_size must be divisible by tp_size"
|
||||
)
|
||||
|
||||
# When group_norm_size == 1, group_size equals hidden_size // tp_size
|
||||
self.g_norm = BailingGroupRMSNormGate(
|
||||
hidden_size=self.hidden_inner_size // self.tp_size,
|
||||
eps=self.rms_norm_eps,
|
||||
group_size=(
|
||||
self.hidden_inner_size // self.group_norm_size
|
||||
if self.group_norm_size > 1
|
||||
else self.hidden_inner_size // self.tp_size
|
||||
),
|
||||
)
|
||||
|
||||
# use fp32 rotary embedding
|
||||
rope_parameters = _build_rope_parameters(config)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=self.max_position_embeddings,
|
||||
is_neox_style=True,
|
||||
rope_parameters=rope_parameters or None,
|
||||
)
|
||||
|
||||
# Build slope tensor for linear attention decay
|
||||
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
|
||||
if self.num_hidden_layers <= 1:
|
||||
self.slope_rate = slope_rate * (1 + 1e-5)
|
||||
else:
|
||||
self.slope_rate = slope_rate * (
|
||||
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
|
||||
)
|
||||
self.tp_slope = self.slope_rate[
|
||||
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
|
||||
].contiguous()
|
||||
|
||||
# Register for compilation
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def get_attn_backend(self):
|
||||
from vllm.v1.attention.backends.linear_attn import (
|
||||
BailingLinearAttentionBackend,
|
||||
)
|
||||
|
||||
return BailingLinearAttentionBackend
|
||||
|
||||
@staticmethod
|
||||
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
"""Load weight for linear attention layers.
|
||||
|
||||
For FP8 quantized parameters, we need to use the weight_loader if available,
|
||||
as it handles special cases like tensor parallelism sharding.
|
||||
"""
|
||||
# Check if param has a weight_loader (for vLLM ModelWeightParameter)
|
||||
weight_loader = getattr(param, "weight_loader", None)
|
||||
if weight_loader is not None:
|
||||
# Use the weight_loader which handles TP sharding and quantization
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
# Fall back to direct copy for standard tensors
|
||||
assert param.size() == loaded_weight.size(), (
|
||||
f"Shape mismatch: {param.shape} vs {loaded_weight.shape}"
|
||||
)
|
||||
param.data.copy_(loaded_weight)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""Forward method called by torch.ops.vllm.linear_attention."""
|
||||
torch.ops.vllm.linear_attention(
|
||||
hidden_states,
|
||||
output,
|
||||
positions,
|
||||
self.prefix,
|
||||
)
|
||||
|
||||
def _forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""Actual forward implementation."""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, dict)
|
||||
attn_metadata = attn_metadata.get(self.prefix) # type: ignore
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
||||
num_actual_tokens = (
|
||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||
)
|
||||
else:
|
||||
num_actual_tokens = hidden_states.shape[0]
|
||||
|
||||
# QKV projection
|
||||
qkv, _ = self.query_key_value(hidden_states[:num_actual_tokens])
|
||||
|
||||
# use rotary_emb support fp32
|
||||
qkv = qkv.to(torch.float32)
|
||||
if self.linear_silu:
|
||||
qkv = F.silu(qkv)
|
||||
|
||||
# Split q, k, v
|
||||
q, k, v = torch.split(
|
||||
qkv,
|
||||
[self.q_size_per_rank, self.kv_size_per_rank, self.kv_size_per_rank],
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
# Apply QK norm if needed
|
||||
if self.use_qk_norm:
|
||||
q = q.reshape(-1, self.tp_heads, self.head_dim)
|
||||
k = k.reshape(-1, self.tp_kv_heads, self.head_dim)
|
||||
q = layernorm_fn(
|
||||
q,
|
||||
self.query_layernorm.weight.data,
|
||||
bias=None,
|
||||
eps=self.rms_norm_eps,
|
||||
is_rms_norm=True,
|
||||
)
|
||||
k = layernorm_fn(
|
||||
k,
|
||||
self.key_layernorm.weight.data,
|
||||
bias=None,
|
||||
eps=self.rms_norm_eps,
|
||||
is_rms_norm=True,
|
||||
)
|
||||
q = q.reshape(-1, self.q_size_per_rank)
|
||||
k = k.reshape(-1, self.kv_size_per_rank)
|
||||
|
||||
# Apply rotary embeddings
|
||||
if self.linear_rope:
|
||||
q, k = self.rotary_emb(positions[:num_actual_tokens], q, k)
|
||||
|
||||
# Reshape to [batch, heads, seq_len, head_dim]
|
||||
q = q.view((qkv.shape[0], self.tp_heads, self.head_dim))
|
||||
k = k.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
|
||||
v = v.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
|
||||
|
||||
# Apply scaling if using minimax backend
|
||||
if self.linear_scale:
|
||||
q = q * self.scaling
|
||||
|
||||
# Get KV cache and state indices
|
||||
if attn_metadata is not None:
|
||||
kv_cache = self.kv_cache[0]
|
||||
state_indices_tensor = attn_metadata.state_indices_tensor
|
||||
clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
|
||||
# Compute attention
|
||||
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
|
||||
if attn_metadata is None:
|
||||
hidden = torch.empty(
|
||||
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
|
||||
)
|
||||
else:
|
||||
if not decode_only:
|
||||
hidden = self._prefill_and_mix_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
else:
|
||||
hidden = self._decode_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
|
||||
# Apply group norm and gate (matching SGLang behavior)
|
||||
gate, _ = self.g_proj(hidden_states[:num_actual_tokens])
|
||||
|
||||
if self.group_norm_size > 1:
|
||||
hidden = self.g_norm(hidden, gate)
|
||||
else:
|
||||
hidden = self.g_norm(hidden)
|
||||
hidden = F.sigmoid(gate) * hidden
|
||||
|
||||
hidden = hidden.to(hidden_states.dtype)
|
||||
|
||||
# Output projection
|
||||
dense_out, _ = self.dense(hidden)
|
||||
output[:num_actual_tokens] = dense_out
|
||||
|
||||
def _prefill_and_mix_infer(
|
||||
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
):
|
||||
"""Handle prefill (mixed with decode if any)."""
|
||||
return linear_attention_prefill_and_mix(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
kv_cache=kv_cache,
|
||||
state_indices_tensor=state_indices_tensor,
|
||||
attn_metadata=attn_metadata,
|
||||
slope_rate=self.tp_slope,
|
||||
block_size=self.BLOCK,
|
||||
decode_fn=self._decode_infer,
|
||||
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
|
||||
layer_idx=self.layer_idx,
|
||||
)
|
||||
|
||||
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
|
||||
"""Handle decode (single token per sequence)."""
|
||||
decode_state_indices = getattr(attn_metadata, "state_indices_tensor_d", None)
|
||||
num_accepted_tokens = getattr(attn_metadata, "num_accepted_tokens", None)
|
||||
query_start_loc = getattr(attn_metadata, "query_start_loc_d", None)
|
||||
if (
|
||||
decode_state_indices is not None
|
||||
and decode_state_indices.dim() > 1
|
||||
and num_accepted_tokens is not None
|
||||
and query_start_loc is not None
|
||||
):
|
||||
return bailing_linear_attention_decode_spec(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_cache,
|
||||
self.tp_slope,
|
||||
decode_state_indices,
|
||||
query_start_loc,
|
||||
num_accepted_tokens,
|
||||
q_start=0,
|
||||
q_end=attn_metadata.num_decode_tokens,
|
||||
slot_start=0,
|
||||
slot_end=attn_metadata.num_decodes,
|
||||
block_size=32,
|
||||
)
|
||||
decode_state_indices = (
|
||||
state_indices_tensor
|
||||
if decode_state_indices is None
|
||||
else decode_state_indices
|
||||
)
|
||||
hidden = linear_attention_decode(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_cache,
|
||||
self.tp_slope,
|
||||
decode_state_indices,
|
||||
q_start=0,
|
||||
q_end=attn_metadata.num_decode_tokens,
|
||||
slot_start=0,
|
||||
slot_end=attn_metadata.num_decodes,
|
||||
block_size=32,
|
||||
)
|
||||
return hidden
|
||||
@@ -1,66 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
|
||||
class LinearAttention(PluggableLayer, MambaBase):
|
||||
"""Base class for Linear attention layer."""
|
||||
|
||||
def __init__(
|
||||
self, config: PreTrainedConfig, vllm_config: VllmConfig, prefix: str = ""
|
||||
):
|
||||
super().__init__()
|
||||
self.layer_idx = extract_layer_index(prefix)
|
||||
self.prefix = prefix
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
self.BLOCK = getattr(config, "block", 256)
|
||||
self.hidden_size = config.hidden_size
|
||||
self.num_heads = config.num_attention_heads
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.head_dim = (
|
||||
config.head_dim
|
||||
if hasattr(config, "head_dim")
|
||||
else config.hidden_size // self.num_heads
|
||||
)
|
||||
self.hidden_inner_size = self.head_dim * self.num_heads
|
||||
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
assert self.num_heads % self.tp_size == 0
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.LINEAR
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
return MambaStateDtypeCalculator.linear_attention_state_dtype(
|
||||
self.model_config.dtype,
|
||||
self.cache_config.mamba_cache_dtype,
|
||||
)
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, int, int], ...]:
|
||||
return MambaStateShapeCalculator.linear_attention_state_shape(
|
||||
num_heads=self.num_heads, tp_size=self.tp_size, head_dim=self.head_dim
|
||||
)
|
||||
@@ -1,345 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.lightning_attn import (
|
||||
lightning_attention,
|
||||
linear_decode_forward_triton,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
|
||||
from vllm.model_executor.layers.minimax_rms_norm import MiniMaxText01RMSNormTP
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
|
||||
|
||||
|
||||
def clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
attn_metadata: LinearAttentionMetadata,
|
||||
) -> None:
|
||||
num_prefills = getattr(attn_metadata, "num_prefills", 0)
|
||||
if num_prefills <= 0:
|
||||
return
|
||||
|
||||
num_decode_tokens = getattr(attn_metadata, "num_decode_tokens", 0)
|
||||
for prefill_idx in range(num_prefills):
|
||||
q_start = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx]
|
||||
q_end = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx + 1]
|
||||
query_len = q_end - q_start
|
||||
context_len = (
|
||||
attn_metadata.seq_lens[num_decode_tokens + prefill_idx] - query_len
|
||||
)
|
||||
if context_len == 0:
|
||||
block_to_clear = state_indices_tensor[num_decode_tokens + prefill_idx]
|
||||
kv_cache[block_to_clear, ...] = 0
|
||||
|
||||
|
||||
def linear_attention_decode(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slope_rate: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
q_start: int = 0,
|
||||
q_end: int | None = None,
|
||||
slot_start: int = 0,
|
||||
slot_end: int | None = None,
|
||||
block_size: int = 32,
|
||||
) -> torch.Tensor:
|
||||
q = q[q_start:q_end].unsqueeze(2).contiguous()
|
||||
k = k[q_start:q_end].unsqueeze(2).contiguous()
|
||||
v = v[q_start:q_end].unsqueeze(2).contiguous()
|
||||
slot_id = state_indices_tensor[slot_start:slot_end]
|
||||
return linear_decode_forward_triton(
|
||||
q, k, v, kv_cache, slope_rate, slot_id, block_size
|
||||
)
|
||||
|
||||
|
||||
def linear_attention_prefill_and_mix(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
attn_metadata: LinearAttentionMetadata,
|
||||
slope_rate: torch.Tensor,
|
||||
block_size: int,
|
||||
decode_fn: Callable[..., torch.Tensor],
|
||||
prefix_fn: Callable[..., torch.Tensor],
|
||||
layer_idx: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
hidden = []
|
||||
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
|
||||
if _prefill_idx >= len(attn_metadata.query_start_loc):
|
||||
break
|
||||
if _prefill_idx >= len(state_indices_tensor):
|
||||
break
|
||||
offset = attn_metadata.num_decode_tokens
|
||||
_start = attn_metadata.query_start_loc[offset + _prefill_idx]
|
||||
_end = attn_metadata.query_start_loc[offset + _prefill_idx + 1]
|
||||
slot_id = state_indices_tensor[offset + _prefill_idx]
|
||||
qs = q[_start:_end].transpose(0, 1).contiguous()
|
||||
ks = k[_start:_end].transpose(0, 1).contiguous()
|
||||
vs = v[_start:_end].transpose(0, 1).contiguous()
|
||||
slice_layer_cache = kv_cache[slot_id, ...]
|
||||
out_slice = prefix_fn(
|
||||
qs,
|
||||
ks,
|
||||
vs,
|
||||
slice_layer_cache,
|
||||
slope_rate,
|
||||
block_size,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
hidden.append(out_slice.contiguous())
|
||||
|
||||
if attn_metadata.num_decode_tokens > 0:
|
||||
hidden_decode = decode_fn(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
hidden.insert(0, hidden_decode)
|
||||
|
||||
if not hidden:
|
||||
return torch.empty((0, q.size(-1)), device=q.device, dtype=q.dtype)
|
||||
|
||||
hidden = torch.concat(hidden, dim=0).contiguous()
|
||||
return hidden
|
||||
|
||||
|
||||
class MiniMaxText01LinearKernel:
|
||||
@staticmethod
|
||||
def jit_linear_forward_prefix(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_caches: torch.Tensor,
|
||||
slope_rate: torch.Tensor,
|
||||
block_size: int,
|
||||
layer_idx: int | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
slope_rate = slope_rate.to(torch.float32)
|
||||
should_pad_dim = q.dim() == 3
|
||||
if should_pad_dim:
|
||||
q = q.unsqueeze(0)
|
||||
k = k.unsqueeze(0)
|
||||
v = v.unsqueeze(0)
|
||||
b, h, n, d = q.shape
|
||||
e = d
|
||||
kv_history = kv_caches.reshape(1, h, d, e).contiguous()
|
||||
output, kv_history = lightning_attention(
|
||||
q, k, v, slope_rate, block_size=block_size, kv_history=kv_history
|
||||
)
|
||||
kv_caches.copy_(kv_history[:, :, -1, :, :].reshape(h, d, e))
|
||||
assert output.shape[0] == 1, "batch size must be 1"
|
||||
return rearrange(output.squeeze(0), "h n d -> n (h d)")
|
||||
|
||||
|
||||
@PluggableLayer.register("minimax_text_01_attention")
|
||||
class MiniMaxText01LinearAttention(LinearAttention):
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
vllm_config,
|
||||
prefix: str = "linear_attn",
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
self.tp_heads = self.num_heads // self.tp_size
|
||||
self.qkv_size = self.num_heads * self.head_dim
|
||||
self.tp_hidden = self.head_dim * self.tp_heads
|
||||
|
||||
self.qkv_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_inner_size * 3,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.output_gate = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_inner_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.output_gate",
|
||||
)
|
||||
self.out_proj = RowParallelLinear(
|
||||
self.hidden_inner_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
self.norm = MiniMaxText01RMSNormTP(
|
||||
self.hidden_inner_size,
|
||||
eps=1e-5,
|
||||
)
|
||||
|
||||
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
|
||||
if self.num_hidden_layers <= 1:
|
||||
self.slope_rate = slope_rate * (1 + 1e-5)
|
||||
else:
|
||||
self.slope_rate = slope_rate * (
|
||||
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
|
||||
)
|
||||
self.tp_slope = self.slope_rate[
|
||||
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
|
||||
].contiguous()
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
@staticmethod
|
||||
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
assert param.size() == loaded_weight.size()
|
||||
param.data.copy_(loaded_weight)
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def _build_slope_tensor(n_attention_heads: int):
|
||||
def get_slopes(n):
|
||||
def get_slopes_power_of_2(n):
|
||||
start = 2 ** (-(2 ** -(math.log2(n) - 3)))
|
||||
ratio = start
|
||||
return [start * ratio**i for i in range(n)]
|
||||
|
||||
if math.log2(n).is_integer():
|
||||
return get_slopes_power_of_2(n)
|
||||
else:
|
||||
closest_power_of_2 = 2 ** math.floor(math.log2(n))
|
||||
return (
|
||||
get_slopes_power_of_2(closest_power_of_2)
|
||||
+ get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2]
|
||||
)
|
||||
|
||||
slopes = torch.tensor(
|
||||
get_slopes(n_attention_heads), dtype=torch.float32
|
||||
).reshape(n_attention_heads, 1, 1)
|
||||
return slopes
|
||||
|
||||
def _prefill_and_mix_infer(
|
||||
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
):
|
||||
return linear_attention_prefill_and_mix(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
kv_cache=kv_cache,
|
||||
state_indices_tensor=state_indices_tensor,
|
||||
attn_metadata=attn_metadata,
|
||||
slope_rate=self.tp_slope,
|
||||
block_size=self.BLOCK,
|
||||
decode_fn=self._decode_infer,
|
||||
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
|
||||
layer_idx=self.layer_idx,
|
||||
)
|
||||
|
||||
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
|
||||
hidden = linear_attention_decode(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_cache,
|
||||
self.tp_slope,
|
||||
state_indices_tensor,
|
||||
q_start=0,
|
||||
q_end=attn_metadata.num_decode_tokens,
|
||||
slot_start=0,
|
||||
slot_end=attn_metadata.num_decodes,
|
||||
block_size=32,
|
||||
)
|
||||
return hidden
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
|
||||
) -> None:
|
||||
torch.ops.vllm.linear_attention(
|
||||
hidden_states,
|
||||
output,
|
||||
positions,
|
||||
self.prefix,
|
||||
)
|
||||
|
||||
def _forward(
|
||||
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
||||
num_actual_tokens = (
|
||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||
)
|
||||
else:
|
||||
num_actual_tokens = hidden_states.shape[0]
|
||||
|
||||
qkv, _ = self.qkv_proj(hidden_states[:num_actual_tokens])
|
||||
qkv32 = qkv.to(torch.float32)
|
||||
qkvact = torch.nn.functional.silu(qkv32)
|
||||
qkvact = qkvact.view((qkv.shape[0], self.tp_heads, -1))
|
||||
q, k, v = torch.split(qkvact, [self.head_dim] * 3, dim=-1)
|
||||
if attn_metadata is not None:
|
||||
kv_cache = self.kv_cache[0]
|
||||
state_indices_tensor = attn_metadata.state_indices_tensor
|
||||
clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
|
||||
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
|
||||
if attn_metadata is None:
|
||||
hidden = torch.empty(
|
||||
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
|
||||
)
|
||||
else:
|
||||
if not decode_only:
|
||||
hidden = self._prefill_and_mix_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
else:
|
||||
hidden = self._decode_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
hidden = self.norm._forward(hidden)
|
||||
gate, _ = self.output_gate(hidden_states[:num_actual_tokens])
|
||||
hidden = F.sigmoid(gate) * hidden
|
||||
hidden = hidden.to(hidden_states.dtype)
|
||||
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def linear_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._forward(hidden_states=hidden_states, output=output, positions=positions)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="linear_attention",
|
||||
op_func=linear_attention,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
@@ -1,485 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_scan_fn
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
|
||||
# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
|
||||
# --8<-- [start:mamba_mixer]
|
||||
@PluggableLayer.register("mamba_mixer")
|
||||
class MambaMixer(MambaBase, PluggableLayer):
|
||||
"""Compute ∆, A, B, C, and D the state space parameters and compute
|
||||
the `contextualized_states`. A, D are input independent
|
||||
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
|
||||
for why A isn't selective) ∆, B, C are input-dependent
|
||||
(this is a key difference between Mamba and the linear time
|
||||
invariant S4, and is why Mamba is called
|
||||
**selective** state spaces)
|
||||
"""
|
||||
|
||||
# --8<-- [end:mamba_mixer]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
ssm_state_size: int,
|
||||
conv_kernel_size: int,
|
||||
intermediate_size: int,
|
||||
time_step_rank: int,
|
||||
use_conv_bias: bool,
|
||||
use_bias: bool,
|
||||
use_rms_norm: bool,
|
||||
rms_norm_has_weight: bool = True,
|
||||
rms_norm_eps: float = 1e-5,
|
||||
activation="silu",
|
||||
is_lora_enabled: bool = False,
|
||||
model_config: ModelConfig | None = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.time_step_rank = time_step_rank
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.use_rms_norm = use_rms_norm
|
||||
self.activation = activation
|
||||
self.is_lora_enabled = is_lora_enabled
|
||||
self.conv_kernel_size = conv_kernel_size
|
||||
self.intermediate_size = intermediate_size
|
||||
|
||||
self.conv1d = ColumnParallelLinear(
|
||||
input_size=conv_kernel_size,
|
||||
output_size=intermediate_size,
|
||||
bias=use_conv_bias,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
# unsqueeze to fit conv1d weights shape into the linear weights shape.
|
||||
# Can't do this in `weight_loader` since it already exists in
|
||||
# `ColumnParallelLinear` and `set_weight_attrs`
|
||||
# doesn't allow to override it
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
|
||||
self.in_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=use_bias,
|
||||
prefix=f"{prefix}.in_proj",
|
||||
)
|
||||
|
||||
# selective projection used to make dt, B and C input dependent
|
||||
self.x_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
time_step_rank + ssm_state_size * 2,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.x_proj",
|
||||
)
|
||||
# time step projection (discretization) -
|
||||
# In the forward we need to apply dt_proj without the bias,
|
||||
# as the bias is added in the selective scan kernel.
|
||||
self.dt_proj = ColumnParallelLinear(
|
||||
time_step_rank,
|
||||
intermediate_size,
|
||||
bias=True,
|
||||
skip_bias_add=True,
|
||||
prefix=f"{prefix}.dt_proj",
|
||||
)
|
||||
|
||||
def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
param.data.copy_(
|
||||
loaded_weight.data.split(loaded_weight.shape[0] // tp_size, dim=0)[
|
||||
tp_rank
|
||||
]
|
||||
)
|
||||
|
||||
def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
|
||||
weight_loader(param, -torch.exp(loaded_weight.float()))
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.A = nn.Parameter(
|
||||
torch.empty(
|
||||
intermediate_size // tp_size,
|
||||
ssm_state_size,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
)
|
||||
self.D = nn.Parameter(torch.ones(intermediate_size // tp_size))
|
||||
|
||||
set_weight_attrs(self.D, {"weight_loader": weight_loader})
|
||||
set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
|
||||
|
||||
self.out_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=use_bias,
|
||||
input_is_parallel=True,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
|
||||
self.dt_layernorm = (
|
||||
RMSNorm(
|
||||
time_step_rank,
|
||||
eps=rms_norm_eps,
|
||||
has_weight=rms_norm_has_weight,
|
||||
)
|
||||
if use_rms_norm
|
||||
else None
|
||||
)
|
||||
|
||||
self.b_layernorm = (
|
||||
RMSNorm(
|
||||
ssm_state_size,
|
||||
eps=rms_norm_eps,
|
||||
has_weight=rms_norm_has_weight,
|
||||
)
|
||||
if use_rms_norm
|
||||
else None
|
||||
)
|
||||
|
||||
self.c_layernorm = (
|
||||
RMSNorm(
|
||||
ssm_state_size,
|
||||
eps=rms_norm_eps,
|
||||
has_weight=rms_norm_has_weight,
|
||||
)
|
||||
if use_rms_norm
|
||||
else None
|
||||
)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
# The inner tuple is (conv_state, ssm_state)
|
||||
self.kv_cache = (torch.tensor([]), torch.tensor([]))
|
||||
|
||||
self.model_config = model_config
|
||||
self.cache_config = cache_config
|
||||
self.prefix = prefix
|
||||
|
||||
def _ssm_transform(
|
||||
self, x: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# LoRA kernel requires contiguous tensor.
|
||||
# ROCm: Non-contiguous tensors cause incorrect GEMM
|
||||
# results when batch > 1.
|
||||
if self.is_lora_enabled or current_platform.is_rocm():
|
||||
x = x.contiguous()
|
||||
ssm_params = self.x_proj(x)[0]
|
||||
time_step, B, C = torch.split(
|
||||
ssm_params,
|
||||
[self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
|
||||
dim=-1,
|
||||
)
|
||||
if self.use_rms_norm:
|
||||
assert self.dt_layernorm is not None
|
||||
assert self.b_layernorm is not None
|
||||
assert self.c_layernorm is not None
|
||||
time_step = self.dt_layernorm(time_step.contiguous())
|
||||
B = self.b_layernorm(B.contiguous())
|
||||
C = self.c_layernorm(C.contiguous())
|
||||
|
||||
# ROCm: tensor from split is non-contiguous, causing incorrect
|
||||
# GEMM results in dt_proj.
|
||||
if current_platform.is_rocm():
|
||||
time_step = time_step.contiguous()
|
||||
|
||||
discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
|
||||
return discrete_time_step, B, C
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor, output: torch.Tensor):
|
||||
torch.ops.vllm.mamba_mixer(
|
||||
hidden_states,
|
||||
output,
|
||||
_encode_layer_name(self.prefix),
|
||||
)
|
||||
|
||||
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
|
||||
"""Run the Mamba-1 SSM pipeline.
|
||||
|
||||
Steps
|
||||
-----
|
||||
1. Apply the gated-MLP linear projection to the raw input.
|
||||
2. Pass the projected sequence through the convolutional mixing layer.
|
||||
3. Feed the result into the State-Space Model (SSM) blocks.
|
||||
4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
|
||||
to produce contextual representations.
|
||||
5. Project the contextualised sequence back
|
||||
to the output embedding dimension.
|
||||
|
||||
Batch handling
|
||||
--------------
|
||||
Prefill and decode tokens are processed by dedicated CUDA
|
||||
kernels for both the convolutional (conv1d) and SSM stages.
|
||||
In the case of a mixed batch (containing both prefill and
|
||||
decode tokens), both sets of kernels are executed independently
|
||||
and their outputs are concatenated before the final output projection.
|
||||
"""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
assert self.cache_config is not None
|
||||
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, Mamba1AttentionMetadata)
|
||||
query_start_loc_p = attn_metadata.query_start_loc_p
|
||||
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
|
||||
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
|
||||
conv_state = (
|
||||
self.kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self.kv_cache[0].transpose(-1, -2)
|
||||
)
|
||||
ssm_state = self.kv_cache[1]
|
||||
has_initial_states_p = attn_metadata.has_initial_states_p
|
||||
cu_chunk_seqlen_p = attn_metadata.cu_chunk_seqlen_p
|
||||
last_chunk_indices_p = attn_metadata.last_chunk_indices_p
|
||||
|
||||
# 1. Gated MLP's linear projection
|
||||
projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
|
||||
hidden_states_BC, gate = projected_states.chunk(2, dim=-2)
|
||||
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
# V1 profile run
|
||||
hidden_states_BC = hidden_states_BC.contiguous()
|
||||
return self.out_proj(hidden_states_BC.transpose(-2, -1))[0]
|
||||
|
||||
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
has_prefill = num_prefill_tokens > 0
|
||||
has_decode = num_decode_tokens > 0
|
||||
num_actual_tokens = num_prefill_tokens + num_decode_tokens
|
||||
|
||||
prefill_decode_split = split_batch_to_prefill_and_decode(
|
||||
hidden_states_BC,
|
||||
gate,
|
||||
num_prefill_tokens,
|
||||
num_decode_tokens,
|
||||
)
|
||||
hidden_states_BC_p = prefill_decode_split.hidden_states_BC_p
|
||||
hidden_states_BC_d = prefill_decode_split.hidden_states_BC_d
|
||||
gate_p = prefill_decode_split.gate_p
|
||||
gate_d = prefill_decode_split.gate_d
|
||||
|
||||
ssm_outputs = []
|
||||
|
||||
if has_prefill:
|
||||
# 2. Convolution sequence transformation
|
||||
conv_out_p = causal_conv1d_fn(
|
||||
hidden_states_BC_p,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation=self.activation,
|
||||
conv_states=conv_state,
|
||||
has_initial_state=has_initial_states_p,
|
||||
cache_indices=state_indices_tensor_p,
|
||||
query_start_loc=query_start_loc_p,
|
||||
metadata=attn_metadata,
|
||||
)
|
||||
# 3. State Space Model sequence transformations.
|
||||
discrete_time_step_p, B_p, C_p = self._ssm_transform(
|
||||
conv_out_p.transpose(-2, -1)
|
||||
)
|
||||
time_proj_bias = self._time_proj_bias()
|
||||
|
||||
# 4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
|
||||
scan_out_p = selective_scan_fn(
|
||||
conv_out_p,
|
||||
ssm_state,
|
||||
discrete_time_step_p,
|
||||
self.A,
|
||||
B_p.transpose(-2, -1),
|
||||
C_p.transpose(-2, -1),
|
||||
self.D.float(),
|
||||
gate_p,
|
||||
time_proj_bias,
|
||||
delta_softplus=True,
|
||||
cache_indices=state_indices_tensor_p,
|
||||
has_initial_state=has_initial_states_p,
|
||||
query_start_loc=query_start_loc_p,
|
||||
cu_chunk_seqlen=cu_chunk_seqlen_p,
|
||||
last_chunk_indices=last_chunk_indices_p,
|
||||
)
|
||||
ssm_outputs.append(scan_out_p)
|
||||
|
||||
if has_decode:
|
||||
# state_indices_tensor_d is assigned when attn_metadata is not None,
|
||||
# and has_decode is only True when attn_metadata is not None
|
||||
assert state_indices_tensor_d is not None
|
||||
# 2. Convolution sequence transformation
|
||||
conv_out_d = causal_conv1d_update(
|
||||
hidden_states_BC_d.transpose(0, 1),
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
self.activation,
|
||||
conv_state_indices=state_indices_tensor_d,
|
||||
).transpose(0, 1)
|
||||
|
||||
# 3. State Space Model sequence transformation.
|
||||
discrete_time_step_d, B_d, C_d = self._ssm_transform(
|
||||
conv_out_d.transpose(-2, -1)
|
||||
)
|
||||
time_proj_bias = self._time_proj_bias()
|
||||
|
||||
# 4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
|
||||
scan_outputs_d = torch.empty_like(hidden_states_BC_d.transpose(0, 1))
|
||||
selective_state_update(
|
||||
ssm_state,
|
||||
conv_out_d.transpose(0, 1),
|
||||
discrete_time_step_d.transpose(0, 1),
|
||||
self.A,
|
||||
B_d,
|
||||
C_d,
|
||||
self.D,
|
||||
time_proj_bias,
|
||||
z=gate_d.transpose(0, 1),
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_indices_tensor_d,
|
||||
dst_state_batch_indices=state_indices_tensor_d,
|
||||
out=scan_outputs_d,
|
||||
)
|
||||
scan_outputs_d = scan_outputs_d.transpose(0, 1)
|
||||
|
||||
ssm_outputs.insert(0, scan_outputs_d)
|
||||
|
||||
scan_outputs_combined = (
|
||||
ssm_outputs[0] if len(ssm_outputs) == 1 else torch.cat(ssm_outputs, dim=-1)
|
||||
)
|
||||
|
||||
# 5. Final output projection
|
||||
if self.is_lora_enabled: # Lora kernel requires contiguous tensor.
|
||||
scan_outputs_combined = scan_outputs_combined.transpose(-2, -1).contiguous()
|
||||
out = self.out_proj(scan_outputs_combined)[0]
|
||||
else:
|
||||
out = self.out_proj(scan_outputs_combined.transpose(-2, -1))[0]
|
||||
|
||||
output[:num_actual_tokens] = out
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
return MambaStateDtypeCalculator.mamba1_state_dtype(
|
||||
self.model_config.dtype,
|
||||
self.cache_config.mamba_cache_dtype,
|
||||
self.cache_config.mamba_ssm_cache_dtype,
|
||||
)
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.mamba1_state_shape(
|
||||
tp_world_size=get_tensor_model_parallel_world_size(),
|
||||
intermediate_size=self.intermediate_size,
|
||||
state_size=self.ssm_state_size,
|
||||
conv_kernel=self.conv_kernel_size,
|
||||
)
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.MAMBA1
|
||||
|
||||
def _time_proj_bias(self) -> torch.Tensor | None:
|
||||
if hasattr(self.dt_proj, "bias") and self.dt_proj.bias is not None:
|
||||
return self.dt_proj.bias.float()
|
||||
return None
|
||||
|
||||
|
||||
class PrefillDecodeSplit(NamedTuple):
|
||||
hidden_states_BC_p: torch.Tensor
|
||||
hidden_states_BC_d: torch.Tensor
|
||||
gate_p: torch.Tensor
|
||||
gate_d: torch.Tensor
|
||||
|
||||
|
||||
def split_batch_to_prefill_and_decode(
|
||||
hidden_states_BC: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
num_prefill_tokens: int,
|
||||
num_decode_tokens: int,
|
||||
) -> PrefillDecodeSplit:
|
||||
num_actual_tokens = num_prefill_tokens + num_decode_tokens
|
||||
|
||||
# In v1, decode tokens come first, then prefill tokens.
|
||||
hidden_states_BC_d, hidden_states_BC_p = torch.split(
|
||||
hidden_states_BC[..., :num_actual_tokens],
|
||||
[num_decode_tokens, num_prefill_tokens],
|
||||
dim=-1,
|
||||
)
|
||||
gate_d, gate_p = torch.split(
|
||||
gate[..., :num_actual_tokens], [num_decode_tokens, num_prefill_tokens], dim=-1
|
||||
)
|
||||
|
||||
return PrefillDecodeSplit(
|
||||
hidden_states_BC_p=hidden_states_BC_p,
|
||||
hidden_states_BC_d=hidden_states_BC_d,
|
||||
gate_p=gate_p,
|
||||
gate_d=gate_d,
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def mamba_mixer(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self.forward_impl(hidden_states=hidden_states, output=output)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mamba_mixer",
|
||||
op_func=mamba_mixer,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_initial_states_kernel(
|
||||
state_ptr,
|
||||
indices_ptr,
|
||||
has_initial_state_ptr,
|
||||
output_ptr,
|
||||
stride_state_batch,
|
||||
stride_indices,
|
||||
stride_has_initial_state,
|
||||
row_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
block_idx = tl.program_id(0)
|
||||
batch_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < row_size
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
has_initial_state = tl.load(
|
||||
has_initial_state_ptr + batch_idx * stride_has_initial_state
|
||||
).to(tl.int1)
|
||||
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
|
||||
state_idx = tl.where(has_initial_state, state_idx, 0)
|
||||
values = tl.load(
|
||||
state_ptr + state_idx * stride_state_batch + offsets,
|
||||
mask=mask & has_initial_state,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(output_ptr + batch_idx * row_size + offsets, values, mask=mask)
|
||||
|
||||
|
||||
def gather_initial_states(
|
||||
state: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
has_initial_state: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Gather dense state rows, replacing uninitialized rows with zeros."""
|
||||
assert state.ndim >= 2
|
||||
assert state.is_cuda or state.is_xpu
|
||||
assert indices.ndim == 1 and has_initial_state.ndim == 1
|
||||
assert indices.shape == has_initial_state.shape
|
||||
assert indices.device == state.device
|
||||
assert has_initial_state.device == state.device
|
||||
assert indices.dtype in (torch.int32, torch.int64)
|
||||
assert has_initial_state.dtype == torch.bool
|
||||
|
||||
row_size = state[0].numel()
|
||||
# Mamba pages may pad stride(0), but each state row remains dense.
|
||||
assert state[0].is_contiguous()
|
||||
output = torch.empty(
|
||||
(indices.numel(), *state.shape[1:]),
|
||||
dtype=state.dtype,
|
||||
device=state.device,
|
||||
)
|
||||
block_size = min(triton.next_power_of_2(row_size), 1024)
|
||||
grid = (triton.cdiv(row_size, block_size), indices.numel())
|
||||
_gather_initial_states_kernel[grid](
|
||||
state,
|
||||
indices,
|
||||
has_initial_state,
|
||||
output,
|
||||
state.stride(0),
|
||||
indices.stride(0),
|
||||
has_initial_state.stride(0),
|
||||
row_size=row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return output
|
||||
@@ -1,105 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
DispatchSpec,
|
||||
TritonWarmupTensor,
|
||||
triton_kernel_dispatcher_with_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _scatter_states_kernel(
|
||||
state_ptr,
|
||||
src_ptr,
|
||||
indices_ptr,
|
||||
stride_state_batch,
|
||||
stride_src_batch,
|
||||
stride_indices,
|
||||
row_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
block_idx = tl.program_id(0)
|
||||
batch_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < row_size
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
|
||||
values = tl.load(src_ptr + batch_idx * stride_src_batch + offsets, mask=mask)
|
||||
tl.store(state_ptr + state_idx * stride_state_batch + offsets, values, mask=mask)
|
||||
|
||||
|
||||
def _scatter_states_warmup_inputs(
|
||||
*,
|
||||
state_shape: tuple[int, ...],
|
||||
state_dtype: torch.dtype,
|
||||
indices_dtype: torch.dtype,
|
||||
max_num_tokens: int,
|
||||
) -> dict[str, object]:
|
||||
num_tokens: Any = WarmupIntRange(1, max_num_tokens + 1)
|
||||
return dict(
|
||||
state=TritonWarmupTensor(
|
||||
state_dtype,
|
||||
shape=(1,) + state_shape,
|
||||
),
|
||||
src=TritonWarmupTensor(
|
||||
state_dtype,
|
||||
shape=(num_tokens,) + state_shape,
|
||||
),
|
||||
indices=TritonWarmupTensor(
|
||||
indices_dtype,
|
||||
shape=(num_tokens,),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_scatter_states_kernel,
|
||||
warmup_inputs=_scatter_states_warmup_inputs,
|
||||
)
|
||||
def scatter_states(
|
||||
state: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> DispatchSpec:
|
||||
"""Scatter ``src`` rows into ``state`` at ``indices`` (in place).
|
||||
|
||||
Equivalent to ``state[indices] = src`` but non-atomic and bandwidth-bound,
|
||||
since mamba cache slots are unique per sequence. ``gather_initial_states``
|
||||
is the read-side counterpart.
|
||||
"""
|
||||
assert state.ndim >= 2
|
||||
assert state.is_cuda
|
||||
assert src.ndim == state.ndim
|
||||
assert indices.ndim == 1
|
||||
assert indices.device == state.device
|
||||
assert src.shape[1:] == state.shape[1:]
|
||||
assert src.shape[0] == indices.shape[0]
|
||||
assert indices.dtype in (torch.int32, torch.int64)
|
||||
|
||||
row_size = state[0].numel()
|
||||
assert state[0].is_contiguous()
|
||||
assert src[0].is_contiguous()
|
||||
block_size = min(triton.next_power_of_2(row_size), 1024)
|
||||
grid = (triton.cdiv(row_size, block_size), indices.numel())
|
||||
return grid, dict(
|
||||
stride_state_batch=state.stride(0),
|
||||
stride_src_batch=src.stride(0),
|
||||
stride_indices=indices.stride(0),
|
||||
row_size=row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
Reference in New Issue
Block a user