[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,646 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_rank
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
sharded_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
from ...linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from ..mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from ..ops.causal_conv1d import causal_conv1d_fn, causal_conv1d_update
|
||||
from ..ops.gather_initial_states import gather_initial_states
|
||||
|
||||
# Empirical lower bound for the KDA gate to avoid numerical underflow.
|
||||
_KDA_GATE_LOGBOUND_MIN = -5.0
|
||||
|
||||
|
||||
def a_log_weight_loader(
|
||||
shard_axis: int,
|
||||
) -> Callable[[torch.Tensor, torch.Tensor], None]:
|
||||
"""Load KDA A_log stored as either old 4D or current 1D weights."""
|
||||
|
||||
def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
shard_size = param.data.shape[shard_axis]
|
||||
start_idx = tp_rank * shard_size
|
||||
|
||||
if loaded_weight.dim() == 4:
|
||||
assert loaded_weight.shape[:2] == (1, 1), (
|
||||
f"Expected old A_log shape (1, 1, H, 1), got {loaded_weight.shape}"
|
||||
)
|
||||
assert loaded_weight.shape[-1] == 1, (
|
||||
f"Expected old A_log last dim to be 1, got {loaded_weight.shape}"
|
||||
)
|
||||
loaded_weight = loaded_weight.view(loaded_weight.shape[2])
|
||||
|
||||
loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size)
|
||||
return default_weight_loader(param, loaded_weight)
|
||||
|
||||
return loader
|
||||
|
||||
|
||||
def _make_fused_conv1d_weight_loader(
|
||||
dims: list[int],
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
) -> Callable[..., None]:
|
||||
sharded_dims = [dim // tp_size for dim in dims]
|
||||
|
||||
def weight_loader(
|
||||
param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: int,
|
||||
) -> None:
|
||||
if loaded_weight.dim() == 2:
|
||||
loaded_weight = loaded_weight.unsqueeze(1)
|
||||
shard_size = sharded_dims[loaded_shard_id]
|
||||
source_start = tp_rank * shard_size
|
||||
target_start = sum(sharded_dims[:loaded_shard_id])
|
||||
loaded_shard = loaded_weight[source_start : source_start + shard_size]
|
||||
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
|
||||
|
||||
return weight_loader
|
||||
|
||||
|
||||
class _KimiGDNMergedColumnParallelLinear(MergedColumnParallelLinear):
|
||||
"""Merged projection with one output replicated across TP ranks.
|
||||
|
||||
The replicated shard is represented as ``size * tp_size`` so the merged
|
||||
parameter reserves ``size`` local rows on every rank. Loading that shard
|
||||
from rank zero then gives every rank the complete checkpoint weight.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_size: int,
|
||||
output_sizes: list[int],
|
||||
replicated_shard_id: int,
|
||||
tp_size: int,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.replicated_shard_id = replicated_shard_id
|
||||
output_sizes = output_sizes.copy()
|
||||
output_sizes[replicated_shard_id] *= tp_size
|
||||
super().__init__(input_size, output_sizes, **kwargs)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
def weight_loader_v2(
|
||||
self,
|
||||
param: BasevLLMParameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader_v2(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
|
||||
@PluggableLayer.register("kimi_gated_delta_net_attention")
|
||||
class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
def get_state_dtype(
|
||||
self,
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
if self.model_config is None or self.cache_config is None:
|
||||
raise ValueError("model_config and cache_config must be set")
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
self.model_config.dtype, self.cache_config.mamba_cache_dtype
|
||||
)
|
||||
|
||||
def get_state_shape(
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
self.tp_size,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
conv_kernel_size=self.conv_size,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
kda_config = config.linear_attn_config # type: ignore[attr-defined]
|
||||
assert kda_config is not None, "linear_attn_config must be set"
|
||||
self.head_dim = kda_config["head_dim"]
|
||||
self.num_heads = kda_config["num_heads"]
|
||||
assert self.num_heads % self.tp_size == 0
|
||||
self.local_num_heads = divide(self.num_heads, self.tp_size)
|
||||
|
||||
self.projection_size = self.head_dim * self.num_heads
|
||||
self.local_projection_size = divide(self.projection_size, self.tp_size)
|
||||
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||
self.use_full_rank_gate = kda_config.get("use_full_rank_gate", False)
|
||||
|
||||
if self.use_full_rank_gate:
|
||||
# Keep f_a before the narrow beta shard, then pad each TP-local row
|
||||
# to select the aligned BF16 GEMM path. The padding also avoids an
|
||||
# Inductor correctness issue seen with the row-strided G view.
|
||||
qkvg_output_sizes = [self.projection_size] * 4
|
||||
in_proj_output_sizes = qkvg_output_sizes + [
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
]
|
||||
local_output_size = (
|
||||
4 * self.local_projection_size + self.head_dim + self.local_num_heads
|
||||
)
|
||||
self.in_proj_padding = -local_output_size % 16
|
||||
if self.in_proj_padding:
|
||||
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
|
||||
else:
|
||||
in_proj_output_sizes = [self.projection_size] * 3 + [
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
]
|
||||
self.in_proj_padding = 0
|
||||
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
in_proj_output_sizes,
|
||||
replicated_shard_id=4,
|
||||
tp_size=self.tp_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.in_proj_qkvgfab",
|
||||
)
|
||||
if self.in_proj_padding:
|
||||
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
|
||||
|
||||
self.f_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
self.projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
)
|
||||
self.dt_bias = nn.Parameter(
|
||||
torch.empty(self.local_projection_size, dtype=torch.float32)
|
||||
)
|
||||
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
# One packed parameter and cache let decode run a single conv update.
|
||||
# Prefill slices them back into Q/K/V to obtain dense outputs cheaply.
|
||||
self.conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=3 * self.projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
delattr(self.conv1d.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.conv1d.weight,
|
||||
{
|
||||
"weight_loader": _make_fused_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
self.A_log = nn.Parameter(
|
||||
torch.empty(self.local_num_heads, dtype=torch.float32)
|
||||
)
|
||||
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
|
||||
|
||||
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
|
||||
if self.gate_lower_bound is not None:
|
||||
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
|
||||
"KDA gate lower bound must be in "
|
||||
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
|
||||
f"Got {self.gate_lower_bound}."
|
||||
)
|
||||
self.use_safe_gate = self.gate_lower_bound is not None
|
||||
additional_config = vllm_config.additional_config
|
||||
backend = (
|
||||
additional_config.get("kda_prefill_backend", "auto")
|
||||
if isinstance(additional_config, dict)
|
||||
else "auto"
|
||||
)
|
||||
backend = "triton" if backend == "auto" else backend
|
||||
assert backend == "triton", (
|
||||
"The shared Kimi GDN layer only supports the Triton KDA "
|
||||
f"prefill backend, got {backend!r}."
|
||||
)
|
||||
if not self.use_full_rank_gate:
|
||||
self.g_a_proj = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_a_proj",
|
||||
)
|
||||
self.g_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
self.projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
)
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.projection_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def rearrange_mixed_qkv(
|
||||
self, mixed_qkv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
seq_len = mixed_qkv.shape[0]
|
||||
qkv = mixed_qkv.view(seq_len, 3, self.local_num_heads, self.head_dim)
|
||||
# Materialize all three row-strided inputs with one token-major to
|
||||
# QKV-major permutation. Each unbound tensor is then contiguous.
|
||||
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
|
||||
return qkv.unbind(0)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
|
||||
if self.use_full_rank_gate:
|
||||
split_sizes = [
|
||||
3 * self.local_projection_size,
|
||||
self.local_projection_size,
|
||||
self.head_dim,
|
||||
self.local_num_heads,
|
||||
]
|
||||
if self.in_proj_padding:
|
||||
split_sizes.append(self.in_proj_padding)
|
||||
projected = projected_qkvgfab.split(split_sizes, dim=-1)
|
||||
mixed_qkv, g_proj_states, f_a, beta = projected[:4]
|
||||
else:
|
||||
mixed_qkv, beta, f_a = projected_qkvgfab.split(
|
||||
[
|
||||
3 * self.local_projection_size,
|
||||
self.local_num_heads,
|
||||
self.head_dim,
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0]
|
||||
|
||||
g1 = self.f_b_proj(f_a)[0]
|
||||
beta = beta.unsqueeze(0)
|
||||
g1 = rearrange(g1, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
|
||||
g2 = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim)
|
||||
|
||||
core_attn_out = torch.empty(
|
||||
(1, num_tokens, self.local_num_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
self._forward(
|
||||
mixed_qkv=mixed_qkv,
|
||||
g1=g1,
|
||||
g2=g2,
|
||||
beta=beta,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
|
||||
return self.o_proj(core_attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
def _forward(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
g2: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata_raw is None:
|
||||
return
|
||||
|
||||
# Vendor-specific KDA kernels: AMD/ROCm and NVIDIA keep their own copies
|
||||
# under kimi_k3/{amd,nvidia}/ops so each can diverge independently.
|
||||
# These copies may have different signatures for Kimi-K3, but they agree
|
||||
# on the arguments used here.
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
elif current_platform.is_rocm():
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import ( # type: ignore[assignment]
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
else:
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata_narrowed is None:
|
||||
# Profile/warmup dummy runs skip mamba-family metadata.
|
||||
return
|
||||
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
||||
m = attn_metadata_narrowed
|
||||
has_initial_state = m.has_initial_state
|
||||
non_spec_query_start_loc = m.non_spec_query_start_loc
|
||||
non_spec_state_indices_tensor = m.non_spec_state_indices_tensor
|
||||
spec_sequence_masks = m.spec_sequence_masks
|
||||
spec_token_indx = m.spec_token_indx
|
||||
non_spec_token_indx = m.non_spec_token_indx
|
||||
spec_state_indices_tensor = m.spec_state_indices_tensor
|
||||
spec_query_start_loc = m.spec_query_start_loc
|
||||
num_accepted_tokens = m.num_accepted_tokens
|
||||
num_actual_tokens = m.num_actual_tokens
|
||||
mixed_qkv = mixed_qkv[:num_actual_tokens]
|
||||
g1 = g1[:, :num_actual_tokens]
|
||||
beta = beta[:, :num_actual_tokens]
|
||||
|
||||
constant_caches = self.kv_cache
|
||||
|
||||
conv_state, recurrent_state = constant_caches
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
q_conv_weight, k_conv_weight, v_conv_weight = conv_weights.split(
|
||||
self.local_projection_size, dim=0
|
||||
)
|
||||
q_conv_state, k_conv_state, v_conv_state = conv_state.split(
|
||||
self.local_projection_size, dim=-2
|
||||
)
|
||||
|
||||
# Split tokens into the multi-query spec-decode part and the remaining
|
||||
# (prefill / plain decode) part.
|
||||
if spec_sequence_masks is not None:
|
||||
if m.num_prefills == 0 and m.num_decodes == 0:
|
||||
mixed_qkv_spec = mixed_qkv
|
||||
g1_spec, beta_spec = g1, beta
|
||||
mixed_qkv_ns = g1_ns = beta_ns = None
|
||||
else:
|
||||
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
|
||||
g1_spec = g1.index_select(1, spec_token_indx)
|
||||
beta_spec = beta.index_select(1, spec_token_indx)
|
||||
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
|
||||
g1_ns = g1.index_select(1, non_spec_token_indx)
|
||||
beta_ns = beta.index_select(1, non_spec_token_indx)
|
||||
else:
|
||||
mixed_qkv_spec = g1_spec = beta_spec = None
|
||||
mixed_qkv_ns, g1_ns, beta_ns = mixed_qkv, g1, beta
|
||||
|
||||
# ---------- spec-decode multi-query path ----------
|
||||
core_attn_out_spec = None
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert spec_query_start_loc is not None
|
||||
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
|
||||
spec_max_query_len = spec_state_indices_tensor.size(-1)
|
||||
|
||||
# Sibling beta and, for full-rank gates, output-gate views remain
|
||||
# live, so write the convolution output separately.
|
||||
spec_conv_out = torch.empty(
|
||||
mixed_qkv_spec.shape,
|
||||
dtype=mixed_qkv_spec.dtype,
|
||||
device=mixed_qkv_spec.device,
|
||||
)
|
||||
mixed_qkv_spec = causal_conv1d_update(
|
||||
mixed_qkv_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=spec_conv_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=spec_max_query_len,
|
||||
validate_data=False,
|
||||
out=spec_conv_out,
|
||||
)
|
||||
q_spec, k_spec, v_spec = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
|
||||
)
|
||||
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
|
||||
# Spec-only batches write directly into core_attn_out.
|
||||
spec_out = (
|
||||
core_attn_out[:, : q_spec.shape[1]]
|
||||
if m.num_prefills == 0 and m.num_decodes == 0
|
||||
else None
|
||||
)
|
||||
core_attn_out_spec, _ = fused_recurrent_kda(
|
||||
q=q_spec,
|
||||
k=k_spec,
|
||||
v=v_spec,
|
||||
raw_g=g1_spec,
|
||||
raw_beta=beta_spec,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=spec_cu_seqlens,
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=spec_out,
|
||||
)
|
||||
|
||||
# ---------- non-spec path (prefill or plain decode) ----------
|
||||
core_attn_out_non_spec = None
|
||||
if mixed_qkv_ns is not None:
|
||||
assert g1_ns is not None and beta_ns is not None
|
||||
if m.num_prefills > 0:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
# Packed prefill conv would require copying V solely to make
|
||||
# it dense for KDA. Separate calls accept the strided inputs
|
||||
# and produce dense Q/K/V without that extra traffic.
|
||||
# TODO: Use packed conv once every KDA prefill backend accepts
|
||||
# row-strided Q/K/V directly.
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
q_ns, k_ns, v_ns = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in (q_ns, k_ns, v_ns)
|
||||
)
|
||||
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
initial_state = gather_initial_states(
|
||||
recurrent_state,
|
||||
non_spec_state_indices_tensor,
|
||||
has_initial_state,
|
||||
)
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_with_fused_gate(
|
||||
q=q_ns,
|
||||
k=k_ns,
|
||||
v=v_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
)
|
||||
# Init cache
|
||||
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
|
||||
|
||||
else:
|
||||
# pure-decode non-spec batch
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
decode_conv_indices = non_spec_state_indices_tensor[
|
||||
: mixed_qkv_ns.size(0)
|
||||
]
|
||||
# Sibling beta and, for full-rank gates, output-gate views
|
||||
# remain live, so write the conv output separately.
|
||||
packed_conv_out = torch.empty(
|
||||
mixed_qkv_ns.shape,
|
||||
dtype=mixed_qkv_ns.dtype,
|
||||
device=mixed_qkv_ns.device,
|
||||
)
|
||||
mixed_qkv_ns = causal_conv1d_update(
|
||||
mixed_qkv_ns,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
out=packed_conv_out,
|
||||
)
|
||||
core_attn_out_non_spec, _ = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
state_indices=decode_conv_indices,
|
||||
)
|
||||
|
||||
# ---------- merge spec and non-spec outputs ----------
|
||||
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
|
||||
# Mixed batches require indexed placement in the original order.
|
||||
merged = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_spec.dtype,
|
||||
device=core_attn_out_spec.device,
|
||||
)
|
||||
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||
elif core_attn_out_non_spec is not None:
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
else:
|
||||
assert core_attn_out_spec is not None
|
||||
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
|
||||
@@ -1,626 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.layernorm import RMSNormGated
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
sharded_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops import (
|
||||
chunk_gated_delta_rule,
|
||||
fused_recurrent_gated_delta_rule,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.triton_utils.allocation import set_triton_allocator
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
|
||||
@PluggableLayer.register("olmo_hybrid_gated_delta_net_attention")
|
||||
class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
"""Gated DeltaNet linear attention layer for OLMo Hybrid.
|
||||
|
||||
This implements the linear attention mechanism that replaces sliding window
|
||||
attention in the hybrid architecture.
|
||||
"""
|
||||
|
||||
def get_state_shape(
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||
self.tp_size,
|
||||
self.num_k_heads,
|
||||
self.num_v_heads,
|
||||
self.head_k_dim,
|
||||
self.head_v_dim,
|
||||
self.conv_kernel_size,
|
||||
self.num_spec,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix=prefix)
|
||||
|
||||
assert getattr(config, "linear_use_gate", True), (
|
||||
"OlmoHybridGatedDeltaNet requires linear_use_gate=True"
|
||||
)
|
||||
self.num_k_heads = config.linear_num_key_heads
|
||||
self.num_v_heads = config.linear_num_value_heads
|
||||
self.head_k_dim = config.linear_key_head_dim
|
||||
self.head_v_dim = config.linear_value_head_dim
|
||||
self.conv_kernel_size = config.linear_conv_kernel_dim
|
||||
self.key_dim = self.head_k_dim * self.num_k_heads
|
||||
self.value_dim = self.head_v_dim * self.num_v_heads
|
||||
self.allow_neg_eigval = getattr(config, "linear_allow_neg_eigval", False)
|
||||
|
||||
# Fused QKVG projection: 1 matmul instead of 4
|
||||
self.in_proj_qkvg = MergedColumnParallelLinear(
|
||||
input_size=self.hidden_size,
|
||||
output_sizes=[self.key_dim, self.key_dim, self.value_dim, self.value_dim],
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.in_proj_qkvg",
|
||||
)
|
||||
|
||||
# Separate B and A projections to preserve numerical precision.
|
||||
# Fusing these into one matmul changes FP accumulation order for the
|
||||
# gating scalars, which compounds through the GDN recurrent state.
|
||||
self.b_proj = ColumnParallelLinear(
|
||||
input_size=self.hidden_size,
|
||||
output_size=self.num_v_heads,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.b_proj",
|
||||
)
|
||||
self.a_proj = ColumnParallelLinear(
|
||||
input_size=self.hidden_size,
|
||||
output_size=self.num_v_heads,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.a_proj",
|
||||
)
|
||||
|
||||
# Fused conv1d: single parameter instead of 3
|
||||
self.conv_dim = self.key_dim * 2 + self.value_dim
|
||||
self.conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_kernel_size,
|
||||
output_size=self.conv_dim,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
delattr(self.conv1d.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.conv1d.weight,
|
||||
{
|
||||
"weight_loader": _make_fused_conv1d_weight_loader(
|
||||
[self.key_dim, self.key_dim, self.value_dim],
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
self.dt_bias = nn.Parameter(
|
||||
torch.ones(self.num_v_heads // self.tp_size),
|
||||
)
|
||||
self.A_log = nn.Parameter(
|
||||
torch.empty(
|
||||
divide(self.num_v_heads, self.tp_size),
|
||||
)
|
||||
)
|
||||
|
||||
set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(0)})
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
# use eps=1e-5 to match FLA's FusedRMSNormGated
|
||||
self.o_norm = RMSNormGated(
|
||||
self.head_v_dim,
|
||||
eps=1e-5,
|
||||
group_size=None,
|
||||
norm_before_gate=True,
|
||||
device=current_platform.current_device(),
|
||||
dtype=config.torch_dtype if hasattr(config, "torch_dtype") else None,
|
||||
)
|
||||
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.value_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
# FLA triton kernels need a PyTorch-backed allocator for scratch
|
||||
# memory (required by triton >= 3.x autotuner). Set once at init.
|
||||
set_triton_allocator(current_platform.current_device())
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def rearrange_mixed_qkv(self, mixed_qkv):
|
||||
if mixed_qkv is None:
|
||||
return None, None, None
|
||||
query, key, value = torch.split(
|
||||
mixed_qkv,
|
||||
[
|
||||
self.key_dim // self.tp_size,
|
||||
self.key_dim // self.tp_size,
|
||||
self.value_dim // self.tp_size,
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
num_k_heads = self.num_k_heads // self.tp_size
|
||||
num_v_heads = self.num_v_heads // self.tp_size
|
||||
|
||||
query = rearrange(query, "l (h d) -> 1 l h d", h=num_k_heads, d=self.head_k_dim)
|
||||
key = rearrange(key, "l (h d) -> 1 l h d", h=num_k_heads, d=self.head_k_dim)
|
||||
value = rearrange(value, "l (h d) -> 1 l h d", h=num_v_heads, d=self.head_v_dim)
|
||||
|
||||
# GQA expansion if needed
|
||||
if num_v_heads > num_k_heads:
|
||||
expand_ratio = num_v_heads // num_k_heads
|
||||
query = query.unsqueeze(3).expand(-1, -1, -1, expand_ratio, -1)
|
||||
query = query.reshape(1, query.shape[1], num_v_heads, self.head_k_dim)
|
||||
key = key.unsqueeze(3).expand(-1, -1, -1, expand_ratio, -1)
|
||||
key = key.reshape(1, key.shape[1], num_v_heads, self.head_k_dim)
|
||||
|
||||
return query.contiguous(), key.contiguous(), value.contiguous()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
# NOTE: We wrap the ENTIRE linear attention forward (projections +
|
||||
# core recurrence + output norm + output projection) in a single
|
||||
# custom op, rather than just wrapping the recurrent core like
|
||||
# other GDN models (e.g. Qwen3Next) do.
|
||||
#
|
||||
# Why: torch.compile with inductor generates fused kernels for
|
||||
# matmuls and pointwise ops. These fused kernels can differ in
|
||||
# floating-point accumulation order from eager-mode cuBLAS,
|
||||
# introducing small numerical differences (~1e-7 per op). For
|
||||
# standard transformer attention this is harmless because each
|
||||
# position is computed independently. But for the GDN recurrent
|
||||
# state, these tiny input differences compound at every timestep
|
||||
# across the full sequence length, causing severe logprob
|
||||
# divergence (e.g. ~15% top-1 agreement with eager baseline).
|
||||
#
|
||||
# By making the full forward opaque to inductor, the projections
|
||||
# and output norm run with eager-mode kernels (cuBLAS, triton),
|
||||
# preserving numerical consistency. The tradeoff is reduced
|
||||
# compilation speedup (~1.5x vs ~3x), but logprob agreement
|
||||
# improves from ~15% to ~83% top-1 vs eager.
|
||||
#
|
||||
# The remaining ~17% divergence comes from inductor compiling
|
||||
# the MLP and transformer attention layers that are NOT wrapped
|
||||
# in custom ops -- their small precision differences propagate
|
||||
# as inputs to the GDN layers from outside.
|
||||
torch.ops.vllm.olmo_hybrid_gdn_full_forward(
|
||||
hidden_states,
|
||||
output,
|
||||
self.prefix,
|
||||
)
|
||||
|
||||
def _full_forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
num_tokens = hidden_states.size(0)
|
||||
|
||||
# ============================================================
|
||||
# Part 1: Input Projection (2 fused matmuls instead of 6)
|
||||
# ============================================================
|
||||
projected_qkvg, _ = self.in_proj_qkvg(hidden_states)
|
||||
conv_dim_sharded = (self.key_dim * 2 + self.value_dim) // self.tp_size
|
||||
mixed_qkv = projected_qkvg[..., :conv_dim_sharded]
|
||||
gate = projected_qkvg[..., conv_dim_sharded:]
|
||||
|
||||
b, _ = self.b_proj(hidden_states)
|
||||
a, _ = self.a_proj(hidden_states)
|
||||
|
||||
# ============================================================
|
||||
# Part 2: Core Attention
|
||||
# ============================================================
|
||||
core_attn_out = torch.zeros(
|
||||
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
self._forward_core(
|
||||
mixed_qkv=mixed_qkv,
|
||||
b=b,
|
||||
a=a,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# Part 3: Output Projection
|
||||
# ============================================================
|
||||
gate = gate.view(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim)
|
||||
core_attn_out_flat = core_attn_out.reshape(-1, core_attn_out.shape[-1])
|
||||
gate_flat = gate.reshape(-1, gate.shape[-1])
|
||||
core_attn_out_normed = self.o_norm(core_attn_out_flat, gate_flat)
|
||||
core_attn_out = core_attn_out_normed.view(
|
||||
num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim
|
||||
)
|
||||
|
||||
core_attn_out = rearrange(core_attn_out, "l h d -> l (h d)")
|
||||
output[:num_tokens], _ = self.o_proj(core_attn_out)
|
||||
|
||||
def _forward_core(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
):
|
||||
"""Core attention computation (called by custom op)."""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata is None:
|
||||
# V1 profile run
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
attn_metadata = attn_metadata.get(self.prefix) # type: ignore[assignment]
|
||||
if attn_metadata is None:
|
||||
# Profile/warmup dummy runs skip mamba-family metadata.
|
||||
return
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
has_initial_state = attn_metadata.has_initial_state
|
||||
spec_query_start_loc = attn_metadata.spec_query_start_loc
|
||||
non_spec_query_start_loc = attn_metadata.non_spec_query_start_loc
|
||||
spec_sequence_masks = attn_metadata.spec_sequence_masks
|
||||
spec_token_indx = attn_metadata.spec_token_indx
|
||||
non_spec_token_indx = attn_metadata.non_spec_token_indx
|
||||
spec_state_indices_tensor = attn_metadata.spec_state_indices_tensor
|
||||
non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor
|
||||
self_kv_cache = self.kv_cache
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||
conv_state = (
|
||||
self_kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self_kv_cache[0].transpose(-1, -2)
|
||||
)
|
||||
ssm_state = self_kv_cache[1]
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
num_accepted_tokens = attn_metadata.num_accepted_tokens
|
||||
|
||||
mixed_qkv = mixed_qkv[:num_actual_tokens]
|
||||
b = b[:num_actual_tokens]
|
||||
a = a[:num_actual_tokens]
|
||||
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
|
||||
if spec_sequence_masks is not None:
|
||||
if attn_metadata.num_prefills == 0 and attn_metadata.num_decodes == 0:
|
||||
mixed_qkv_spec = mixed_qkv
|
||||
mixed_qkv_non_spec = None
|
||||
else:
|
||||
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
|
||||
mixed_qkv_non_spec = mixed_qkv.index_select(0, non_spec_token_indx)
|
||||
else:
|
||||
mixed_qkv_spec = None
|
||||
mixed_qkv_non_spec = mixed_qkv
|
||||
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_query_start_loc is not None
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert num_accepted_tokens is not None
|
||||
mixed_qkv_spec = causal_conv1d_update(
|
||||
mixed_qkv_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
None, # no bias
|
||||
self.activation,
|
||||
conv_state_indices=spec_state_indices_tensor[:, 0][
|
||||
: attn_metadata.num_spec_decodes
|
||||
],
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=spec_state_indices_tensor.size(-1),
|
||||
validate_data=False,
|
||||
)
|
||||
|
||||
if attn_metadata.num_prefills > 0:
|
||||
assert mixed_qkv_non_spec is not None
|
||||
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
|
||||
mixed_qkv_non_spec = causal_conv1d_fn(
|
||||
mixed_qkv_non_spec_T,
|
||||
conv_weights,
|
||||
None,
|
||||
activation=self.activation,
|
||||
conv_states=conv_state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=attn_metadata,
|
||||
).transpose(0, 1)
|
||||
elif attn_metadata.num_decodes > 0:
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
mixed_qkv_non_spec = causal_conv1d_update(
|
||||
mixed_qkv_non_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
None,
|
||||
self.activation,
|
||||
conv_state_indices=non_spec_state_indices_tensor[
|
||||
: attn_metadata.num_decodes
|
||||
],
|
||||
validate_data=True,
|
||||
)
|
||||
else:
|
||||
mixed_qkv_non_spec = None
|
||||
|
||||
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
|
||||
query_non_spec, key_non_spec, value_non_spec = self.rearrange_mixed_qkv(
|
||||
mixed_qkv_non_spec
|
||||
)
|
||||
|
||||
g, beta = fused_olmo_hybrid_gdn_gating(
|
||||
self.A_log, a, b, self.dt_bias, self.allow_neg_eigval
|
||||
)
|
||||
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_token_indx is not None
|
||||
assert non_spec_token_indx is not None
|
||||
if attn_metadata.num_prefills == 0 and attn_metadata.num_decodes == 0:
|
||||
g_spec = g
|
||||
beta_spec = beta
|
||||
g_non_spec = None
|
||||
beta_non_spec = None
|
||||
else:
|
||||
g_spec = g.index_select(1, spec_token_indx)
|
||||
beta_spec = beta.index_select(1, spec_token_indx)
|
||||
g_non_spec = g.index_select(1, non_spec_token_indx)
|
||||
beta_non_spec = beta.index_select(1, non_spec_token_indx)
|
||||
else:
|
||||
g_spec = None
|
||||
beta_spec = None
|
||||
g_non_spec = g
|
||||
beta_non_spec = beta
|
||||
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_query_start_loc is not None
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert num_accepted_tokens is not None
|
||||
core_attn_out_spec, last_recurrent_state = fused_recurrent_gated_delta_rule(
|
||||
q=query_spec,
|
||||
k=key_spec,
|
||||
v=value_spec,
|
||||
g=g_spec,
|
||||
beta=beta_spec,
|
||||
initial_state=ssm_state,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=spec_query_start_loc[: attn_metadata.num_spec_decodes + 1],
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
else:
|
||||
core_attn_out_spec, last_recurrent_state = None, None
|
||||
|
||||
if attn_metadata.num_prefills > 0:
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
assert non_spec_query_start_loc is not None
|
||||
initial_state = ssm_state[non_spec_state_indices_tensor].contiguous()
|
||||
initial_state[~has_initial_state, ...] = 0
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
ssm_state[non_spec_state_indices_tensor] = last_recurrent_state.to(
|
||||
ssm_state.dtype
|
||||
)
|
||||
elif attn_metadata.num_decodes > 0:
|
||||
assert non_spec_query_start_loc is not None
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
core_attn_out_non_spec, last_recurrent_state = (
|
||||
fused_recurrent_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=ssm_state,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=non_spec_query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
],
|
||||
ssm_state_indices=non_spec_state_indices_tensor,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
)
|
||||
else:
|
||||
core_attn_out_non_spec, last_recurrent_state = None, None
|
||||
|
||||
if spec_sequence_masks is not None and core_attn_out_non_spec is not None:
|
||||
merged_out = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_non_spec.dtype,
|
||||
device=core_attn_out_non_spec.device,
|
||||
)
|
||||
merged_out.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged_out.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[:num_actual_tokens] = merged_out.squeeze(0)
|
||||
elif spec_sequence_masks is not None:
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0)
|
||||
else:
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0)
|
||||
|
||||
|
||||
def _make_fused_conv1d_weight_loader(dims, tp_size, tp_rank):
|
||||
"""Weight loader for loading separate HF conv weights into a fused conv1d.
|
||||
|
||||
dims: list of original (un-sharded) dims per section,
|
||||
e.g. [key_dim, key_dim, value_dim]
|
||||
"""
|
||||
sharded_dims = [d // tp_size for d in dims]
|
||||
|
||||
def weight_loader(param, loaded_weight, loaded_shard_id=None):
|
||||
if loaded_weight.dim() == 2:
|
||||
loaded_weight = loaded_weight.unsqueeze(1)
|
||||
dim = dims[loaded_shard_id]
|
||||
shard_size = dim // tp_size
|
||||
tp_start = tp_rank * shard_size
|
||||
sharded_weight = loaded_weight[tp_start : tp_start + shard_size]
|
||||
offset = sum(sharded_dims[:loaded_shard_id])
|
||||
param.data[offset : offset + shard_size].copy_(sharded_weight)
|
||||
|
||||
return weight_loader
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def olmo_hybrid_gdn_full_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Full linear attention forward wrapped as a custom op.
|
||||
|
||||
Prevents inductor from compiling the projections around the GDN core,
|
||||
which would introduce numerical divergence that compounds through
|
||||
the recurrent state.
|
||||
"""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._full_forward(
|
||||
hidden_states=hidden_states,
|
||||
output=output,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="olmo_hybrid_gdn_full_forward",
|
||||
op_func=olmo_hybrid_gdn_full_forward,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fused_olmo_hybrid_gdn_gating_kernel(
|
||||
g,
|
||||
beta_output,
|
||||
A_log,
|
||||
a,
|
||||
b,
|
||||
dt_bias,
|
||||
seq_len,
|
||||
allow_neg_eigval: tl.constexpr,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
beta: tl.constexpr,
|
||||
threshold: tl.constexpr,
|
||||
BLK_HEADS: tl.constexpr,
|
||||
):
|
||||
i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS)
|
||||
off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off
|
||||
mask = head_off < NUM_HEADS
|
||||
blk_A_log = tl.load(A_log + head_off, mask=mask)
|
||||
blk_a = tl.load(a + off, mask=mask)
|
||||
blk_b = tl.load(b + off, mask=mask)
|
||||
blk_bias = tl.load(dt_bias + head_off, mask=mask)
|
||||
|
||||
# g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
|
||||
x = blk_a.to(tl.float32) + blk_bias.to(tl.float32)
|
||||
softplus_x = tl.where(
|
||||
beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x
|
||||
)
|
||||
blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x
|
||||
tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask)
|
||||
|
||||
# beta = self.b_proj(hidden_states).sigmoid()
|
||||
# if self.allow_neg_eigval: beta = beta * 2.0
|
||||
blk_beta_output = tl.sigmoid(blk_b.to(tl.float32))
|
||||
if allow_neg_eigval:
|
||||
blk_beta_output = blk_beta_output * 2.0
|
||||
tl.store(
|
||||
beta_output + off, blk_beta_output.to(beta_output.dtype.element_ty), mask=mask
|
||||
)
|
||||
|
||||
|
||||
def fused_olmo_hybrid_gdn_gating(
|
||||
A_log: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
allow_neg_eigval: bool = False,
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch, num_heads = a.shape
|
||||
seq_len = 1
|
||||
grid = (batch, seq_len, triton.cdiv(num_heads, 8))
|
||||
g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device)
|
||||
beta_output = torch.empty(1, batch, num_heads, dtype=torch.float32, device=b.device)
|
||||
fused_olmo_hybrid_gdn_gating_kernel[grid](
|
||||
g,
|
||||
beta_output,
|
||||
A_log,
|
||||
a,
|
||||
b,
|
||||
dt_bias,
|
||||
seq_len,
|
||||
allow_neg_eigval,
|
||||
num_heads,
|
||||
beta,
|
||||
threshold,
|
||||
8,
|
||||
num_warps=1,
|
||||
)
|
||||
return g, beta_output
|
||||
@@ -54,7 +54,12 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
|
||||
try:
|
||||
# inc/ loader was removed in the SM86 mini-vLLM cut.
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
except ImportError:
|
||||
INCConfig = type("AbsentINCConfig", (), {})
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
sharded_weight_loader,
|
||||
)
|
||||
|
||||
@@ -1,247 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import Int32, cute
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
from .kernel_h import h_cutedsl
|
||||
from .kernel_kkt_inv_uw import kkt_inv_uw_cutedsl
|
||||
from .kernel_o import o_cutedsl
|
||||
|
||||
|
||||
class PrepMetaKernel:
|
||||
def __init__(self, BT: int) -> None:
|
||||
self.BT = BT
|
||||
self.num_warps = 8
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
chunk_offsets: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
block = (self.num_warps * 32, 1, 1)
|
||||
self.kernel(
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
chunk_offsets,
|
||||
).launch(grid=(1, 1, 1), block=block, stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
chunk_offsets: cute.Tensor,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
|
||||
num_seqs = cu_seqlens.shape[0] - 1
|
||||
num_warps = self.num_warps
|
||||
tb_size = num_warps * 32
|
||||
|
||||
if tid == 0:
|
||||
chunk_offsets[0] = 0
|
||||
|
||||
coarsen = cute.ceil_div(num_seqs, tb_size)
|
||||
seq_start = tid * coarsen
|
||||
num_iters = cutlass.min(seq_start + coarsen, num_seqs) - seq_start
|
||||
|
||||
# First pass: compute this thread's total chunk count.
|
||||
thread_sum = Int32(0)
|
||||
for i in range(num_iters):
|
||||
seq_id = seq_start + i
|
||||
seqlen = cu_seqlens[seq_id + 1] - cu_seqlens[seq_id]
|
||||
thread_sum += cute.ceil_div(seqlen, self.BT)
|
||||
|
||||
# warp parallel scan
|
||||
cu_num_chunks = thread_sum
|
||||
for i in cutlass.range_constexpr(5):
|
||||
offset = cutlass.const_expr(1 << i)
|
||||
lower = cute.arch.shuffle_sync_up(
|
||||
cu_num_chunks, offset=offset, mask_and_clamp=0
|
||||
)
|
||||
if lane_id >= offset:
|
||||
cu_num_chunks += lower
|
||||
|
||||
# cross-warp cumsum (CTA-wide)
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
warp_num_chunks = smem.allocate_array(Int32, num_warps)
|
||||
if lane_id == 31:
|
||||
warp_num_chunks[warp_id] = cu_num_chunks
|
||||
cute.arch.sync_threads()
|
||||
|
||||
for i in cutlass.range_constexpr(1, num_warps):
|
||||
if warp_id >= i:
|
||||
cu_num_chunks += warp_num_chunks[i - 1]
|
||||
|
||||
chunk_start = cu_num_chunks - thread_sum
|
||||
|
||||
# Second pass: recompute per-sequence chunk counts and write results.
|
||||
for i in range(num_iters):
|
||||
seq_id = seq_start + i
|
||||
seqlen = cu_seqlens[seq_id + 1] - cu_seqlens[seq_id]
|
||||
num_chunks = cute.ceil_div(seqlen, self.BT)
|
||||
chunk_end = chunk_start + num_chunks
|
||||
chunk_offsets[seq_id + 1] = chunk_end
|
||||
|
||||
for chunk_id in range(num_chunks):
|
||||
chunk_indices[chunk_start + chunk_id, 0] = seq_id
|
||||
chunk_indices[chunk_start + chunk_id, 1] = chunk_id
|
||||
|
||||
chunk_start = chunk_end
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(BT: int):
|
||||
cu_entries = cute.sym_int()
|
||||
upper_bound_chunks = cute.sym_int()
|
||||
|
||||
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
|
||||
chunk_indices = make_fake_tensor(Int32, (upper_bound_chunks, 2), divisibility=2)
|
||||
chunk_offsets = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
|
||||
|
||||
kernel = PrepMetaKernel(BT)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
chunk_offsets,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def _upper_bound_chunks(num_seqs: int, total_tokens: int, chunk_size: int) -> int:
|
||||
return (num_seqs - 1) + triton.cdiv(total_tokens - (num_seqs - 1), chunk_size)
|
||||
|
||||
|
||||
def prepare_metadata_cutedsl(
|
||||
cu_seqlens: torch.Tensor,
|
||||
total_tokens: int,
|
||||
chunk_size: int = 64,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_seqs = cu_seqlens.numel() - 1
|
||||
upper_bound_chunks = _upper_bound_chunks(num_seqs, total_tokens, chunk_size)
|
||||
chunk_offsets = cu_seqlens.new_empty(num_seqs + 1, dtype=torch.int32)
|
||||
chunk_indices = cu_seqlens.new_empty((upper_bound_chunks, 2), dtype=torch.int32)
|
||||
|
||||
PrepMetaKernel.compile(chunk_size)(cu_seqlens, chunk_indices, chunk_offsets)
|
||||
return chunk_indices, chunk_offsets
|
||||
|
||||
|
||||
def chunk_gated_delta_rule_cutedsl(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
chunk_indices: torch.Tensor,
|
||||
chunk_offsets: torch.Tensor,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the GDN chunk CuteDSL prefill kernels.
|
||||
|
||||
Args:
|
||||
q: Query tensor with shape ``[1, T, H, K]``.
|
||||
k: Key tensor with shape ``[1, T, H, K]``.
|
||||
v: Value tensor with shape ``[1, T, Hv, V]``.
|
||||
g: Log-space decay tensor with shape ``[1, T, Hv]``.
|
||||
beta: Delta-rule beta tensor with shape ``[1, T, Hv]``.
|
||||
initial_state: Recurrent state with shape ``[N, Hv, V, K]``.
|
||||
cu_seqlens: Cumulative sequence lengths with shape ``[N + 1]``.
|
||||
chunk_indices: Chunk index metadata with shape ``[NT, 2]``.
|
||||
chunk_offsets: Cumulative chunk offsets with shape ``[N + 1]``.
|
||||
core_attn_out: Optional output buffer with shape ``[T, Hv, V]``.
|
||||
|
||||
Returns:
|
||||
A tuple ``(output, final_state)`` where ``output`` has shape
|
||||
``[1, T, Hv, V]`` and ``final_state`` has shape ``[N, Hv, V, K]``.
|
||||
When ``core_attn_out`` is provided, ``output`` is an unsqueezed view of
|
||||
that buffer.
|
||||
|
||||
"""
|
||||
q = q.squeeze(0)
|
||||
k = k.squeeze(0)
|
||||
v = v.squeeze(0)
|
||||
g = g.squeeze(0)
|
||||
beta = beta.squeeze(0)
|
||||
|
||||
_, _, K_dim = k.shape
|
||||
_, num_v_heads, V_dim = v.shape
|
||||
chunk_size = 64
|
||||
upper_bound_chunks = chunk_indices.shape[0]
|
||||
pad_t = upper_bound_chunks * chunk_size
|
||||
total_chunks_ptr = chunk_offsets[-1:]
|
||||
|
||||
g_cu = torch.empty_like(g, dtype=torch.float32)
|
||||
u = q.new_empty(pad_t, num_v_heads, V_dim)
|
||||
w = q.new_empty(pad_t, num_v_heads, K_dim)
|
||||
|
||||
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
|
||||
kkt_inv_uw_cutedsl(
|
||||
k,
|
||||
v,
|
||||
u,
|
||||
w,
|
||||
g,
|
||||
beta,
|
||||
g_cu,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks_ptr,
|
||||
num_sms=num_sms,
|
||||
)
|
||||
|
||||
h = k.new_empty(upper_bound_chunks, num_v_heads, V_dim, K_dim)
|
||||
v_new = q.new_empty(pad_t, num_v_heads, V_dim)
|
||||
final_state = torch.empty_like(initial_state)
|
||||
h_cutedsl(
|
||||
k,
|
||||
u,
|
||||
w,
|
||||
v_new,
|
||||
g_cu,
|
||||
h,
|
||||
initial_state,
|
||||
final_state,
|
||||
cu_seqlens,
|
||||
chunk_offsets,
|
||||
)
|
||||
|
||||
output = core_attn_out if core_attn_out is not None else torch.empty_like(v)
|
||||
scale = K_dim**-0.5
|
||||
o_cutedsl(
|
||||
q,
|
||||
k,
|
||||
v_new,
|
||||
h,
|
||||
g_cu,
|
||||
output,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks_ptr,
|
||||
scale,
|
||||
num_sms=num_sms,
|
||||
)
|
||||
return output.unsqueeze(0), final_state
|
||||
|
||||
|
||||
__all__ = [
|
||||
"chunk_gated_delta_rule_cutedsl",
|
||||
"prepare_metadata_cutedsl",
|
||||
]
|
||||
@@ -1,751 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
|
||||
from cutlass.cute.nvgpu import cpasync, warp
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import (
|
||||
EVICT_FIRST,
|
||||
_tcgen05,
|
||||
cvt,
|
||||
fence_before_tma_store,
|
||||
simple_tma_copy,
|
||||
)
|
||||
|
||||
|
||||
class Sm100ChunkHKernel:
|
||||
"""For each sequence, compute the chunk recurrent update.
|
||||
|
||||
The input V tile is the U output from the KKT/UW kernel. For each chunk:
|
||||
V_new = U - W @ H.T
|
||||
(we actually do V_new.T = U.T - H @ W.T instead)
|
||||
|
||||
H_scaled = H * exp(g_last)
|
||||
V_scaled = V_new * exp(g_last - g)
|
||||
H_new = H_scaled + V_scaled.T @ K
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
H: int,
|
||||
Hv: int,
|
||||
K_dim: int,
|
||||
V_dim: int,
|
||||
h_dtype: cutlass.Numeric = Float32,
|
||||
BT: int = 64,
|
||||
num_stages: int = 2,
|
||||
) -> None:
|
||||
assert Hv % H == 0
|
||||
assert K_dim == V_dim == 128
|
||||
assert BT == 64
|
||||
self.H = H
|
||||
self.Hv = Hv
|
||||
self.K_dim = K_dim
|
||||
self.V_dim = V_dim
|
||||
self.h_dtype = h_dtype
|
||||
self.BT = BT
|
||||
self.num_stages = num_stages
|
||||
self.num_warps = 10
|
||||
|
||||
@cute.jit
|
||||
def _make_bf16_tma_args(
|
||||
self,
|
||||
tensor: cute.Tensor,
|
||||
dim: cutlass.Constexpr[int],
|
||||
op: cpasync.TmaCopyOp,
|
||||
stages: cutlass.Constexpr[int],
|
||||
):
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
slayout = cute.make_layout(
|
||||
(self.BT, 1, (64, dim // 64), stages),
|
||||
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
return cpasync.make_tiled_tma_atom(
|
||||
op,
|
||||
cute.logical_divide(tensor, (None, None, 64)),
|
||||
slayout,
|
||||
cta_tiler=(self.BT, 1, dim),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _make_h_tma_args(self, tensor: cute.Tensor, op: cpasync.TmaCopyOp):
|
||||
# number of elements to fill 128B
|
||||
num_elems = 128 // (tensor.element_type.width // 8)
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
slayout = cute.make_layout(
|
||||
(1, 1, self.V_dim, (num_elems, self.K_dim // num_elems)),
|
||||
stride=(0, 0, num_elems, (1, self.V_dim * num_elems)),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
return cpasync.make_tiled_tma_atom(
|
||||
op,
|
||||
cute.logical_divide(tensor, (None, None, None, num_elems)),
|
||||
slayout,
|
||||
cta_tiler=(1, 1, self.V_dim, self.K_dim),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
K: cute.Tensor,
|
||||
V: cute.Tensor,
|
||||
W: cute.Tensor,
|
||||
V_new: cute.Tensor,
|
||||
g_cu: cute.Tensor,
|
||||
h: cute.Tensor,
|
||||
h0: cute.Tensor,
|
||||
ht: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_offsets: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
|
||||
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
|
||||
|
||||
K_tma = self._make_bf16_tma_args(K, self.K_dim, tma_g2s, self.num_stages)
|
||||
V_tma = self._make_bf16_tma_args(V, self.V_dim, tma_g2s, self.num_stages)
|
||||
W_tma = self._make_bf16_tma_args(W, self.K_dim, tma_g2s, self.num_stages)
|
||||
V_new_tma = self._make_bf16_tma_args(V_new, self.V_dim, tma_s2g, 1)
|
||||
H0_tma = self._make_h_tma_args(h0, tma_g2s)
|
||||
HT_tma = self._make_h_tma_args(ht, tma_s2g)
|
||||
H_tma = self._make_h_tma_args(h, tma_s2g)
|
||||
|
||||
grid = (self.Hv, h0.shape[0], 1)
|
||||
block = (self.num_warps * 32, 1, 1)
|
||||
self.kernel(
|
||||
K_tma,
|
||||
V_tma,
|
||||
W_tma,
|
||||
V_new_tma,
|
||||
H0_tma,
|
||||
HT_tma,
|
||||
H_tma,
|
||||
g_cu,
|
||||
cu_seqlens,
|
||||
chunk_offsets,
|
||||
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
K_tma: cpasync.TmaInfo,
|
||||
V_tma: cpasync.TmaInfo,
|
||||
W_tma: cpasync.TmaInfo,
|
||||
V_new_tma: cpasync.TmaInfo,
|
||||
H0_tma: cpasync.TmaInfo,
|
||||
HT_tma: cpasync.TmaInfo,
|
||||
H_tma: cpasync.TmaInfo,
|
||||
g_cu: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_offsets: cute.Tensor,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
head_id, seq_id, _ = cute.arch.block_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
|
||||
BT = self.BT
|
||||
V_dim = self.V_dim
|
||||
K_dim = self.K_dim
|
||||
num_stages = self.num_stages
|
||||
is_f32 = self.h_dtype == Float32
|
||||
|
||||
def allocate_tensor(smem, dtype, layout):
|
||||
return smem.allocate_tensor(
|
||||
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
|
||||
)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
|
||||
# remove size=1 modes
|
||||
sW = allocate_tensor(smem, BFloat16, W_tma.smem_layout)[None, 0, None, None]
|
||||
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
|
||||
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
|
||||
sH0 = allocate_tensor(smem, self.h_dtype, H0_tma.smem_layout)[0, 0, None, None]
|
||||
sH = allocate_tensor(smem, BFloat16, H_tma.smem_layout)[0, 0, None, None]
|
||||
sV_new = allocate_tensor(smem, BFloat16, V_new_tma.smem_layout)[
|
||||
None, 0, None, 0
|
||||
]
|
||||
|
||||
s_v_scale = smem.allocate_array(Float32, BT)
|
||||
tma_mbar = smem.allocate_array(Int64, num_stages)
|
||||
wh_in_mbar = smem.allocate_array(Int64, num_stages)
|
||||
wh_done_mbar = smem.allocate_array(Int64, num_stages)
|
||||
vk_in_mbar = smem.allocate_array(Int64, num_stages)
|
||||
vk_done_mbar = smem.allocate_array(Int64, num_stages)
|
||||
h0_mbar = smem.allocate_array(Int64, 1)
|
||||
taddr = smem.allocate(Int32, 4)
|
||||
|
||||
wh_tmem = 0
|
||||
vk_tmem = wh_tmem + BT
|
||||
h_tmem_base = vk_tmem + K_dim
|
||||
v_tmem_base = h_tmem_base + K_dim // 2
|
||||
|
||||
if warp_id == 0:
|
||||
with cute.arch.elect_one():
|
||||
for i in cutlass.range_constexpr(num_stages):
|
||||
cute.arch.mbarrier_init(tma_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(wh_in_mbar + i, 256)
|
||||
cute.arch.mbarrier_init(wh_done_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(vk_in_mbar + i, 256)
|
||||
cute.arch.mbarrier_init(vk_done_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(h0_mbar, 1)
|
||||
cute.arch.mbarrier_init_fence()
|
||||
elif warp_id == 1:
|
||||
cpasync.prefetch_descriptor(H0_tma.atom)
|
||||
cpasync.prefetch_descriptor(W_tma.atom)
|
||||
cpasync.prefetch_descriptor(V_tma.atom)
|
||||
cpasync.prefetch_descriptor(K_tma.atom)
|
||||
cpasync.prefetch_descriptor(HT_tma.atom)
|
||||
cpasync.prefetch_descriptor(H_tma.atom)
|
||||
cpasync.prefetch_descriptor(V_new_tma.atom)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
bos = cu_seqlens[seq_id]
|
||||
eos = cu_seqlens[seq_id + 1]
|
||||
seqlen = eos - bos
|
||||
num_chunks = cute.ceil_div(seqlen, BT)
|
||||
|
||||
if warp_id == 9:
|
||||
# TMA warp
|
||||
stage_id = 0
|
||||
parity = 1
|
||||
|
||||
k_head_id = head_id // (self.Hv // self.H)
|
||||
chunk_offset = chunk_offsets[seq_id]
|
||||
|
||||
# load H0
|
||||
with cute.arch.elect_one():
|
||||
H0_size = V_dim * K_dim * self.h_dtype.width // 8
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(h0_mbar, H0_size)
|
||||
simple_tma_copy(
|
||||
H0_tma.atom,
|
||||
H0_tma.tma_tensor[seq_id, head_id, None, None],
|
||||
sH0,
|
||||
h0_mbar,
|
||||
)
|
||||
|
||||
# shape: ((BT, num_BT_tiles), (64, 2))
|
||||
gW_tiles = cute.logical_divide(
|
||||
W_tma.tma_tensor[None, head_id, None], (BT, None)
|
||||
)
|
||||
gV_tiles = cute.logical_divide(
|
||||
V_tma.tma_tensor[None, head_id, None], (BT, None)
|
||||
)
|
||||
gK_tiles = cute.logical_divide(
|
||||
cute.domain_offset((bos, 0), K_tma.tma_tensor[None, k_head_id, None]),
|
||||
(BT, None),
|
||||
)
|
||||
|
||||
for chunk_id in range(num_chunks):
|
||||
mbar = tma_mbar + stage_id
|
||||
gW = gW_tiles[(None, chunk_offset + chunk_id), None]
|
||||
gV = gV_tiles[(None, chunk_offset + chunk_id), None]
|
||||
gK = gK_tiles[(None, chunk_id), None]
|
||||
|
||||
# wait for MMA to release the buffer
|
||||
cute.arch.mbarrier_wait(vk_done_mbar + stage_id, parity)
|
||||
|
||||
# load W, V (i.e. U), and K
|
||||
with cute.arch.elect_one():
|
||||
STAGE_SIZE = BT * (K_dim + V_dim + K_dim) * 2
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
|
||||
simple_tma_copy(
|
||||
W_tma.atom, gW, sW[None, None, stage_id], mbar, EVICT_FIRST
|
||||
)
|
||||
simple_tma_copy(
|
||||
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
|
||||
)
|
||||
simple_tma_copy(K_tma.atom, gK, sK[None, None, stage_id], mbar)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id == 8:
|
||||
# MMA warp
|
||||
_tcgen05.alloc(taddr)
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
wh_idesc = _tcgen05.make_bf16_idesc(V_dim, BT, negate_A=True)
|
||||
vk_idesc = _tcgen05.make_bf16_idesc(V_dim, K_dim, transpose_B=True)
|
||||
|
||||
# LBO=BT*128 is ignored for K-major
|
||||
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
|
||||
|
||||
# when using BF16 state, H is read from smem for the 1st iteration
|
||||
# variable names in this conditional branch can't be the same as those
|
||||
# in the mainloop below due to CuteDSL restrictions.
|
||||
if cutlass.const_expr(not is_f32):
|
||||
##### 1st MMA: V_new.T = V.T - H @ W.T #####
|
||||
Haddr0 = sH0[None, None].iterator.toint()
|
||||
Waddr0 = sW[None, None, stage_id].iterator.toint()
|
||||
hdesc0_base = sdesc_template | (Haddr0 >> 4)
|
||||
wdesc0_base = sdesc_template | (Waddr0 >> 4)
|
||||
|
||||
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
|
||||
cute.arch.mbarrier_wait(wh_in_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(K_dim // 64):
|
||||
for j in cutlass.range_constexpr(64 // 16):
|
||||
hdesc0 = hdesc0_base | ((i * V_dim * 128 + j * 32) >> 4)
|
||||
wdesc0 = wdesc0_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
_tcgen05.mma_f16(wh_tmem, hdesc0, wdesc0, wh_idesc, True)
|
||||
_tcgen05.commit(wh_done_mbar + stage_id)
|
||||
|
||||
##### 2nd MMA: H_new = H + V_new.T @ K #####
|
||||
Kaddr0 = sK[None, None, stage_id].iterator.toint()
|
||||
kdesc0_base = sdesc_template | (Kaddr0 >> 4)
|
||||
|
||||
cute.arch.mbarrier_wait(vk_in_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for k in cutlass.range_constexpr(BT // 16):
|
||||
vtmem0 = v_tmem_base + k * 8
|
||||
kdesc0 = kdesc0_base | ((k * 16 * 128) >> 4)
|
||||
_tcgen05.mma_ts_f16(vk_tmem, vtmem0, kdesc0, vk_idesc, True)
|
||||
_tcgen05.commit(vk_done_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
num_iters = num_chunks - int(not is_f32)
|
||||
for _ in range(num_iters):
|
||||
##### 1st MMA: V_new.T = V.T - H @ W.T #####
|
||||
Waddr = sW[None, None, stage_id].iterator.toint()
|
||||
wdesc_base = sdesc_template | (Waddr >> 4)
|
||||
|
||||
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
|
||||
cute.arch.mbarrier_wait(wh_in_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(K_dim // 64):
|
||||
for j in cutlass.range_constexpr(64 // 16):
|
||||
htmem = h_tmem_base + i * 32 + j * 8
|
||||
wdesc = wdesc_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
_tcgen05.mma_ts_f16(wh_tmem, htmem, wdesc, wh_idesc, True)
|
||||
_tcgen05.commit(wh_done_mbar + stage_id)
|
||||
|
||||
##### 2nd MMA: H_new = H + V_new.T @ K #####
|
||||
Kaddr = sK[None, None, stage_id].iterator.toint()
|
||||
kdesc_base = sdesc_template | (Kaddr >> 4)
|
||||
|
||||
cute.arch.mbarrier_wait(vk_in_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for k in cutlass.range_constexpr(BT // 16):
|
||||
vtmem = v_tmem_base + k * 8
|
||||
kdesc = kdesc_base | ((k * 16 * 128) >> 4)
|
||||
_tcgen05.mma_ts_f16(vk_tmem, vtmem, kdesc, vk_idesc, True)
|
||||
_tcgen05.commit(vk_done_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id >= 4:
|
||||
# H warps
|
||||
tid_ = tid % 128
|
||||
warp_id_ = warp_id % 4
|
||||
chunk_offset = chunk_offsets[seq_id]
|
||||
|
||||
stage_id = 0
|
||||
vk_stage_id = 0
|
||||
vk_parity = 0
|
||||
|
||||
op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_16B = cute.make_copy_atom(op, Float32, num_bits_per_copy=128)
|
||||
|
||||
##### chunk_id = 0 #####
|
||||
if True:
|
||||
chunk_id = 0
|
||||
end_t = min(bos + (chunk_id + 1) * BT, eos)
|
||||
last_idx = end_t - 1
|
||||
h_scale = cute.math.exp(g_cu[last_idx, head_id], fastmath=True)
|
||||
|
||||
# for 1st chunk, wait for H0 transfer from gmem
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(h0_mbar, 0)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
|
||||
# when H0 is FP32, we need to pack it to BF16
|
||||
# also store to smem for TMA store later.
|
||||
if cutlass.const_expr(is_f32):
|
||||
for i in cutlass.range_constexpr(K_dim // 32):
|
||||
# H0 smem layout: (V_dim, (32, K_dim/32))
|
||||
h_f32 = cute.make_rmem_tensor(32, Float32)
|
||||
cute.copy(cp_16B, sH0[tid_, (None, i)], h_f32)
|
||||
|
||||
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
|
||||
h_bf16.store(h_f32.load().to(BFloat16))
|
||||
_tcgen05.st(
|
||||
warp_id_ * 32, h_tmem_base + i * 16, "32x32b", 16, h_bf16
|
||||
)
|
||||
|
||||
# H smem layout: (V_dim, (64, K_dim/64))
|
||||
dst = cute.local_tile(sH[tid_, None], (32,), (i,))
|
||||
cute.copy(cp_16B, h_bf16, dst)
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
|
||||
|
||||
# scale H for 2nd MMA
|
||||
for i in cutlass.range_constexpr(K_dim // 32):
|
||||
h_f32 = cute.make_rmem_tensor(32, Float32)
|
||||
|
||||
if cutlass.const_expr(is_f32):
|
||||
cute.copy(cp_16B, sH0[tid_, (None, i)], h_f32)
|
||||
|
||||
else:
|
||||
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
|
||||
sH_src = cute.local_tile(sH0[tid_, None], (32,), (i,))
|
||||
cute.copy(cp_16B, sH_src, h_bf16)
|
||||
h_f32.store(
|
||||
cvt.bf16x2_to_fp32x2(
|
||||
cute.recast_tensor(h_bf16, Uint32)
|
||||
).load()
|
||||
)
|
||||
|
||||
for j in cutlass.range(32, vectorize=True):
|
||||
h_f32[j] *= h_scale
|
||||
_tcgen05.st(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32, h_f32)
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
|
||||
|
||||
# for BF16 H0, we issue TMA store from H0 smem
|
||||
# for FP32 H0, we issue TMA store from H smem (after packing)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id_ == 3:
|
||||
h_src = sH if cutlass.const_expr(is_f32) else sH0
|
||||
h_dst = H_tma.tma_tensor[
|
||||
chunk_offset + chunk_id, head_id, None, None
|
||||
]
|
||||
simple_tma_copy(H_tma.atom, h_src, h_dst)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
|
||||
# When H0 is BF16, and there is only 1 chunk, storing
|
||||
# the final state to sH0 can race before this store
|
||||
# has finished. hence, we need to wait here.
|
||||
if cutlass.const_expr(not is_f32):
|
||||
cute.arch.cp_async_bulk_wait_group(0, read=True)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
|
||||
##### subsequent chunks #####
|
||||
for chunk_id in range(1, num_chunks):
|
||||
end_t = min(bos + (chunk_id + 1) * BT, eos)
|
||||
last_idx = end_t - 1
|
||||
h_scale = cute.math.exp(g_cu[last_idx, head_id], fastmath=True)
|
||||
|
||||
# wait for H from previous vk MMA
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(vk_done_mbar + vk_stage_id, vk_parity)
|
||||
vk_stage_id = (vk_stage_id + 1) % num_stages
|
||||
if vk_stage_id == 0:
|
||||
vk_parity ^= 1
|
||||
elif warp_id_ == 3:
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_wait_group(0, read=True)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
# load FP32 H from tmem, convert to BF16, store to tmem for 1st MMA,
|
||||
# store to smem for TMA store later.
|
||||
for i in cutlass.range_constexpr(K_dim // 32):
|
||||
h_f32 = _tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32)
|
||||
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
|
||||
h_bf16.store(h_f32.to(BFloat16))
|
||||
_tcgen05.st(
|
||||
warp_id_ * 32, h_tmem_base + i * 16, "32x32b", 16, h_bf16
|
||||
)
|
||||
|
||||
# H smem layout: (V_dim, (64, K_dim/64))
|
||||
dst = cute.local_tile(sH[tid_, None], (32,), (i,))
|
||||
cute.copy(cp_16B, h_bf16, dst)
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
|
||||
|
||||
# scale H for 2nd MMA
|
||||
for i in cutlass.range_constexpr(K_dim // 32):
|
||||
h_f32 = cute.make_rmem_tensor(32, Float32)
|
||||
h_f32.store(
|
||||
_tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32)
|
||||
)
|
||||
for j in cutlass.range(32, vectorize=True):
|
||||
h_f32[j] *= h_scale
|
||||
_tcgen05.st(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32, h_f32)
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
|
||||
|
||||
# issue TMA store for O kernel
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id_ == 3:
|
||||
h_dst = H_tma.tma_tensor[
|
||||
chunk_offset + chunk_id, head_id, None, None
|
||||
]
|
||||
simple_tma_copy(H_tma.atom, sH, h_dst)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
|
||||
# handle final state. reuse H0 smem.
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(vk_done_mbar + vk_stage_id, vk_parity)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(K_dim // 32):
|
||||
h_f32 = cute.make_rmem_tensor(32, Float32)
|
||||
h_f32.store(_tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32))
|
||||
|
||||
if cutlass.const_expr(is_f32):
|
||||
cute.copy(cp_16B, h_f32, sH0[tid_, (None, i)])
|
||||
|
||||
else:
|
||||
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
|
||||
h_bf16.store(h_f32.load().to(BFloat16))
|
||||
sH0_dst = cute.local_tile(sH0[tid_, None], (32,), (i,))
|
||||
cute.copy(cp_16B, h_bf16, sH0_dst)
|
||||
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
|
||||
if warp_id_ == 0:
|
||||
ht_dst = HT_tma.tma_tensor[seq_id, head_id, None, None]
|
||||
simple_tma_copy(HT_tma.atom, sH0, ht_dst)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
if warp_id_ == 1:
|
||||
_tcgen05.dealloc()
|
||||
|
||||
else:
|
||||
# V warps
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
chunk_offset = chunk_offsets[seq_id]
|
||||
|
||||
ldsm_trans_op = warp.LdMatrix8x8x16bOp(num_matrices=4, transpose=True)
|
||||
stsm_trans_op = warp.StMatrix8x8x16bOp(num_matrices=4, transpose=True)
|
||||
ldsm_trans_atom = cute.make_copy_atom(ldsm_trans_op, BFloat16)
|
||||
stsm_trans_atom = cute.make_copy_atom(stsm_trans_op, BFloat16)
|
||||
|
||||
# ((BT, num_BT_tiles), V_dim)
|
||||
gV_new_tiles = cute.logical_divide(
|
||||
V_new_tma.tma_tensor[None, head_id, None], (BT, None)
|
||||
)
|
||||
|
||||
# sV shape: [BT, (64, V_dim/64), num_stages]
|
||||
# sV_view shape: [BT, (8, (8,2)), num_stages]
|
||||
sV_view = cute.logical_divide(sV, (None, 8, None))
|
||||
sV_new_view = cute.logical_divide(sV_new, (None, 8))
|
||||
|
||||
# [BT, 8, num_stages]
|
||||
s_col = warp_id * 4 + (lane_id // 8)
|
||||
sV_view = sV_view[None, (None, s_col), None]
|
||||
sV_new_view = sV_new_view[None, (None, s_col)]
|
||||
|
||||
for chunk_id in range(num_chunks):
|
||||
# wait for V to arrive
|
||||
if warp_id == 0:
|
||||
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=128)
|
||||
|
||||
# unpack V BF16->FP32, then store to tmem for 1st MMA
|
||||
# V smem layout: [BT, (64, V_dim/64)] / [BT, V_dim]
|
||||
# each iteration, CTA loads [8, V_dim] tile
|
||||
# (warp loads [8, 32] tile)
|
||||
for i in cutlass.range_constexpr(BT // 8):
|
||||
s_row = i * 8 + (lane_id % 8)
|
||||
v_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
cute.copy(ldsm_trans_atom, sV_view[s_row, None, stage_id], v_bf16)
|
||||
v_fp32 = cvt.bf16x2_to_fp32x2(cute.recast_tensor(v_bf16, Uint32))
|
||||
v_fp32 = cute.logical_divide(v_fp32, 4) # (4, 2)
|
||||
|
||||
tcol = wh_tmem + i * 8
|
||||
_tcgen05.st(warp_id * 32 + 0, tcol, "16x256b", 1, v_fp32[None, 0])
|
||||
_tcgen05.st(warp_id * 32 + 16, tcol, "16x256b", 1, v_fp32[None, 1])
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
|
||||
|
||||
# load g_cu for scaling
|
||||
if tid < BT:
|
||||
end_t = min(bos + (chunk_id + 1) * BT, eos)
|
||||
last_idx = end_t - 1
|
||||
t = bos + chunk_id * BT + tid
|
||||
val = Float32(0.0)
|
||||
if t < eos:
|
||||
val = cute.math.exp(
|
||||
g_cu[last_idx, head_id] - g_cu[t, head_id],
|
||||
fastmath=True,
|
||||
)
|
||||
s_v_scale[tid] = val
|
||||
|
||||
# wait for 1st MMA to finish
|
||||
if warp_id == 2:
|
||||
cute.arch.mbarrier_wait(wh_done_mbar + stage_id, parity)
|
||||
elif warp_id == 3:
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_wait_group(0, read=True)
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(BT // 8):
|
||||
v_new = cute.make_rmem_tensor((4, 2), Float32)
|
||||
tcol = wh_tmem + i * 8
|
||||
v_new[None, 0].store(
|
||||
_tcgen05.ld(warp_id * 32 + 0, tcol, "16x256b", 1)
|
||||
)
|
||||
v_new[None, 1].store(
|
||||
_tcgen05.ld(warp_id * 32 + 16, tcol, "16x256b", 1)
|
||||
)
|
||||
v_new_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
v_new_bf16.store(v_new.load().to(BFloat16))
|
||||
|
||||
# scale V_new for 2nd MMA
|
||||
scale0 = s_v_scale[i * 8 + (lane_id % 4) * 2 + 0]
|
||||
scale1 = s_v_scale[i * 8 + (lane_id % 4) * 2 + 1]
|
||||
v_scaled = cute.make_rmem_tensor(8, Float32)
|
||||
for k in cutlass.range_constexpr(4):
|
||||
v_scaled[k * 2] = v_new[k * 2] * scale0
|
||||
v_scaled[k * 2 + 1] = v_new[k * 2 + 1] * scale1
|
||||
v_scaled_bf16 = v_scaled.load().to(BFloat16).reshape((4, 2))
|
||||
|
||||
# store V_new BF16 for O kernel
|
||||
s_row = i * 8 + (lane_id % 8)
|
||||
cute.copy(stsm_trans_atom, v_new_bf16, sV_new_view[s_row, None])
|
||||
|
||||
# store to tmem
|
||||
tcol = v_tmem_base + i * 4
|
||||
_tcgen05.st(
|
||||
warp_id * 32 + 0, tcol, "16x128b", 1, v_scaled_bf16[None, 0]
|
||||
)
|
||||
_tcgen05.st(
|
||||
warp_id * 32 + 16, tcol, "16x128b", 1, v_scaled_bf16[None, 1]
|
||||
)
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
|
||||
|
||||
# issue TMA store for V_new
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id == 3:
|
||||
gV = gV_new_tiles[(None, chunk_offset + chunk_id), None]
|
||||
simple_tma_copy(V_new_tma.atom, sV_new, gV)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(
|
||||
H: int,
|
||||
Hv: int,
|
||||
K_dim: int,
|
||||
V_dim: int,
|
||||
h_dtype: cutlass.Numeric = Float32,
|
||||
BT: int = 64,
|
||||
num_stages: int = 2,
|
||||
):
|
||||
total_t = cute.sym_int()
|
||||
pad_t = cute.sym_int()
|
||||
total_chunks_n = cute.sym_int()
|
||||
num_sequences = cute.sym_int()
|
||||
cu_entries = cute.sym_int()
|
||||
|
||||
K = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
|
||||
V = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
|
||||
W = make_fake_tensor(BFloat16, (pad_t, Hv, K_dim), divisibility=16)
|
||||
V_new = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
|
||||
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
|
||||
h = make_fake_tensor(
|
||||
BFloat16, (total_chunks_n, Hv, V_dim, K_dim), divisibility=16
|
||||
)
|
||||
h0 = make_fake_tensor(
|
||||
h_dtype, (num_sequences, Hv, V_dim, K_dim), divisibility=16
|
||||
)
|
||||
ht = make_fake_tensor(
|
||||
h_dtype, (num_sequences, Hv, V_dim, K_dim), divisibility=16
|
||||
)
|
||||
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
|
||||
chunk_offsets = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
|
||||
|
||||
kernel = Sm100ChunkHKernel(H, Hv, K_dim, V_dim, h_dtype, BT, num_stages)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
K,
|
||||
V,
|
||||
W,
|
||||
V_new,
|
||||
g_cu,
|
||||
h,
|
||||
h0,
|
||||
ht,
|
||||
cu_seqlens,
|
||||
chunk_offsets,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def h_cutedsl(
|
||||
K: torch.Tensor,
|
||||
V: torch.Tensor,
|
||||
W: torch.Tensor,
|
||||
V_new: torch.Tensor,
|
||||
g_cu: torch.Tensor,
|
||||
h: torch.Tensor,
|
||||
h0: torch.Tensor,
|
||||
ht: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
chunk_offsets: torch.Tensor,
|
||||
BT: int = 64,
|
||||
num_stages: int = 2,
|
||||
) -> None:
|
||||
"""Compute H/V_new with the same argument order as the CUDA wrapper."""
|
||||
_, H, K_dim = K.shape
|
||||
_, Hv, V_dim = V.shape
|
||||
h_dtype = {
|
||||
torch.bfloat16: BFloat16,
|
||||
torch.float32: Float32,
|
||||
}[h0.dtype]
|
||||
Sm100ChunkHKernel.compile(H, Hv, K_dim, V_dim, h_dtype, BT, num_stages)(
|
||||
K,
|
||||
V,
|
||||
W,
|
||||
V_new,
|
||||
g_cu,
|
||||
h,
|
||||
h0,
|
||||
ht,
|
||||
cu_seqlens,
|
||||
chunk_offsets,
|
||||
)
|
||||
|
||||
|
||||
h_v2b_cutedsl = h_cutedsl
|
||||
@@ -1,925 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
|
||||
from cutlass.cute.nvgpu import cpasync, warp
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import (
|
||||
EVICT_FIRST,
|
||||
_bf16x2_neg,
|
||||
_bf16x2_sub,
|
||||
_tcgen05,
|
||||
cvt,
|
||||
fence_before_tma_store,
|
||||
mma_sync,
|
||||
simple_tma_copy,
|
||||
)
|
||||
|
||||
|
||||
class Sm100ChunkUWKernel:
|
||||
"""Compute per-chunk KKT inverse preprocessing and U/W tiles.
|
||||
|
||||
Gamma[i,j] = exp(g_cu[i] - g_cu[j])
|
||||
A = strictLower(beta * (K @ K.T) * Gamma)
|
||||
Ai = inverse(I + A)
|
||||
U = (Ai * beta) @ V
|
||||
W = (Ai * beta * exp(g_cu)) @ K
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
H: int,
|
||||
Hv: int,
|
||||
K_dim: int,
|
||||
V_dim: int,
|
||||
num_stages: int = 2,
|
||||
) -> None:
|
||||
assert Hv % H == 0
|
||||
assert K_dim == V_dim == 128
|
||||
self.H = H
|
||||
self.Hv = Hv
|
||||
self.K_dim = K_dim
|
||||
self.V_dim = V_dim
|
||||
self.num_stages = num_stages
|
||||
|
||||
# hard-code
|
||||
self.BT = 64
|
||||
self.num_warps = 4 + 4 + 4
|
||||
|
||||
@cute.jit
|
||||
def _make_tma_args(
|
||||
self,
|
||||
tensor: cute.Tensor,
|
||||
dim: cutlass.Constexpr[int],
|
||||
num_stages: int,
|
||||
op: cpasync.TmaCopyOp,
|
||||
):
|
||||
# logical layout: [BT, dim]
|
||||
# permute for TMA: [dim/64, BT, 64] with swizzling
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
slayout = cute.make_layout(
|
||||
(self.BT, 1, (64, dim // 64), num_stages),
|
||||
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
|
||||
# we need to convert gmem layout to (T, H, (64, D/64)) for make_tiled_tma_atom()
|
||||
# to emit a single 4D TMA. otherwise, it will emit (D/64)x 3D TMA.
|
||||
return cpasync.make_tiled_tma_atom(
|
||||
op,
|
||||
cute.logical_divide(tensor, (None, None, 64)),
|
||||
slayout,
|
||||
cta_tiler=(self.BT, 1, dim),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
K: cute.Tensor,
|
||||
V: cute.Tensor,
|
||||
U: cute.Tensor,
|
||||
W: cute.Tensor,
|
||||
g: cute.Tensor,
|
||||
beta: cute.Tensor,
|
||||
g_cu: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
total_chunks: cute.Tensor,
|
||||
num_sms: Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
|
||||
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
|
||||
|
||||
K_tma = self._make_tma_args(K, self.K_dim, self.num_stages, tma_g2s)
|
||||
V_tma = self._make_tma_args(V, self.V_dim, self.num_stages, tma_g2s)
|
||||
U_tma = self._make_tma_args(U, self.V_dim, 1, tma_s2g)
|
||||
W_tma = self._make_tma_args(W, self.K_dim, 1, tma_s2g)
|
||||
|
||||
grid = (num_sms // self.Hv, self.Hv, 1)
|
||||
block = (self.num_warps * 32, 1, 1)
|
||||
self.kernel(
|
||||
K_tma,
|
||||
V_tma,
|
||||
U_tma,
|
||||
W_tma,
|
||||
g,
|
||||
beta,
|
||||
g_cu,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
K_tma: cpasync.TmaInfo,
|
||||
V_tma: cpasync.TmaInfo,
|
||||
U_tma: cpasync.TmaInfo,
|
||||
W_tma: cpasync.TmaInfo,
|
||||
g: cute.Tensor,
|
||||
beta: cute.Tensor,
|
||||
g_cu: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
total_chunks: cute.Tensor,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
bid, head_id, _ = cute.arch.block_idx()
|
||||
grid_x, _, _ = cute.arch.grid_dim()
|
||||
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
k_head_id = head_id // (self.Hv // self.H)
|
||||
|
||||
BT = self.BT
|
||||
K_dim = self.K_dim
|
||||
V_dim = self.V_dim
|
||||
num_stages = self.num_stages
|
||||
|
||||
INV_BAR = 1
|
||||
EPI_BAR = 2
|
||||
SCAN_BAR = 3
|
||||
TMEM_ALLOC_BAR = 4
|
||||
|
||||
def allocate_tensor(smem, dtype, layout):
|
||||
return smem.allocate_tensor(
|
||||
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
|
||||
)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
|
||||
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
|
||||
sU = allocate_tensor(smem, BFloat16, U_tma.smem_layout)[None, 0, None, 0]
|
||||
sW = allocate_tensor(smem, BFloat16, W_tma.smem_layout)[None, 0, None, 0]
|
||||
|
||||
sA_ptr = smem.allocate_array(BFloat16, BT * BT, byte_alignment=16)
|
||||
sAi_ptr = smem.allocate_array(BFloat16, BT * BT, byte_alignment=16)
|
||||
|
||||
s_beta = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
|
||||
s_g_cu = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
|
||||
s_beta_g = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
|
||||
|
||||
tma_mbar = smem.allocate_array(Int64, num_stages)
|
||||
prep_mbar = smem.allocate_array(Int64, num_stages)
|
||||
mma_kkt_mbar = smem.allocate_array(Int64, num_stages)
|
||||
inv_mbar = smem.allocate_array(Int64, num_stages)
|
||||
mma_u_mbar = smem.allocate_array(Int64, num_stages)
|
||||
mma_w_mbar = smem.allocate_array(Int64, num_stages)
|
||||
epi_mbar = smem.allocate_array(Int64, num_stages)
|
||||
taddr = smem.allocate(Int32, 4)
|
||||
|
||||
kkt_tmem = 0
|
||||
U_tmem_base = kkt_tmem + BT
|
||||
Ab_tmem_base = U_tmem_base + V_dim * num_stages
|
||||
assert Ab_tmem_base + (BT // 2) <= 512
|
||||
|
||||
# prepare ldmatrix/stmatrix ops
|
||||
ldsm_op = warp.LdMatrix8x8x16bOp(num_matrices=4)
|
||||
stsm_op = warp.StMatrix8x8x16bOp(num_matrices=4)
|
||||
ldsm_trans_op = warp.LdMatrix8x8x16bOp(num_matrices=4, transpose=True)
|
||||
ldsm_atom = cute.make_copy_atom(ldsm_op, BFloat16)
|
||||
stsm_atom = cute.make_copy_atom(stsm_op, BFloat16)
|
||||
ldsm_trans_atom = cute.make_copy_atom(ldsm_trans_op, BFloat16)
|
||||
|
||||
if warp_id == 0:
|
||||
with cute.arch.elect_one():
|
||||
for i in cutlass.range_constexpr(num_stages):
|
||||
cute.arch.mbarrier_init(tma_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(prep_mbar + i, 64)
|
||||
cute.arch.mbarrier_init(mma_kkt_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(inv_mbar + i, 128)
|
||||
cute.arch.mbarrier_init(mma_u_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(mma_w_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(epi_mbar + i, 128)
|
||||
cute.arch.mbarrier_init_fence()
|
||||
elif warp_id == 1:
|
||||
cpasync.prefetch_descriptor(K_tma.atom)
|
||||
cpasync.prefetch_descriptor(V_tma.atom)
|
||||
cpasync.prefetch_descriptor(U_tma.atom)
|
||||
cpasync.prefetch_descriptor(W_tma.atom)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
num_global_chunks = total_chunks[0]
|
||||
if warp_id == 11:
|
||||
# TMA warp
|
||||
stage_id = 0
|
||||
parity = 1
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
|
||||
# since off_t is not a multiple of BT, we need to use
|
||||
# domain_offset() to shift the pointer first.
|
||||
mbar = tma_mbar + stage_id
|
||||
gK = cute.local_tile(
|
||||
cute.domain_offset(
|
||||
(bos, 0), K_tma.tma_tensor[None, k_head_id, None]
|
||||
),
|
||||
tiler=(BT, K_dim),
|
||||
coord=(chunk_id, 0),
|
||||
)
|
||||
gV = cute.local_tile(
|
||||
cute.domain_offset((bos, 0), V_tma.tma_tensor[None, head_id, None]),
|
||||
tiler=(BT, V_dim),
|
||||
coord=(chunk_id, 0),
|
||||
)
|
||||
|
||||
# when UW MMA is done, K and V TMA buffers are released
|
||||
cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity)
|
||||
|
||||
with cute.arch.elect_one():
|
||||
STAGE_SIZE = BT * (K_dim + V_dim) * 2
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
|
||||
simple_tma_copy(K_tma.atom, gK, sK[None, None, stage_id], mbar)
|
||||
simple_tma_copy(
|
||||
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
|
||||
)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id == 10:
|
||||
# MMA warp
|
||||
_tcgen05.alloc(taddr)
|
||||
cute.arch.barrier(barrier_id=TMEM_ALLOC_BAR, number_of_threads=160)
|
||||
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
kkt_idesc = _tcgen05.make_bf16_idesc(BT, BT)
|
||||
u_idesc = _tcgen05.make_bf16_idesc(BT, V_dim, transpose_B=True)
|
||||
w_idesc = _tcgen05.make_bf16_idesc(BT, K_dim, transpose_B=True)
|
||||
|
||||
# LBO=BT*128 is ignored for K-major
|
||||
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
U_tmem = U_tmem_base + V_dim * stage_id
|
||||
W_tmem = U_tmem | (16 << 16)
|
||||
Ab_tmem = Ab_tmem_base
|
||||
Abg_tmem = Ab_tmem_base | (16 << 16)
|
||||
|
||||
##### KKT MMA: KKT = K @ K.T #####
|
||||
kaddr = sK[None, None, stage_id].iterator.toint()
|
||||
kdesc_base = sdesc_template | (kaddr >> 4)
|
||||
|
||||
# wait for TMA data to arrive
|
||||
# kkt tmem is guaranteed to be free as this is issued
|
||||
# after the previous kkt's consumer (inv warps)
|
||||
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(K_dim // 64):
|
||||
for j in cutlass.range_constexpr(64 // 16):
|
||||
kdesc = kdesc_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
enable_d = (i > 0) or (j > 0)
|
||||
_tcgen05.mma_f16(kkt_tmem, kdesc, kdesc, kkt_idesc, enable_d)
|
||||
_tcgen05.commit(mma_kkt_mbar + stage_id)
|
||||
|
||||
##### U/W MMA: U = Ab @ V, W = Abg @ K #####
|
||||
vaddr = sV[None, None, stage_id].iterator.toint()
|
||||
vdesc = sdesc_template | (vaddr >> 4)
|
||||
kdesc = sdesc_template | (kaddr >> 4)
|
||||
|
||||
# wait for epilogue to release tmem buffer
|
||||
cute.arch.mbarrier_wait(epi_mbar + stage_id, parity ^ 1)
|
||||
cute.arch.mbarrier_wait(inv_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(BT // 16):
|
||||
_tcgen05.mma_ts_f16(W_tmem, Abg_tmem + i * 8, kdesc, w_idesc, i > 0)
|
||||
kdesc += (16 * 128) >> 4
|
||||
_tcgen05.commit(mma_w_mbar + stage_id)
|
||||
|
||||
for i in cutlass.range_constexpr(BT // 16):
|
||||
_tcgen05.mma_ts_f16(U_tmem, Ab_tmem + i * 8, vdesc, u_idesc, i > 0)
|
||||
vdesc += (16 * 128) >> 4
|
||||
_tcgen05.commit(mma_u_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
cute.arch.mbarrier_wait(epi_mbar + stage_id, parity ^ 1)
|
||||
_tcgen05.dealloc()
|
||||
|
||||
elif warp_id >= 8:
|
||||
# dedicated prep warps for beta and gate, consumed by INV warps
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
tid_ = tid % 128
|
||||
warp_id_ = warp_id % 4
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
eos = cu_seqlens[seq_id + 1]
|
||||
off_t = bos + chunk_id * BT
|
||||
t = off_t + tid_
|
||||
|
||||
in_bounds = t < eos
|
||||
beta_val = beta[t, head_id] if in_bounds else Float32(0.0)
|
||||
g_val = g[t, head_id] if in_bounds else Float32(0.0)
|
||||
|
||||
# warp-local prefix scan
|
||||
for i in cutlass.range_constexpr(5):
|
||||
offset = cutlass.const_expr(1 << i)
|
||||
lower = cute.arch.shuffle_sync_up(g_val, offset, mask_and_clamp=0)
|
||||
if lane_id >= offset:
|
||||
g_val += lower
|
||||
|
||||
# Delay the stage-reuse wait until just before touching smem:
|
||||
# global loads and the warp-local scan do not use staged buffers.
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(inv_mbar + stage_id, parity ^ 1)
|
||||
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
|
||||
|
||||
# Store beta and the per-warp scan totals for the cross-warp fixup.
|
||||
s_beta[tid_, stage_id] = beta_val
|
||||
if lane_id == 31:
|
||||
s_g_cu[warp_id_, stage_id] = g_val
|
||||
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
|
||||
|
||||
# Add the sum from the lower prep warp.
|
||||
if warp_id_ == 1:
|
||||
g_val += s_g_cu[0, stage_id]
|
||||
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
|
||||
|
||||
if in_bounds:
|
||||
g_cu[t, head_id] = g_val
|
||||
|
||||
s_g_cu[tid_, stage_id] = g_val
|
||||
s_beta_g[tid_, stage_id] = beta_val * cute.math.exp(g_val)
|
||||
cute.arch.mbarrier_arrive(prep_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id >= 4:
|
||||
# inv warps
|
||||
tid_ = tid % 128
|
||||
warp_id_ = warp_id % 4
|
||||
|
||||
def store_ab_abg(
|
||||
Ai_f32, s_beta, s_beta_g, warp_id_, lane_id, tile_col, Ab_tmem_base
|
||||
):
|
||||
# compute Ab and Abg from Ai, then store to tmem
|
||||
beta_col = cute.make_rmem_tensor((2, 2), Float32)
|
||||
beta_g_col = cute.make_rmem_tensor((2, 2), Float32)
|
||||
|
||||
for i in cutlass.range_constexpr(2):
|
||||
base = tile_col * 16 + i * 8 + (lane_id % 4) * 2
|
||||
for j in cutlass.range_constexpr(2):
|
||||
beta_col[j, i] = s_beta[base + j]
|
||||
beta_g_col[j, i] = s_beta_g[base + j]
|
||||
|
||||
# without conversion to TensorSSA, cutlass.range(vectorize=True) fails
|
||||
beta_col = beta_col.load()
|
||||
beta_g_col = beta_g_col.load()
|
||||
|
||||
Ab_f32 = cute.make_rmem_tensor(8, Float32)
|
||||
Abg_f32 = cute.make_rmem_tensor(8, Float32)
|
||||
for i in cutlass.range(8, vectorize=True):
|
||||
scale_idx = (i // 4) * 2 + (i % 2)
|
||||
Ab_f32[i] = Ai_f32[i] * beta_col[scale_idx]
|
||||
Abg_f32[i] = Ai_f32[i] * beta_g_col[scale_idx]
|
||||
|
||||
Ab = Ab_f32.load().to(BFloat16)
|
||||
Abg = Abg_f32.load().to(BFloat16)
|
||||
Ab_tmem = Ab_tmem_base + tile_col * 8
|
||||
_tcgen05.st(warp_id_ * 32, Ab_tmem, "16x128b", 2, Ab)
|
||||
_tcgen05.st(warp_id_ * 32 + 16, Ab_tmem, "16x128b", 2, Abg)
|
||||
|
||||
# clear the Ab/Abg tmem buffer once before mainloop
|
||||
# this is to keep the upper triangular tiles zeros
|
||||
cute.arch.barrier(barrier_id=TMEM_ALLOC_BAR, number_of_threads=160)
|
||||
zeros_ = cute.make_rmem_tensor(BT // 2, Float32)
|
||||
zeros_.fill(0.0)
|
||||
_tcgen05.st(warp_id_ * 32, Ab_tmem_base, "32x32b", BT // 2, zeros_)
|
||||
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
# for sA, we can avoid bank conflict without using swizzling because
|
||||
# this is only used as tmp buffer between rmem<->smem, no gmem interactions.
|
||||
# to do so, we can put (8,8) tile contiguous in memory. logically, we are
|
||||
# partitioning (64,64) tile into 4x (16,16) tiles. to make indexing easier
|
||||
# later, we view (16,16) tile as (32,8) tile here.
|
||||
# (4,4) is (row_tile, col_tile).
|
||||
sA_layout = cute.make_layout(((8, 32), (4, 4)))
|
||||
sA = cute.make_tensor(sA_ptr, sA_layout)
|
||||
sAi = cute.make_tensor(sAi_ptr, sA_layout)
|
||||
|
||||
# pre-compute ldmatrix addresses
|
||||
sA_ldsm = sA[(None, lane_id), None]
|
||||
sAi_ldsm = sAi[(None, lane_id), None]
|
||||
|
||||
# init Ai smem buffer with zeros (only the first 48 rows)
|
||||
zeros_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
zeros_bf16.fill(0.0)
|
||||
for i in cutlass.range_constexpr(3):
|
||||
cute.copy(stsm_atom, zeros_bf16, sAi_ldsm[None, (i, warp_id_)])
|
||||
|
||||
# indices for ldmatrix layout later
|
||||
row_indices = cute.make_rmem_tensor((1, 2, 1), Int32)
|
||||
row_indices[0, 0, 0] = warp_id_ * 16 + (lane_id // 4)
|
||||
row_indices[0, 1, 0] = warp_id_ * 16 + (lane_id // 4) + 8
|
||||
row_indices = row_indices.load()
|
||||
|
||||
col_indices = cute.make_rmem_tensor((2, 1, 2), Int32)
|
||||
col_indices[0, 0, 0] = (lane_id % 4) * 2 + 0
|
||||
col_indices[1, 0, 0] = (lane_id % 4) * 2 + 1
|
||||
col_indices[0, 0, 1] = (lane_id % 4) * 2 + 8
|
||||
col_indices[1, 0, 1] = (lane_id % 4) * 2 + 9
|
||||
col_indices = col_indices.load()
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
eos = cu_seqlens[seq_id + 1]
|
||||
off_t = bos + chunk_id * BT
|
||||
|
||||
##### Phase 1: A = strictLower(beta * kkt * Gamma) #####
|
||||
# Ab/Abg share one tmem slot across stages. The MMA warp commits
|
||||
# tcgen05 groups in program order: KKT_i, W_i, U_i, KKT_{i+1}.
|
||||
# Waiting for KKT_i means the previous W/U commits have completed,
|
||||
# so the INV warps can safely overwrite Ab/Abg for this iteration.
|
||||
# Wait for prep warps to publish beta/gate and for KKT MMA.
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(prep_mbar + stage_id, parity)
|
||||
cute.arch.mbarrier_wait(mma_kkt_mbar + stage_id, parity)
|
||||
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
# tmem 16x256b layout / ldmatrix layout
|
||||
beta_row = cute.make_rmem_tensor(2, Float32)
|
||||
g_cu_row = cute.make_rmem_tensor(2, Float32)
|
||||
for i in cutlass.range_constexpr(2):
|
||||
idx = warp_id_ * 16 + i * 8 + (lane_id // 4)
|
||||
beta_row[i] = s_beta[idx, stage_id]
|
||||
g_cu_row[i] = s_g_cu[idx, stage_id]
|
||||
|
||||
# mode0 is 8 rows together
|
||||
# mode1 is top and bottom 8 rows
|
||||
# mode2 is groups of 16 rows
|
||||
beta_row = beta_row.load().reshape((1, 2, 1))
|
||||
g_cu_row = g_cu_row.load().reshape((1, 2, 1))
|
||||
|
||||
# mode0 is 2 consecutive elems
|
||||
# mode1 is top and bottom 8 rows
|
||||
# mode2 is next 8 columns
|
||||
# mode3 is repeating that 16x16 tile pattern
|
||||
kkt = _tcgen05.ld(kkt_tmem, 0, "16x256b", BT // 8)
|
||||
kkt = kkt.reshape((2, 2, 2, BT // 16))
|
||||
|
||||
for i in cutlass.range_constexpr(BT // 16):
|
||||
# mode0 is 2 elems next to each other
|
||||
# mode1 is 4 pairs of elems on 1 row
|
||||
# mode2 is top and bottom 8 rows
|
||||
# mode3 is next 16 columns
|
||||
col_coord = (None, lane_id % 4, None, i)
|
||||
s_g_cu_view = cute.make_tensor(
|
||||
s_g_cu[None, stage_id].iterator, (2, 4, 2, BT // 16)
|
||||
)
|
||||
g_cu_col = s_g_cu_view[col_coord].load().reshape((2, 1, 2))
|
||||
|
||||
Gamma = cute.math.exp(g_cu_row - g_cu_col, fastmath=True)
|
||||
A = kkt[None, None, None, i] * beta_row * Gamma
|
||||
|
||||
# strict lower mask
|
||||
# NOTE: for OOB t position, s_beta is filled with zeros.
|
||||
# hence, we don't need to apply bounds check for columns.
|
||||
A_masked = cute.where(row_indices > col_indices + i * 16, A, 0.0)
|
||||
|
||||
# pack to BF16
|
||||
# CuteDSL doesn't generate cvt.bf16x2.f32 here for some reasons
|
||||
packed = cute.make_rmem_tensor(4, Uint32)
|
||||
for j in cutlass.range_constexpr(4):
|
||||
packed[j] = cvt.fp32x2_to_bf16x2(
|
||||
A_masked[j * 2], A_masked[j * 2 + 1]
|
||||
)
|
||||
|
||||
# store to smem
|
||||
cute.copy(
|
||||
stsm_atom,
|
||||
cute.recast_tensor(packed, BFloat16),
|
||||
sA_ldsm[None, (warp_id_, i)],
|
||||
)
|
||||
|
||||
# use sync warp instead of bar.sync because for block-diagonal inverse,
|
||||
# each warp reads its own private smem memory.
|
||||
cute.arch.sync_warp()
|
||||
|
||||
##### Phase 2: matrix inverse #####
|
||||
# we use Newton-Schulz iterations to compute the inverse
|
||||
# of the four 16x16 diagonal blocks.
|
||||
# Ai_new = 2 Ai - Ai @ M @ Ai
|
||||
# where M = I + A
|
||||
#
|
||||
# we do this with 2 MMAs:
|
||||
# 1. -AiM = Ai @ (-M)
|
||||
# 2. Ai_new = 2 Ai + (-AiM) @ Ai
|
||||
zeros_f32 = cute.make_rmem_tensor(4, Float32)
|
||||
zeros_f32.fill(0.0)
|
||||
|
||||
Ai_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
mma_B_bf16 = cute.make_rmem_tensor((4, 2), BFloat16)
|
||||
M_bf16 = cute.make_rmem_tensor((4, 2), BFloat16)
|
||||
acc = cute.make_rmem_tensor((4, 2), Float32)
|
||||
|
||||
# ldmatrix copies require rank-1 destination views.
|
||||
mma_B_bf16_ldsm = cute.group_modes(mma_B_bf16, 0, 2)
|
||||
|
||||
# Packed aliases for BF16x2 arithmetic.
|
||||
Ai = cute.recast_tensor(Ai_bf16, Uint32)
|
||||
M = cute.recast_tensor(M_bf16, Uint32)
|
||||
|
||||
# construct rmem-backed identity matrix
|
||||
eye = cute.make_rmem_tensor(4, Uint32)
|
||||
eye[0] = Uint32(lane_id % 9 == 0) * Uint32(0x00003F80) + Uint32(
|
||||
lane_id % 9 == 4
|
||||
) * Uint32(0x3F800000)
|
||||
eye[1] = 0
|
||||
eye[2] = 0
|
||||
eye[3] = eye[0]
|
||||
|
||||
# initial guess: Ai = I-A
|
||||
cute.copy(ldsm_atom, sA_ldsm[None, (warp_id_, warp_id_)], Ai_bf16)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
Ai[i] = _bf16x2_sub(eye[i], Ai[i])
|
||||
|
||||
# (4, 2)
|
||||
Ai_f32 = cute.logical_divide(cvt.bf16x2_to_fp32x2(Ai), 4)
|
||||
|
||||
# M is holding -(I+A), stay constant throughout the iterations
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sA_ldsm[None, (warp_id_, warp_id_)],
|
||||
cute.group_modes(M_bf16, 0, 2),
|
||||
)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
M[i] = _bf16x2_sub(_bf16x2_neg(eye[i]), M[i])
|
||||
|
||||
# The fourth round keeps correlated-key KKT blocks stable in BF16.
|
||||
for _ in cutlass.range_constexpr(4):
|
||||
# First MMA: -AiM = Ai @ (-M)
|
||||
cute.copy(stsm_atom, Ai_bf16, sA_ldsm[None, (warp_id_, warp_id_)])
|
||||
cute.arch.sync_warp()
|
||||
acc[None, 0] = mma_sync(Ai_bf16, M_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, M_bf16[None, 1], zeros_f32)
|
||||
Ai_bf16.store(acc.load().to(BFloat16))
|
||||
|
||||
# Second MMA: Ai_new = 2Ai + (-AiM) @ Ai
|
||||
for j in cutlass.range(8, vectorize=True):
|
||||
Ai_f32[j] *= 2.0
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sA_ldsm[None, (warp_id_, warp_id_)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
Ai_f32[None, 0] = mma_sync(
|
||||
Ai_bf16, mma_B_bf16[None, 0], Ai_f32[None, 0]
|
||||
)
|
||||
Ai_f32[None, 1] = mma_sync(
|
||||
Ai_bf16, mma_B_bf16[None, 1], Ai_f32[None, 1]
|
||||
)
|
||||
Ai_bf16.store(Ai_f32.load().to(BFloat16))
|
||||
|
||||
cute.copy(stsm_atom, Ai_bf16, sAi_ldsm[None, (warp_id_, warp_id_)])
|
||||
store_ab_abg(
|
||||
Ai_f32,
|
||||
s_beta[None, stage_id],
|
||||
s_beta_g[None, stage_id],
|
||||
warp_id_,
|
||||
lane_id,
|
||||
warp_id_,
|
||||
Ab_tmem_base,
|
||||
)
|
||||
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
|
||||
|
||||
# off-diagonal by 1
|
||||
# given
|
||||
# [ Ai00 ]
|
||||
# [ A10 Ai11 ]
|
||||
# [ A20 A21 Ai22 ]
|
||||
# [ A30 A31 A32 Ai33]
|
||||
# warp1: Ai10 = -Ai11 @ A10 @ Ai00
|
||||
# warp2: Ai21 = -Ai22 @ A21 @ Ai11
|
||||
# warp3: Ai32 = -Ai33 @ A32 @ Ai22
|
||||
if warp_id_ >= 1:
|
||||
neg_Ai = cute.make_rmem_tensor(4, Uint32)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
neg_Ai[i] = _bf16x2_neg(Ai[i])
|
||||
neg_Ai_bf16 = cute.recast_tensor(neg_Ai, BFloat16)
|
||||
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sA_ldsm[None, (warp_id_, warp_id_ - 1)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(neg_Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(neg_Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
Ai_bf16.store(acc.load().to(BFloat16))
|
||||
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (warp_id_ - 1, warp_id_ - 1)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
Ai_bf16.store(acc.load().to(BFloat16))
|
||||
store_ab_abg(
|
||||
acc,
|
||||
s_beta[None, stage_id],
|
||||
s_beta_g[None, stage_id],
|
||||
warp_id_,
|
||||
lane_id,
|
||||
warp_id_ - 1,
|
||||
Ab_tmem_base,
|
||||
)
|
||||
cute.copy(
|
||||
stsm_atom,
|
||||
Ai_bf16,
|
||||
sAi_ldsm[None, (warp_id_, warp_id_ - 1)],
|
||||
)
|
||||
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
|
||||
|
||||
# off-diagonal by 2
|
||||
# warp2: Ai20 = -Ai22 @ (A20 @ Ai00 + A21 @ Ai10)
|
||||
# warp3: Ai31 = -Ai33 @ (A31 @ Ai11 + A32 @ Ai21)
|
||||
if warp_id_ >= 2:
|
||||
tile_col = warp_id_ - 2
|
||||
cute.copy(
|
||||
ldsm_atom,
|
||||
sA_ldsm[None, (warp_id_, tile_col)],
|
||||
Ai_bf16,
|
||||
)
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (tile_col, tile_col)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
|
||||
cute.copy(
|
||||
ldsm_atom, sA_ldsm[None, (warp_id_, tile_col + 1)], Ai_bf16
|
||||
)
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (tile_col + 1, tile_col)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], acc[None, 0])
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], acc[None, 1])
|
||||
|
||||
tmp = cute.make_rmem_tensor(8, BFloat16)
|
||||
tmp.store(acc.load().to(BFloat16))
|
||||
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (warp_id_, tile_col)])
|
||||
cute.arch.sync_warp()
|
||||
|
||||
cute.copy(ldsm_atom, sAi_ldsm[None, (warp_id_, warp_id_)], Ai_bf16)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
Ai[i] = _bf16x2_neg(Ai[i])
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (warp_id_, tile_col)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
tmp.store(acc.load().to(BFloat16))
|
||||
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (warp_id_, tile_col)])
|
||||
store_ab_abg(
|
||||
acc,
|
||||
s_beta[None, stage_id],
|
||||
s_beta_g[None, stage_id],
|
||||
warp_id_,
|
||||
lane_id,
|
||||
tile_col,
|
||||
Ab_tmem_base,
|
||||
)
|
||||
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
|
||||
|
||||
# off-diagonal by 3
|
||||
# warp3: Ai30 = -Ai33 @ (A30 @ Ai00 + A31 @ Ai10 + A32 @ Ai20)
|
||||
if warp_id_ == 3:
|
||||
cute.copy(ldsm_atom, sA_ldsm[None, (3, 0)], Ai_bf16)
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (0, 0)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
|
||||
for i in cutlass.range_constexpr(1, 3):
|
||||
cute.copy(ldsm_atom, sA_ldsm[None, (3, i)], Ai_bf16)
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (i, 0)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(
|
||||
Ai_bf16, mma_B_bf16[None, 0], acc[None, 0]
|
||||
)
|
||||
acc[None, 1] = mma_sync(
|
||||
Ai_bf16, mma_B_bf16[None, 1], acc[None, 1]
|
||||
)
|
||||
|
||||
tmp = cute.make_rmem_tensor(8, BFloat16)
|
||||
tmp.store(acc.load().to(BFloat16))
|
||||
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (3, 0)])
|
||||
cute.arch.sync_warp()
|
||||
|
||||
cute.copy(ldsm_atom, sAi_ldsm[None, (3, 3)], Ai_bf16)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
Ai[i] = _bf16x2_neg(Ai[i])
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (3, 0)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
tmp.store(acc.load().to(BFloat16))
|
||||
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (3, 0)])
|
||||
store_ab_abg(
|
||||
acc,
|
||||
s_beta[None, stage_id],
|
||||
s_beta_g[None, stage_id],
|
||||
warp_id_,
|
||||
lane_id,
|
||||
0,
|
||||
Ab_tmem_base,
|
||||
)
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(inv_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
else:
|
||||
# epi warps
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
# ((BT, num_global_chunks), V_dim)
|
||||
gU_tiles = cute.logical_divide(
|
||||
U_tma.tma_tensor[None, head_id, None], (BT, None)
|
||||
)
|
||||
gW_tiles = cute.logical_divide(
|
||||
W_tma.tma_tensor[None, head_id, None], (BT, None)
|
||||
)
|
||||
|
||||
# sW shape: [BT, (64, K_dim/64)]
|
||||
# sW_view shape: [(8, 2), (4, K_dim/64)]
|
||||
s_row = warp_id * 16 + lane_id % 16 # select the rows of [16,16] tile
|
||||
sW_view = cute.zipped_divide(
|
||||
sW[s_row, None],
|
||||
tiler=cute.make_layout((8, 2)),
|
||||
)
|
||||
sU_view = cute.zipped_divide(
|
||||
sU[s_row, None],
|
||||
tiler=cute.make_layout((8, 2)),
|
||||
)
|
||||
|
||||
# select the 8 columns within [16,16] tile
|
||||
sW_view = sW_view[(None, lane_id // 16), None]
|
||||
sU_view = sU_view[(None, lane_id // 16), None]
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
# wait for W MMA + previous TMA store to finish
|
||||
U_tmem = U_tmem_base + V_dim * stage_id
|
||||
if warp_id == 0:
|
||||
cute.arch.mbarrier_wait(mma_w_mbar + stage_id, parity)
|
||||
elif warp_id == 1:
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_wait_group(0, read=True)
|
||||
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
w_f32 = _tcgen05.ld(warp_id * 32 + 16, U_tmem, "16x256b", K_dim // 8)
|
||||
_tcgen05.wait_ld()
|
||||
w_bf16 = cute.make_rmem_tensor((8, K_dim // 16), BFloat16)
|
||||
w_bf16.store(w_f32.to(BFloat16))
|
||||
cute.copy(stsm_atom, w_bf16, sW_view)
|
||||
|
||||
# wait for U MMA + issue W TMA store
|
||||
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id == 0:
|
||||
cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity)
|
||||
elif warp_id == 1:
|
||||
# don't need to commit
|
||||
simple_tma_copy(
|
||||
W_tma.atom, sW, gW_tiles[(None, global_chunk_id), None]
|
||||
)
|
||||
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
u_f32 = _tcgen05.ld(warp_id * 32, U_tmem, "16x256b", V_dim // 8)
|
||||
_tcgen05.wait_ld()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(epi_mbar + stage_id)
|
||||
u_bf16 = cute.make_rmem_tensor((8, V_dim // 16), BFloat16)
|
||||
u_bf16.store(u_f32.to(BFloat16))
|
||||
cute.copy(stsm_atom, u_bf16, sU_view)
|
||||
|
||||
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id == 1:
|
||||
simple_tma_copy(
|
||||
U_tma.atom, sU, gU_tiles[(None, global_chunk_id), None]
|
||||
)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(H: int, Hv: int, K_dim: int, V_dim: int, num_stages: int = 2):
|
||||
total_t = cute.sym_int()
|
||||
pad_t = cute.sym_int()
|
||||
total_chunks_n = cute.sym_int()
|
||||
num_sequences = cute.sym_int()
|
||||
|
||||
K = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
|
||||
V = make_fake_tensor(BFloat16, (total_t, Hv, V_dim), divisibility=16)
|
||||
U = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
|
||||
W = make_fake_tensor(BFloat16, (pad_t, Hv, K_dim), divisibility=16)
|
||||
g = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
|
||||
beta = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
|
||||
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
|
||||
cu_seqlens = make_fake_tensor(Int32, (num_sequences,), divisibility=1)
|
||||
chunk_indices = make_fake_tensor(Int32, (total_chunks_n, 2), divisibility=2)
|
||||
total_chunks = make_fake_tensor(Int32, (1,), divisibility=1)
|
||||
|
||||
kernel = Sm100ChunkUWKernel(H, Hv, K_dim, V_dim, num_stages)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
K,
|
||||
V,
|
||||
U,
|
||||
W,
|
||||
g,
|
||||
beta,
|
||||
g_cu,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
Int32(148),
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def kkt_inv_uw_cutedsl(
|
||||
K: torch.Tensor,
|
||||
V: torch.Tensor,
|
||||
U: torch.Tensor,
|
||||
W: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
g_cu: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
chunk_indices: torch.Tensor,
|
||||
total_chunks: torch.Tensor,
|
||||
num_sms: int = 148,
|
||||
) -> None:
|
||||
_, Hv, V_dim = V.shape
|
||||
_, H, K_dim = K.shape
|
||||
|
||||
Sm100ChunkUWKernel.compile(H, Hv, K_dim, V_dim)(
|
||||
K,
|
||||
V,
|
||||
U,
|
||||
W,
|
||||
g,
|
||||
beta,
|
||||
g_cu,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
num_sms,
|
||||
)
|
||||
@@ -1,632 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
|
||||
from cutlass.cute.nvgpu import cpasync, warp
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import (
|
||||
EVICT_FIRST,
|
||||
_tcgen05,
|
||||
cvt,
|
||||
fence_before_tma_store,
|
||||
simple_tma_copy,
|
||||
)
|
||||
|
||||
|
||||
class Sm100ChunkOKernel:
|
||||
"""Compute per-token output from recurrent and intra-chunk terms.
|
||||
|
||||
Gamma[i,j] = exp(g_cu[i] - g_cu[j])
|
||||
P = mask((Q @ K.T) * Gamma)
|
||||
O = scale * (exp(g_cu) * (Q @ H.T) + P @ V)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
H: int,
|
||||
Hv: int,
|
||||
K_dim: int,
|
||||
V_dim: int,
|
||||
BT: int = 64,
|
||||
num_stages: int = 2,
|
||||
) -> None:
|
||||
assert Hv % H == 0
|
||||
assert K_dim == 128
|
||||
assert V_dim == 128
|
||||
assert BT == 64
|
||||
self.H = H
|
||||
self.Hv = Hv
|
||||
self.K_dim = K_dim
|
||||
self.V_dim = V_dim
|
||||
self.BT = BT
|
||||
self.num_stages = num_stages
|
||||
self.num_warps = 10
|
||||
|
||||
@cute.jit
|
||||
def _make_bf16_tma_args(
|
||||
self,
|
||||
tensor: cute.Tensor,
|
||||
dim: cutlass.Constexpr[int],
|
||||
op: cpasync.TmaCopyOp,
|
||||
stages: cutlass.Constexpr[int],
|
||||
):
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
slayout = cute.make_layout(
|
||||
(self.BT, 1, (64, dim // 64), stages),
|
||||
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
return cpasync.make_tiled_tma_atom(
|
||||
op,
|
||||
cute.logical_divide(tensor, (None, None, 64)),
|
||||
slayout,
|
||||
cta_tiler=(self.BT, 1, dim),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _make_h_tma_args(
|
||||
self,
|
||||
tensor: cute.Tensor,
|
||||
op: cpasync.TmaCopyOp,
|
||||
stages: cutlass.Constexpr[int],
|
||||
):
|
||||
num_elems = 128 // (tensor.element_type.width // 8)
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
slayout = cute.make_layout(
|
||||
(1, 1, self.V_dim, (num_elems, self.K_dim // num_elems), stages),
|
||||
stride=(
|
||||
0,
|
||||
0,
|
||||
num_elems,
|
||||
(1, self.V_dim * num_elems),
|
||||
self.V_dim * self.K_dim,
|
||||
),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
return cpasync.make_tiled_tma_atom(
|
||||
op,
|
||||
cute.logical_divide(tensor, (None, None, None, num_elems)),
|
||||
slayout,
|
||||
cta_tiler=(1, 1, self.V_dim, self.K_dim),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
q: cute.Tensor,
|
||||
k: cute.Tensor,
|
||||
v_new_chunks: cute.Tensor,
|
||||
h: cute.Tensor,
|
||||
g_cu: cute.Tensor,
|
||||
o: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
total_chunks: cute.Tensor,
|
||||
scale: Float32,
|
||||
num_sms: Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (num_sms // self.Hv, self.Hv, 1)
|
||||
block = (self.num_warps * 32, 1, 1)
|
||||
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
|
||||
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
|
||||
Q_tma = self._make_bf16_tma_args(q, self.K_dim, tma_g2s, self.num_stages)
|
||||
K_tma = self._make_bf16_tma_args(k, self.K_dim, tma_g2s, self.num_stages)
|
||||
V_tma = self._make_bf16_tma_args(
|
||||
v_new_chunks, self.V_dim, tma_g2s, self.num_stages
|
||||
)
|
||||
H_tma = self._make_h_tma_args(h, tma_g2s, self.num_stages)
|
||||
O_tma = self._make_bf16_tma_args(o, self.V_dim, tma_s2g, 1)
|
||||
self.kernel(
|
||||
Q_tma,
|
||||
K_tma,
|
||||
V_tma,
|
||||
H_tma,
|
||||
O_tma,
|
||||
g_cu,
|
||||
o,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
scale,
|
||||
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
Q_tma: cpasync.TmaInfo,
|
||||
K_tma: cpasync.TmaInfo,
|
||||
V_tma: cpasync.TmaInfo,
|
||||
H_tma: cpasync.TmaInfo,
|
||||
O_tma: cpasync.TmaInfo,
|
||||
g_cu: cute.Tensor,
|
||||
o: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
total_chunks: cute.Tensor,
|
||||
scale: Float32,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
bid, v_head_id, _ = cute.arch.block_idx()
|
||||
grid_x, _, _ = cute.arch.grid_dim()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
|
||||
BT = self.BT
|
||||
K_dim = self.K_dim
|
||||
V_dim = self.V_dim
|
||||
num_stages = self.num_stages
|
||||
|
||||
heads_per_qk = self.Hv // self.H
|
||||
k_head_id = v_head_id // heads_per_qk
|
||||
num_global_chunks = total_chunks[0]
|
||||
|
||||
def allocate_tensor(smem, dtype, layout):
|
||||
return smem.allocate_tensor(
|
||||
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
|
||||
)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
sQ = allocate_tensor(smem, BFloat16, Q_tma.smem_layout)[None, 0, None, None]
|
||||
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
|
||||
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
|
||||
sH = allocate_tensor(smem, BFloat16, H_tma.smem_layout)[0, 0, None, None, None]
|
||||
sO = allocate_tensor(smem, BFloat16, O_tma.smem_layout)[None, 0, None, 0]
|
||||
|
||||
s_g_cu = smem.allocate_array(Float32, BT)
|
||||
qk_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||
hv_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||
qk_empty_mbar = smem.allocate_array(Int64, num_stages)
|
||||
pv_mma_mbar = smem.allocate_array(Int64, num_stages)
|
||||
qk_mbar = smem.allocate_array(Int64, 1)
|
||||
mask_mbar = smem.allocate_array(Int64, 1)
|
||||
epi_mbar = smem.allocate_array(Int64, 1)
|
||||
taddr = smem.allocate(Int32, 4)
|
||||
|
||||
qk_tmem = 0
|
||||
p_tmem = 64
|
||||
out_tmem = 128
|
||||
qh_tmem = 256
|
||||
|
||||
if warp_id == 0:
|
||||
with cute.arch.elect_one():
|
||||
for i in cutlass.range_constexpr(num_stages):
|
||||
cute.arch.mbarrier_init(qk_full_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(qk_empty_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(hv_full_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(pv_mma_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(qk_mbar, 1)
|
||||
cute.arch.mbarrier_init(mask_mbar, 128)
|
||||
cute.arch.mbarrier_init(epi_mbar, 128)
|
||||
cute.arch.mbarrier_init_fence()
|
||||
elif warp_id == 9:
|
||||
cpasync.prefetch_descriptor(Q_tma.atom)
|
||||
cpasync.prefetch_descriptor(K_tma.atom)
|
||||
cpasync.prefetch_descriptor(V_tma.atom)
|
||||
cpasync.prefetch_descriptor(H_tma.atom)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
if warp_id == 9:
|
||||
# TMA warp
|
||||
stage_id = 0
|
||||
parity = 1
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
|
||||
# copy Q and K
|
||||
q_tile = cute.local_tile(
|
||||
cute.domain_offset(
|
||||
(bos, 0), Q_tma.tma_tensor[None, k_head_id, None]
|
||||
),
|
||||
tiler=(BT, K_dim),
|
||||
coord=(chunk_id, 0),
|
||||
)
|
||||
k_tile = cute.local_tile(
|
||||
cute.domain_offset(
|
||||
(bos, 0), K_tma.tma_tensor[None, k_head_id, None]
|
||||
),
|
||||
tiler=(BT, K_dim),
|
||||
coord=(chunk_id, 0),
|
||||
)
|
||||
mbar = qk_full_mbar + stage_id
|
||||
|
||||
cute.arch.mbarrier_wait(qk_empty_mbar + stage_id, parity)
|
||||
|
||||
with cute.arch.elect_one():
|
||||
STAGE_SIZE = BT * (K_dim + K_dim) * 2
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
|
||||
simple_tma_copy(Q_tma.atom, q_tile, sQ[None, None, stage_id], mbar)
|
||||
simple_tma_copy(K_tma.atom, k_tile, sK[None, None, stage_id], mbar)
|
||||
|
||||
# copy H and V
|
||||
gH = H_tma.tma_tensor[global_chunk_id, v_head_id, None, None]
|
||||
gV = cute.local_tile(
|
||||
V_tma.tma_tensor[None, v_head_id, None],
|
||||
tiler=(BT, V_dim),
|
||||
coord=(global_chunk_id, 0),
|
||||
)
|
||||
mbar = hv_full_mbar + stage_id
|
||||
|
||||
cute.arch.mbarrier_wait(pv_mma_mbar + stage_id, parity)
|
||||
|
||||
with cute.arch.elect_one():
|
||||
H_STAGE_SIZE = V_dim * K_dim * 2
|
||||
V_STAGE_SIZE = BT * V_dim * 2
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(
|
||||
mbar, H_STAGE_SIZE + V_STAGE_SIZE
|
||||
)
|
||||
simple_tma_copy(
|
||||
H_tma.atom, gH, sH[None, None, stage_id], mbar, EVICT_FIRST
|
||||
)
|
||||
simple_tma_copy(
|
||||
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
|
||||
)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id == 8:
|
||||
# MMA warp
|
||||
_tcgen05.alloc(taddr)
|
||||
|
||||
# LBO=BT*128 is ignored for K-major
|
||||
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
|
||||
qk_idesc = _tcgen05.make_bf16_idesc(BT, BT)
|
||||
qh_idesc = _tcgen05.make_bf16_idesc(BT, V_dim)
|
||||
pv_idesc = _tcgen05.make_bf16_idesc(BT, V_dim, transpose_B=True)
|
||||
|
||||
stage_id = 0
|
||||
tma_parity = 0
|
||||
mask_parity = 0
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
qaddr = sQ[None, None, stage_id].iterator.toint()
|
||||
kaddr = sK[None, None, stage_id].iterator.toint()
|
||||
haddr = sH[None, None, stage_id].iterator.toint()
|
||||
vaddr = sV[None, None, stage_id].iterator.toint()
|
||||
qdesc_base = sdesc_template | (qaddr >> 4)
|
||||
kdesc_base = sdesc_template | (kaddr >> 4)
|
||||
hdesc_base = sdesc_template | (haddr >> 4)
|
||||
vdesc_base = sdesc_template | (vaddr >> 4)
|
||||
|
||||
##### 1st MMA: Q @ K.T #####
|
||||
# do this first to unblock mask(QK)
|
||||
cute.arch.mbarrier_wait(epi_mbar, mask_parity ^ 1)
|
||||
cute.arch.mbarrier_wait(qk_full_mbar + stage_id, tma_parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(K_dim // 64):
|
||||
for j in cutlass.range_constexpr(64 // 16):
|
||||
qdesc = qdesc_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
kdesc = kdesc_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
_tcgen05.mma_f16(
|
||||
qk_tmem, qdesc, kdesc, qk_idesc, (i > 0) or (j > 0)
|
||||
)
|
||||
_tcgen05.commit(qk_mbar)
|
||||
|
||||
##### 2nd MMA: Q @ H.T #####
|
||||
cute.arch.mbarrier_wait(hv_full_mbar + stage_id, tma_parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
for i in cutlass.range_constexpr(K_dim // 64):
|
||||
for j in cutlass.range_constexpr(64 // 16):
|
||||
qdesc = qdesc_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
hdesc = hdesc_base | ((i * V_dim * 128 + j * 32) >> 4)
|
||||
_tcgen05.mma_f16(
|
||||
qh_tmem, qdesc, hdesc, qh_idesc, (i > 0) or (j > 0)
|
||||
)
|
||||
_tcgen05.commit(qk_empty_mbar + stage_id)
|
||||
|
||||
##### 3rd MMA: P @ V #####
|
||||
# stalled by mask(QK)
|
||||
cute.arch.mbarrier_wait(mask_mbar, mask_parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
for i in cutlass.range_constexpr(BT // 16):
|
||||
vdesc = vdesc_base | ((i * 16 * 128) >> 4)
|
||||
_tcgen05.mma_ts_f16(
|
||||
out_tmem, p_tmem + i * 8, vdesc, pv_idesc, i > 0
|
||||
)
|
||||
_tcgen05.commit(pv_mma_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
tma_parity ^= 1
|
||||
mask_parity ^= 1
|
||||
|
||||
# wait for epilogue to finish for deallocation
|
||||
cute.arch.mbarrier_wait(epi_mbar, mask_parity ^ 1)
|
||||
_tcgen05.dealloc()
|
||||
|
||||
elif warp_id >= 4:
|
||||
# masking warps
|
||||
warp_id_ = warp_id % 4
|
||||
tid_ = tid % 128
|
||||
row0 = warp_id_ * 16 + lane_id // 4
|
||||
row1 = row0 + 8
|
||||
|
||||
parity = 0
|
||||
|
||||
# for ldmatrix layout later
|
||||
row_indices = cute.make_rmem_tensor(2, Int32)
|
||||
row_indices[0] = warp_id_ * 16 + lane_id // 4
|
||||
row_indices[1] = warp_id_ * 16 + lane_id // 4 + 8
|
||||
row_indices = row_indices.load().reshape((1, 2))
|
||||
|
||||
col_indices = cute.make_rmem_tensor(2, Int32)
|
||||
col_indices[0] = (lane_id % 4) * 2
|
||||
col_indices[1] = (lane_id % 4) * 2 + 1
|
||||
col_indices = col_indices.load().reshape((2, 1))
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
if tid_ < BT:
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
eos = cu_seqlens[seq_id + 1]
|
||||
|
||||
t_ = bos + chunk_id * BT + tid_
|
||||
s_g_cu[tid_] = g_cu[t_, v_head_id] if t_ < eos else Float32(0.0)
|
||||
|
||||
# wait for QK MMA
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(qk_mbar, parity)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
qk = _tcgen05.ld(warp_id_ * 32, qk_tmem, "16x256b", BT // 8)
|
||||
qk = qk.reshape((2, 2, BT // 8))
|
||||
_tcgen05.wait_ld()
|
||||
|
||||
g_cu_rows = cute.make_rmem_tensor(2, Float32)
|
||||
g_cu_rows[0] = s_g_cu[row0]
|
||||
g_cu_rows[1] = s_g_cu[row1]
|
||||
g_cu_rows = g_cu_rows.load().reshape((1, 2))
|
||||
|
||||
for i in cutlass.range_constexpr(BT // 8):
|
||||
col = i * 8 + (lane_id % 4) * 2
|
||||
g_cu_cols = cute.make_rmem_tensor(2, Float32)
|
||||
g_cu_cols[0] = s_g_cu[col]
|
||||
g_cu_cols[1] = s_g_cu[col + 1]
|
||||
g_cu_cols = g_cu_cols.load().reshape((2, 1))
|
||||
|
||||
# apply gamma and causal mask
|
||||
Gamma = cute.math.exp(g_cu_rows - g_cu_cols, fastmath=True)
|
||||
tmp = qk[None, None, i] * Gamma
|
||||
tmp = cute.where(row_indices >= col_indices + i * 8, tmp, 0.0)
|
||||
|
||||
# CuteDSL can't emit cvt.bf16x2.f32 here
|
||||
attn_lo = cute.make_rmem_tensor(2, Uint32)
|
||||
attn_lo[0] = cvt.fp32x2_to_bf16x2(tmp[0, 0], tmp[1, 0])
|
||||
attn_lo[1] = cvt.fp32x2_to_bf16x2(tmp[0, 1], tmp[1, 1])
|
||||
_tcgen05.st(warp_id_ * 32, p_tmem + i * 4, "16x128b", 1, attn_lo)
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(mask_mbar)
|
||||
|
||||
parity ^= 1
|
||||
|
||||
else:
|
||||
# epilogue warps
|
||||
# for ldmatrix layout later
|
||||
row0 = warp_id * 16 + lane_id // 4
|
||||
row1 = row0 + 8
|
||||
|
||||
stage_id = 0
|
||||
mma_parity = 0
|
||||
|
||||
op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_4B = cute.make_copy_atom(op, BFloat16, num_bits_per_copy=32)
|
||||
stsm_op = warp.StMatrix8x8x16bOp(num_matrices=4, transpose=False)
|
||||
stsm_atom = cute.make_copy_atom(stsm_op, BFloat16)
|
||||
|
||||
# ldmatrix layout
|
||||
# [total_seq_len, ((2, 4, WIDTH/8), V_DIM/WIDTH)]
|
||||
WIDTH = 64
|
||||
o_view = cute.logical_divide(
|
||||
o[None, v_head_id, None],
|
||||
(None, cute.make_layout((2, 4, WIDTH // 8))),
|
||||
)
|
||||
# select lane: [total_seq_len, 2, WIDTH/8, V_DIM/WIDTH]
|
||||
o_view = o_view[None, ((None, lane_id % 4, None), None)]
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
eos = cu_seqlens[seq_id + 1]
|
||||
chunk_start = bos + chunk_id * BT
|
||||
full_chunk = chunk_start + BT <= eos
|
||||
|
||||
g_cu_rows = cute.make_rmem_tensor(2, Float32)
|
||||
g_cu_rows.fill(0.0)
|
||||
|
||||
# load g_cu
|
||||
if chunk_start + row0 < eos:
|
||||
g_cu_rows[0] = cute.math.exp(
|
||||
g_cu[chunk_start + row0, v_head_id], fastmath=True
|
||||
)
|
||||
if chunk_start + row1 < eos:
|
||||
g_cu_rows[1] = cute.math.exp(
|
||||
g_cu[chunk_start + row1, v_head_id], fastmath=True
|
||||
)
|
||||
g_cu_rows = g_cu_rows.load().reshape((1, 2, 1))
|
||||
|
||||
if warp_id == 0:
|
||||
cute.arch.mbarrier_wait(pv_mma_mbar + stage_id, mma_parity)
|
||||
elif warp_id == 3 and full_chunk:
|
||||
cute.arch.cp_async_bulk_wait_group(0, read=True)
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
if full_chunk:
|
||||
# use TMA store: tmem->rmem->smem->gmem
|
||||
for i in cutlass.range_constexpr(V_dim // WIDTH):
|
||||
qh = _tcgen05.ld(
|
||||
warp_id * 32, qh_tmem + i * WIDTH, "16x256b", WIDTH // 8
|
||||
)
|
||||
pv = _tcgen05.ld(
|
||||
warp_id * 32, out_tmem + i * WIDTH, "16x256b", WIDTH // 8
|
||||
)
|
||||
_tcgen05.wait_ld()
|
||||
if i == V_dim // WIDTH - 1:
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(epi_mbar)
|
||||
|
||||
qh = qh.reshape((2, 2, WIDTH // 8))
|
||||
pv = pv.reshape((2, 2, WIDTH // 8))
|
||||
|
||||
out_f32 = scale * (g_cu_rows * qh + pv)
|
||||
out_bf16 = cute.make_rmem_tensor((8, WIDTH // 16), BFloat16)
|
||||
out_bf16.store(out_f32.to(BFloat16).reshape((8, WIDTH // 16)))
|
||||
|
||||
# TODO: issue single cute.copy()
|
||||
for j in cutlass.range_constexpr(WIDTH // 16):
|
||||
s_row = warp_id * 16 + lane_id % 16
|
||||
s_col = i * (WIDTH // 8) + j * 2 + lane_id // 16
|
||||
sO_tile = cute.local_tile(sO[s_row, None], (8,), (s_col,))
|
||||
cute.copy(stsm_atom, out_bf16[None, j], sO_tile)
|
||||
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id == 3:
|
||||
gO = cute.local_tile(
|
||||
cute.domain_offset(
|
||||
(bos, 0), O_tma.tma_tensor[None, v_head_id, None]
|
||||
),
|
||||
tiler=(BT, V_dim),
|
||||
coord=(chunk_id, 0),
|
||||
)
|
||||
simple_tma_copy(O_tma.atom, sO, gO)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
|
||||
else:
|
||||
# direct gmem store
|
||||
# TODO: explore doing multiple 1D TMAs
|
||||
for i in cutlass.range_constexpr(V_dim // WIDTH):
|
||||
qh = _tcgen05.ld(
|
||||
warp_id * 32, qh_tmem + i * WIDTH, "16x256b", WIDTH // 8
|
||||
)
|
||||
pv = _tcgen05.ld(
|
||||
warp_id * 32, out_tmem + i * WIDTH, "16x256b", WIDTH // 8
|
||||
)
|
||||
_tcgen05.wait_ld()
|
||||
if i == V_dim // WIDTH - 1:
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(epi_mbar)
|
||||
|
||||
qh = qh.reshape((2, 2, WIDTH // 8))
|
||||
pv = pv.reshape((2, 2, WIDTH // 8))
|
||||
|
||||
out_f32 = scale * (g_cu_rows * qh + pv)
|
||||
out_bf16 = cute.make_rmem_tensor((2, 2, WIDTH // 8), BFloat16)
|
||||
out_bf16.store(out_f32.to(BFloat16))
|
||||
|
||||
if chunk_start + row0 < eos:
|
||||
cute.copy(
|
||||
cp_4B,
|
||||
out_bf16[None, 0, None],
|
||||
o_view[chunk_start + row0, None, None, i],
|
||||
)
|
||||
if chunk_start + row1 < eos:
|
||||
cute.copy(
|
||||
cp_4B,
|
||||
out_bf16[None, 1, None],
|
||||
o_view[chunk_start + row1, None, None, i],
|
||||
)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
mma_parity ^= 1
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(
|
||||
H: int,
|
||||
Hv: int,
|
||||
K_dim: int,
|
||||
V_dim: int,
|
||||
BT: int = 64,
|
||||
num_stages: int = 2,
|
||||
):
|
||||
total_t = cute.sym_int()
|
||||
pad_t = cute.sym_int()
|
||||
total_chunks_n = cute.sym_int()
|
||||
cu_entries = cute.sym_int()
|
||||
|
||||
q = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
|
||||
k = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
|
||||
v_new = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
|
||||
h = make_fake_tensor(
|
||||
BFloat16, (total_chunks_n, Hv, V_dim, K_dim), divisibility=16
|
||||
)
|
||||
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
|
||||
o = make_fake_tensor(BFloat16, (total_t, Hv, V_dim), divisibility=16)
|
||||
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
|
||||
chunk_indices = make_fake_tensor(Int32, (total_chunks_n, 2), divisibility=2)
|
||||
total_chunks = make_fake_tensor(Int32, (1,), divisibility=1)
|
||||
|
||||
kernel = Sm100ChunkOKernel(
|
||||
H,
|
||||
Hv,
|
||||
K_dim,
|
||||
V_dim,
|
||||
BT,
|
||||
num_stages,
|
||||
)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
q,
|
||||
k,
|
||||
v_new,
|
||||
h,
|
||||
g_cu,
|
||||
o,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
Float32(1.0),
|
||||
Int32(148),
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def o_cutedsl(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v_new: torch.Tensor,
|
||||
h: torch.Tensor,
|
||||
g_cu: torch.Tensor,
|
||||
o: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
chunk_indices: torch.Tensor,
|
||||
total_chunks: torch.Tensor,
|
||||
scale: float,
|
||||
num_sms: int = 148,
|
||||
) -> None:
|
||||
_, H, K_dim = q.shape
|
||||
_, Hv, V_dim = o.shape
|
||||
|
||||
Sm100ChunkOKernel.compile(H, Hv, K_dim, V_dim)(
|
||||
q,
|
||||
k,
|
||||
v_new,
|
||||
h,
|
||||
g_cu,
|
||||
o,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
float(scale),
|
||||
num_sms,
|
||||
)
|
||||
Reference in New Issue
Block a user