[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,231 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Replicated input embedding + its fused gather/norm kernels.
|
||||
|
||||
Groups the ``VLLM_REPLICATE_EMBED`` path in one place: the embedding factory,
|
||||
the predicate that says whether the fusions apply, and the two Triton fusions
|
||||
the full on-rank table unlocks --
|
||||
|
||||
* ``fused_embed_norm``: gather + a chained RMSNorm (e.g. the first decoder
|
||||
layer's ``input_layernorm``), and
|
||||
* ``fused_embed_eh_norm``: gather + pos-0 zeroing + enorm/hnorm + cat, the
|
||||
embed/previous-hidden input norm for a speculative (MTP/eagle) depth layer
|
||||
(the replicated-table analogue of the model-local ``fused_eh_norm``, which
|
||||
takes precomputed embeds).
|
||||
|
||||
Self-contained (no model-local imports) so it can live under ``layers/``.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rms_norm(x, w, eps, HIDDEN_SIZE: tl.constexpr):
|
||||
x = x.to(tl.float32)
|
||||
mean_sq = tl.sum(x * x, axis=0) / HIDDEN_SIZE
|
||||
rrms = tl.rsqrt(mean_sq + eps)
|
||||
w = w.to(tl.float32)
|
||||
return (x * rrms) * w
|
||||
|
||||
|
||||
def make_input_embedding(
|
||||
num_embeddings: int,
|
||||
embedding_dim: int,
|
||||
*,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
quant_config=None,
|
||||
prefix: str = "",
|
||||
tie_word_embeddings: bool = False,
|
||||
) -> VocabParallelEmbedding:
|
||||
"""Input token embedding with an optional replicated escape hatch.
|
||||
|
||||
``VLLM_REPLICATE_EMBED=1`` builds the embedding with ``disable_tp``: the full
|
||||
table lives on every rank and the lookup is a local gather with no mask and
|
||||
no all-reduce, which unlocks the fused gather+norm path. The cost is a full
|
||||
table per rank at TP>1 (no extra memory at TP=1, where vocab-parallel is
|
||||
already unsharded). A replicated, unsharded table cannot be tied to a
|
||||
vocab-parallel ``ParallelLMHead``, so tied word embeddings are rejected at
|
||||
TP>1 (at TP=1 ``disable_tp`` is a no-op and tying still works).
|
||||
"""
|
||||
disable_tp = envs.VLLM_REPLICATE_EMBED
|
||||
if disable_tp and tie_word_embeddings:
|
||||
assert get_tensor_model_parallel_world_size() == 1, (
|
||||
"VLLM_REPLICATE_EMBED is unsupported with tied word embeddings "
|
||||
"(the replicated table cannot tie to a vocab-parallel lm_head)"
|
||||
)
|
||||
return VocabParallelEmbedding(
|
||||
num_embeddings,
|
||||
embedding_dim,
|
||||
params_dtype=params_dtype,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
disable_tp=disable_tp,
|
||||
)
|
||||
|
||||
|
||||
def has_full_vocab_on_rank(embedding: torch.nn.Module) -> bool:
|
||||
"""Whether ``embedding.weight`` is the whole vocab as a plain [V, H] table.
|
||||
|
||||
The fused gather kernels index the table directly, so they need every row
|
||||
on-rank (``disable_tp``, or any TP=1 run) and an unquantized weight.
|
||||
"""
|
||||
return getattr(embedding, "tp_size", 0) == 1 and isinstance(
|
||||
getattr(embedding, "quant_method", None), UnquantizedEmbeddingMethod
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_embed_norm_kernel(
|
||||
ids_ptr, # [T] token ids
|
||||
table_ptr, # [V, H] embedding table (full vocab, replicated on-rank)
|
||||
table_stride_0,
|
||||
out_ptr, # [T, H] gathered embedding (the residual stream)
|
||||
normed_ptr, # [T, H] rmsnorm(out, chain_w) (HAS_NORM only)
|
||||
chain_w_ptr, # [H] next norm weight (HAS_NORM only)
|
||||
eps,
|
||||
H: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
HAS_NORM: tl.constexpr,
|
||||
):
|
||||
tok = tl.program_id(0).to(tl.int64)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
row = tl.load(ids_ptr + tok).to(tl.int64)
|
||||
x = tl.load(table_ptr + row * table_stride_0 + off, mask=mask, other=0.0)
|
||||
tl.store(out_ptr + tok * H + off, x, mask=mask)
|
||||
if HAS_NORM:
|
||||
w = tl.load(chain_w_ptr + off, mask=mask)
|
||||
y = _rms_norm(x, w, eps, H).to(normed_ptr.dtype.element_ty)
|
||||
tl.store(normed_ptr + tok * H + off, y, mask=mask)
|
||||
|
||||
|
||||
# Base model fusion
|
||||
def fused_embed_norm(
|
||||
input_ids: torch.Tensor,
|
||||
embed_table: torch.Tensor,
|
||||
chain_weight: torch.Tensor | None = None,
|
||||
eps: float = 0.0,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused embedding row gather (``embed_table[input_ids]``).
|
||||
|
||||
Requires the full vocab on-rank (replicated embedding). When
|
||||
``chain_weight`` is given, also emits ``rmsnorm(gathered, chain_weight)``
|
||||
(the first decoder layer's ``input_layernorm``) as a second output in the
|
||||
same launch, so the returned pair is ``(residual, normed_input)``. Bit-exact
|
||||
vs a plain gather followed by an ``RMSNorm``.
|
||||
"""
|
||||
assert embed_table.ndim == 2, embed_table.shape
|
||||
ids = input_ids.view(-1)
|
||||
(t,) = ids.shape
|
||||
h = embed_table.shape[1]
|
||||
if chain_weight is not None:
|
||||
assert chain_weight.shape == (h,), (chain_weight.shape, h)
|
||||
out = torch.empty((t, h), dtype=embed_table.dtype, device=embed_table.device)
|
||||
normed = torch.empty_like(out) if chain_weight is not None else None
|
||||
if t > 0:
|
||||
block = triton.next_power_of_2(h)
|
||||
_fused_embed_norm_kernel[(t,)](
|
||||
ids,
|
||||
embed_table,
|
||||
embed_table.stride(0),
|
||||
out,
|
||||
normed if normed is not None else out,
|
||||
chain_weight if chain_weight is not None else embed_table,
|
||||
eps,
|
||||
h,
|
||||
block,
|
||||
HAS_NORM=chain_weight is not None,
|
||||
num_warps=min(32, max(4, block // 512)),
|
||||
)
|
||||
if normed is not None:
|
||||
return out, normed
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_embed_eh_norm_kernel(
|
||||
pos_ptr,
|
||||
ids_ptr, # [T] token ids
|
||||
table_ptr, # [V, H] embedding table (full vocab, replicated on-rank)
|
||||
table_stride,
|
||||
prev_ptr, # [T, H] previous-step hidden
|
||||
prev_stride,
|
||||
enorm_w_ptr,
|
||||
hnorm_w_ptr,
|
||||
eps,
|
||||
out_ptr, # [T, 2H]
|
||||
out_stride,
|
||||
H: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""MTP input fusion with a folded embedding gather: gather
|
||||
``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and
|
||||
RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H])
|
||||
ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm +
|
||||
cat. Requires the full table on-rank (replicated embedding)."""
|
||||
tok = tl.program_id(0)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
|
||||
pos = tl.load(pos_ptr + tok)
|
||||
row = tl.load(ids_ptr + tok).to(tl.int64)
|
||||
e = tl.load(table_ptr + row * table_stride + off, mask=mask, other=0.0)
|
||||
e = tl.where(pos == 0, 0.0, e.to(tl.float32))
|
||||
ew = tl.load(enorm_w_ptr + off, mask=mask)
|
||||
e_normed = _rms_norm(e, ew, eps, H)
|
||||
tl.store(out_ptr + tok * out_stride + off, e_normed, mask=mask)
|
||||
|
||||
p = tl.load(prev_ptr + tok * prev_stride + off, mask=mask, other=0.0)
|
||||
hw = tl.load(hnorm_w_ptr + off, mask=mask)
|
||||
p_normed = _rms_norm(p, hw, eps, H)
|
||||
tl.store(out_ptr + tok * out_stride + H + off, p_normed, mask=mask)
|
||||
|
||||
|
||||
# MTP fusion
|
||||
def fused_embed_eh_norm(
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
embed_table: torch.Tensor,
|
||||
previous_hidden: torch.Tensor,
|
||||
enorm_w: torch.Tensor,
|
||||
hnorm_w: torch.Tensor,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``cat([enorm(masked embed_table[ids]), hnorm(prev_hidden)])`` -> [N, 2H].
|
||||
|
||||
Folds the embedding row gather into the MTP eh-norm launch; requires the full
|
||||
table on-rank (replicated embedding). Bit-exact vs gathering ``embed_table[
|
||||
input_ids]`` and passing it to the model-local ``fused_eh_norm``.
|
||||
"""
|
||||
assert previous_hidden.ndim == 2 and embed_table.ndim == 2
|
||||
n, h = previous_hidden.shape
|
||||
assert positions.shape == (n,) and input_ids.view(-1).shape == (n,)
|
||||
assert embed_table.shape[1] == h, (embed_table.shape, h)
|
||||
assert enorm_w.shape == (h,) and hnorm_w.shape == (h,)
|
||||
out = torch.empty(
|
||||
n, 2 * h, dtype=previous_hidden.dtype, device=previous_hidden.device
|
||||
)
|
||||
_fused_embed_eh_norm_kernel[(n,)](
|
||||
positions,
|
||||
input_ids,
|
||||
embed_table,
|
||||
embed_table.stride(0),
|
||||
previous_hidden,
|
||||
previous_hidden.stride(0),
|
||||
enorm_w,
|
||||
hnorm_w,
|
||||
eps,
|
||||
out,
|
||||
out.stride(0),
|
||||
h,
|
||||
triton.next_power_of_2(h),
|
||||
)
|
||||
return out
|
||||
@@ -1,391 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused MoE Triton kernels."""
|
||||
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import get_device_name_as_file_name
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Valid tiles for the aiter a16w-mix int4 kernels (aiter#4646): stage1 requires
|
||||
# tile_n in {64,128} and tile_k in {128,256} (k_wave=1); stage2 requires tile_n2==128
|
||||
# and tile_k2 in {128,256}. The pre-#4646 defaults (tile_k=512/64, tile_n2=256) are no
|
||||
# longer registered and make the port emit an empty-fragment kernel.
|
||||
_FLYDSL_MOE_DEFAULT_CONFIG = {
|
||||
1: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
2: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
4: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
8: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
16: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
24: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
32: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
48: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
64: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
128: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
256: {"tile_m": 16, "tile_n": 128, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
512: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
1024: {"tile_m": 32, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
2048: {"tile_m": 64, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
4096: {"tile_m": 32, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
8192: {"tile_m": 64, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
}
|
||||
|
||||
# aiter#4646's a16w-mix int4 kernels only register a fixed tile set. Tiles from before
|
||||
# the bump (default config or tuned JSONs) carry tile_k=512/64 and tile_n2=256, which
|
||||
# emit an empty-fragment kernel (IndexError at emit / wrong output). Snap any resolved
|
||||
# tiles to the nearest registered value so a stale tuned config can't crash the kernel.
|
||||
_FLYDSL_INT4_TILE_M = (16, 32, 64, 128)
|
||||
_FLYDSL_INT4_TILE_N1 = (64, 128)
|
||||
_FLYDSL_INT4_TILE_K = (128, 256)
|
||||
|
||||
|
||||
def _snap_flydsl_int4_tiles(tile_m, tile_n, tile_k, tile_n2, tile_k2):
|
||||
def _nearest(v, allowed):
|
||||
return min(allowed, key=lambda a: (abs(a - v), a))
|
||||
|
||||
snapped = (
|
||||
_nearest(tile_m, _FLYDSL_INT4_TILE_M),
|
||||
_nearest(tile_n, _FLYDSL_INT4_TILE_N1),
|
||||
_nearest(tile_k, _FLYDSL_INT4_TILE_K),
|
||||
128, # stage2 N tile is only registered at 128
|
||||
_nearest(tile_k2, _FLYDSL_INT4_TILE_K),
|
||||
)
|
||||
if snapped != (tile_m, tile_n, tile_k, tile_n2, tile_k2):
|
||||
logger.warning_once(
|
||||
"FlyDSL int4 MoE tiles %s are not registered post-aiter#4646; "
|
||||
"using %s. Retune the config for the a16w-mix kernel.",
|
||||
(tile_m, tile_n, tile_k, tile_n2, tile_k2),
|
||||
snapped,
|
||||
scope="local",
|
||||
)
|
||||
return snapped
|
||||
|
||||
|
||||
def moe_sorting(
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
*,
|
||||
num_experts: int,
|
||||
model_dim: int,
|
||||
block_m: int,
|
||||
):
|
||||
# Imported lazily so aiter's import-time sort-backend env vars are read
|
||||
# when this is called, not whenever this module happens to be imported.
|
||||
from aiter.fused_moe import moe_sorting as aiter_moe_sorting
|
||||
|
||||
topk_ids_i32 = topk_ids.to(torch.int32)
|
||||
topk_w_f32 = topk_weights.to(torch.float32)
|
||||
sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids, _moe_buf = (
|
||||
aiter_moe_sorting(
|
||||
topk_ids_i32,
|
||||
topk_w_f32,
|
||||
num_experts,
|
||||
model_dim,
|
||||
torch.float16,
|
||||
block_m,
|
||||
)
|
||||
)
|
||||
if num_valid_ids.numel() > 1:
|
||||
num_valid_ids = num_valid_ids[:1].contiguous()
|
||||
return sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids
|
||||
|
||||
|
||||
def build_routing_buffers(
|
||||
*,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
num_experts: int,
|
||||
model_dim: int,
|
||||
tile_m: int,
|
||||
):
|
||||
res = moe_sorting(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
num_experts=num_experts,
|
||||
model_dim=model_dim,
|
||||
block_m=tile_m,
|
||||
)
|
||||
if res is None:
|
||||
raise RuntimeError(
|
||||
"aiter moe_sorting failed/unavailable; cannot build routing buffers."
|
||||
)
|
||||
sorted_token_ids, sorted_weights, sorted_expert_ids, num_valid_ids = res
|
||||
|
||||
sorted_token_ids = sorted_token_ids.contiguous()
|
||||
sorted_weights = sorted_weights.contiguous()
|
||||
sorted_expert_ids = sorted_expert_ids.contiguous()
|
||||
sorted_size = int(sorted_token_ids.numel())
|
||||
blocks = int(sorted_expert_ids.numel())
|
||||
return (
|
||||
sorted_token_ids,
|
||||
sorted_weights,
|
||||
sorted_expert_ids,
|
||||
num_valid_ids,
|
||||
sorted_size,
|
||||
blocks,
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
def try_get_optimal_config(num_experts, inter_dim):
|
||||
device_name = get_device_name_as_file_name()
|
||||
json_file_name = (
|
||||
f"E={num_experts},N={inter_dim},device_name={device_name},"
|
||||
"dtype=int4_w4a16,backend=flydsl.json"
|
||||
)
|
||||
config_file_path = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name
|
||||
)
|
||||
if os.path.exists(config_file_path):
|
||||
with open(config_file_path) as f:
|
||||
logger.info_once(
|
||||
"Using tuned FlyDSL MoE config from %s",
|
||||
config_file_path,
|
||||
scope="global",
|
||||
)
|
||||
tuned_config = json.load(f)
|
||||
return {int(key): val for key, val in tuned_config.items()}
|
||||
|
||||
logger.warning_once(
|
||||
"Using default FlyDSL MoE config. Performance might be sub-optimal! "
|
||||
"Config file not found at %s",
|
||||
config_file_path,
|
||||
scope="local",
|
||||
)
|
||||
return _FLYDSL_MOE_DEFAULT_CONFIG
|
||||
|
||||
|
||||
def fused_flydsl_moe_impl(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
num_experts: int,
|
||||
inter_dim: int,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
topk: int = 8,
|
||||
group_size: int = 32,
|
||||
doweight_stage1: bool = False,
|
||||
in_dtype: str = "int4_bf16",
|
||||
out_dtype: str = "bf16",
|
||||
scale_is_bf16: bool = True,
|
||||
tile_m: int | None = None,
|
||||
tile_n: int | None = None,
|
||||
tile_k: int | None = None,
|
||||
tile_n2: int | None = None,
|
||||
tile_k2: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Imported lazily -- see the comment in moe_sorting() above.
|
||||
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
|
||||
flydsl_a16w4_gemm1,
|
||||
flydsl_a16w4_gemm2,
|
||||
)
|
||||
|
||||
device = hidden_states.device
|
||||
tokens = hidden_states.shape[0]
|
||||
model_dim = hidden_states.shape[1]
|
||||
|
||||
tuned_config = {}
|
||||
if tile_m and tile_n and tile_k and tile_n2 and tile_k2:
|
||||
tuned_config["tile_m"] = tile_m
|
||||
tuned_config["tile_n"] = tile_n
|
||||
tuned_config["tile_k"] = tile_k
|
||||
tuned_config["tile_n2"] = tile_n2
|
||||
tuned_config["tile_k2"] = tile_k2
|
||||
else:
|
||||
tuned_config = try_get_optimal_config(num_experts, inter_dim)
|
||||
tuned_config = tuned_config[
|
||||
min(tuned_config.keys(), key=lambda x: abs(x - tokens))
|
||||
]
|
||||
out_torch_dtype = torch.bfloat16 if out_dtype == "bf16" else torch.float16
|
||||
|
||||
tile_m = tuned_config["tile_m"]
|
||||
tile_n = tuned_config["tile_n"]
|
||||
tile_k = tuned_config["tile_k"]
|
||||
tile_n2 = tuned_config["tile_n2"]
|
||||
tile_k2 = tuned_config["tile_k2"]
|
||||
tile_m, tile_n, tile_k, tile_n2, tile_k2 = _snap_flydsl_int4_tiles(
|
||||
tile_m, tile_n, tile_k, tile_n2, tile_k2
|
||||
)
|
||||
|
||||
routing = build_routing_buffers(
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
num_experts=num_experts,
|
||||
model_dim=model_dim,
|
||||
tile_m=tile_m,
|
||||
)
|
||||
(
|
||||
sorted_token_ids,
|
||||
sorted_weights,
|
||||
sorted_expert_ids,
|
||||
num_valid_ids,
|
||||
sorted_size,
|
||||
blocks,
|
||||
) = routing
|
||||
|
||||
# aiter#4646 (v0.1.20) removed compile_moe_gemm{1,2} + the flyc.compile pointer ABI
|
||||
# and replaced them with the a16w-mix wrappers, which compile+launch internally
|
||||
# (functools-cached) and take torch tensors directly. w_dtype="int4" is the a16wi4
|
||||
# (W4A16) path. The stage-1 intermediate now lives in sorted
|
||||
# [sorted_size, inter_dim] layout, and topk weighting is applied in stage-2 via
|
||||
# sorted_weights (the port has no stage-1 doweight).
|
||||
sorted_weights_1d = sorted_weights.view(-1).contiguous()
|
||||
empty_scale = torch.empty((0,), device=device, dtype=torch.uint8)
|
||||
cumsum_tensor = num_valid_ids.to(torch.int32).contiguous()
|
||||
m_indices = sorted_token_ids.to(torch.int32).contiguous()
|
||||
|
||||
# stage1: bf16 A x int4 W1 + SiLU -> bf16 sorted intermediate.
|
||||
inter_sorted = torch.empty(
|
||||
(sorted_size, inter_dim), device=device, dtype=torch.bfloat16
|
||||
)
|
||||
w1_scale_u8 = (
|
||||
w1_scale.view(torch.uint8).contiguous().view(-1)
|
||||
if w1_scale is not None
|
||||
else empty_scale
|
||||
)
|
||||
flydsl_a16w4_gemm1(
|
||||
a_bf16=hidden_states.to(torch.bfloat16).contiguous(),
|
||||
w1_u8=w1.view(torch.uint8).contiguous(),
|
||||
w1_scale_u8=w1_scale_u8,
|
||||
sorted_expert_ids=sorted_expert_ids,
|
||||
cumsum_tensor=cumsum_tensor,
|
||||
m_indices=m_indices,
|
||||
inter_sorted_bf16=inter_sorted,
|
||||
n_tokens=tokens,
|
||||
NE=num_experts,
|
||||
D_HIDDEN=model_dim,
|
||||
D_INTER=inter_dim,
|
||||
topk=topk,
|
||||
tile_m=tile_m,
|
||||
tile_n=tile_n,
|
||||
tile_k=tile_k,
|
||||
waves_per_eu=None,
|
||||
act="silu",
|
||||
w_dtype="int4",
|
||||
w_layout="standard",
|
||||
)
|
||||
|
||||
# stage2: bf16 inter x int4 W2 -> weighted reduce into [tokens, model_dim].
|
||||
out_stage2 = torch.zeros((tokens, model_dim), device=device, dtype=out_torch_dtype)
|
||||
w2_scale_u8 = (
|
||||
w2_scale.view(torch.uint8).contiguous().view(-1)
|
||||
if w2_scale is not None
|
||||
else empty_scale
|
||||
)
|
||||
flydsl_a16w4_gemm2(
|
||||
inter_sorted_bf16=inter_sorted,
|
||||
w2_u8=w2.view(torch.uint8).contiguous(),
|
||||
w2_scale_u8=w2_scale_u8,
|
||||
sorted_expert_ids=sorted_expert_ids,
|
||||
cumsum_tensor=cumsum_tensor,
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
sorted_weights=sorted_weights_1d,
|
||||
flat_out=out_stage2.view(-1),
|
||||
M_logical=tokens,
|
||||
max_sorted=sorted_size,
|
||||
NE=num_experts,
|
||||
D_HIDDEN=model_dim,
|
||||
D_INTER=inter_dim,
|
||||
topk=topk,
|
||||
tile_m=tile_m,
|
||||
tile_n=tile_n2,
|
||||
tile_k=tile_k2,
|
||||
waves_per_eu=None,
|
||||
w_dtype="int4",
|
||||
)
|
||||
return out_stage2
|
||||
|
||||
|
||||
def fused_flydsl_moe_impl_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
num_experts: int,
|
||||
inter_dim: int,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
topk: int = 8,
|
||||
group_size: int = 32,
|
||||
doweight_stage1: bool = False,
|
||||
in_dtype: str = "int4_bf16",
|
||||
out_dtype: str = "bf16",
|
||||
scale_is_bf16: bool = True,
|
||||
tile_m: int | None = None,
|
||||
tile_n: int | None = None,
|
||||
tile_k: int | None = None,
|
||||
tile_n2: int | None = None,
|
||||
tile_k2: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(hidden_states)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_flydsl_moe_impl",
|
||||
op_func=fused_flydsl_moe_impl,
|
||||
fake_impl=fused_flydsl_moe_impl_fake,
|
||||
)
|
||||
|
||||
|
||||
def fused_flydsl_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
num_experts: int,
|
||||
inter_dim: int,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
topk: int = 8,
|
||||
group_size: int = 32,
|
||||
doweight_stage1: bool = False,
|
||||
in_dtype: str = "int4_bf16",
|
||||
out_dtype: str = "bf16",
|
||||
scale_is_bf16: bool = True,
|
||||
config: dict | None = None,
|
||||
) -> torch.Tensor:
|
||||
tile_m = None
|
||||
tile_n = None
|
||||
tile_k = None
|
||||
tile_n2 = None
|
||||
tile_k2 = None
|
||||
if config is not None:
|
||||
tile_m = config.get("tile_m")
|
||||
tile_n = config.get("tile_n")
|
||||
tile_k = config.get("tile_k")
|
||||
tile_n2 = config.get("tile_n2")
|
||||
tile_k2 = config.get("tile_k2")
|
||||
return torch.ops.vllm.fused_flydsl_moe_impl(
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
num_experts=num_experts,
|
||||
inter_dim=inter_dim,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
topk=topk,
|
||||
group_size=group_size,
|
||||
doweight_stage1=doweight_stage1,
|
||||
in_dtype=in_dtype,
|
||||
out_dtype=out_dtype,
|
||||
scale_is_bf16=scale_is_bf16,
|
||||
tile_m=tile_m,
|
||||
tile_n=tile_n,
|
||||
tile_k=tile_k,
|
||||
tile_n2=tile_n2,
|
||||
tile_k2=tile_k2,
|
||||
)
|
||||
@@ -1,682 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton input-staging kernels for DeepGEMM MegaMoE.
|
||||
|
||||
Quantize hidden states to fp8 with E8M0 group scales, to NVFP4 with per-token
|
||||
global scales, or copy them as BF16, and repack the routing top-k tensors into
|
||||
the int64/float32 layout that the DeepGEMM MegaMoE kernels consume.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP8_DTYPE,
|
||||
QuantKey,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_E8M0_SCALE_DTYPES = (torch.uint8, torch.float8_e8m0fnu)
|
||||
# Inverse of the NVFP4 range: E2M1 max (6) times E4M3 max (448).
|
||||
_INV_NVFP4_MAX = tl.constexpr(1.0 / 2688.0)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _quantize_fp8_groups(
|
||||
hidden,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
GROUP_K: tl.constexpr,
|
||||
USE_UE8M0: tl.constexpr,
|
||||
):
|
||||
"""Return fp8 values, fp32 group scales and, for UE8M0, the scale exponents."""
|
||||
num_groups: tl.constexpr = BLOCK_K // GROUP_K
|
||||
hidden_groups = tl.reshape(tl.abs(hidden), [BLOCK_M, num_groups, GROUP_K])
|
||||
amax = tl.max(hidden_groups, axis=2)
|
||||
amax = tl.maximum(amax, 1.0e-4)
|
||||
|
||||
scale = amax / 448.0
|
||||
if USE_UE8M0:
|
||||
scale_bits = scale.to(tl.uint32, bitcast=True)
|
||||
scale_exp = ((scale_bits >> 23) & 0xFF) + ((scale_bits & 0x7FFFFF) != 0).to(
|
||||
tl.uint32
|
||||
)
|
||||
scale_exp = tl.minimum(tl.maximum(scale_exp, 1), 254)
|
||||
quant_scale = (scale_exp << 23).to(tl.float32, bitcast=True)
|
||||
else:
|
||||
scale_exp = scale.to(tl.uint32, bitcast=True)
|
||||
quant_scale = scale
|
||||
|
||||
hidden_groups = tl.reshape(hidden, [BLOCK_M, num_groups, GROUP_K])
|
||||
scaled = hidden_groups * (1.0 / quant_scale)[:, :, None]
|
||||
scaled = tl.reshape(scaled, [BLOCK_M, BLOCK_K])
|
||||
return scaled.to(tl.float8e4nv), quant_scale, scale_exp
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_bytes(values, NUM_BYTES: tl.constexpr):
|
||||
"""Pack the last dim of ``[..., NUM_BYTES]`` byte values into int32 words."""
|
||||
offsets = tl.arange(0, NUM_BYTES)
|
||||
return tl.sum(values.to(tl.uint32) << (offsets * 8), axis=-1).to(tl.int32)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _shared_sf_row(token_id, SHARED_BLOCK_M: tl.constexpr):
|
||||
"""Row of ``token_id`` in the shared-expert MN-major UTCCP scale pages."""
|
||||
m_block_id = token_id // SHARED_BLOCK_M
|
||||
m_in_block = token_id % SHARED_BLOCK_M
|
||||
aligned_block_m: tl.constexpr = triton.cdiv(SHARED_BLOCK_M, 128) * 128
|
||||
transposed_m = (
|
||||
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
|
||||
)
|
||||
return m_block_id * aligned_block_m + transposed_m
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_topk(
|
||||
token_id,
|
||||
token_mask,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
topk_weights_out_stride_k: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
) -> None:
|
||||
"""Repack top-k routing as int64 ids and fp32 weights; padding is unrouted."""
|
||||
topk_offsets = tl.arange(0, BLOCK_TOPK)
|
||||
topk_mask = token_mask[:, None] & (topk_offsets[None, :] < top_k)
|
||||
token_is_padding = tl.full((BLOCK_M,), False, tl.int1)
|
||||
if is_padding is not None:
|
||||
token_is_padding = tl.load(
|
||||
is_padding + token_id * is_padding_stride_m,
|
||||
mask=token_mask,
|
||||
other=True,
|
||||
)
|
||||
|
||||
ids = tl.load(
|
||||
topk_ids
|
||||
+ token_id[:, None] * topk_ids_stride_m
|
||||
+ topk_offsets[None, :] * topk_ids_stride_k,
|
||||
mask=topk_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
ids = tl.where(token_is_padding[:, None], -1, ids)
|
||||
tl.store(
|
||||
topk_idx_out
|
||||
+ token_id[:, None] * topk_idx_stride_m
|
||||
+ topk_offsets[None, :] * topk_idx_stride_k,
|
||||
ids,
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
weights = tl.load(
|
||||
topk_weights
|
||||
+ token_id[:, None] * topk_weights_stride_m
|
||||
+ topk_offsets[None, :] * topk_weights_stride_k,
|
||||
mask=topk_mask,
|
||||
other=0.0,
|
||||
)
|
||||
weights = tl.where(token_is_padding[:, None], 0.0, weights)
|
||||
tl.store(
|
||||
topk_weights_out
|
||||
+ token_id[:, None] * topk_weights_out_stride_m
|
||||
+ topk_offsets[None, :] * topk_weights_out_stride_k,
|
||||
weights,
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_tokens"])
|
||||
def _prepare_megamoe_inputs_kernel(
|
||||
hidden_states,
|
||||
x_out,
|
||||
x_sf,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
hidden_stride_m: tl.constexpr,
|
||||
hidden_stride_k: tl.constexpr,
|
||||
x_stride_m: tl.constexpr,
|
||||
x_stride_k: tl.constexpr,
|
||||
x_sf_stride_m: tl.constexpr,
|
||||
x_sf_stride_k: tl.constexpr,
|
||||
shared_x_sf_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_k: tl.constexpr,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
topk_weights_out_stride_k: tl.constexpr,
|
||||
num_tokens,
|
||||
hidden_size: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
GROUP_K: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
SHARED_BLOCK_M: tl.constexpr,
|
||||
QUANTIZE: tl.constexpr,
|
||||
USE_UE8M0: tl.constexpr,
|
||||
) -> None:
|
||||
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
token_mask = token_id < num_tokens
|
||||
k_block_id = tl.program_id(1)
|
||||
|
||||
k_offsets = k_block_id * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
k_mask = token_mask[:, None] & (k_offsets[None, :] < hidden_size)
|
||||
hidden = tl.load(
|
||||
hidden_states
|
||||
+ token_id[:, None] * hidden_stride_m
|
||||
+ k_offsets[None, :] * hidden_stride_k,
|
||||
mask=k_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
if not QUANTIZE:
|
||||
tl.store(
|
||||
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
|
||||
hidden.to(x_out.dtype.element_ty),
|
||||
mask=k_mask,
|
||||
)
|
||||
else:
|
||||
num_groups: tl.constexpr = BLOCK_K // GROUP_K
|
||||
fp8, quant_scale, scale_exp = _quantize_fp8_groups(
|
||||
hidden, BLOCK_M, BLOCK_K, GROUP_K, USE_UE8M0
|
||||
)
|
||||
tl.store(
|
||||
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
|
||||
fp8,
|
||||
mask=k_mask,
|
||||
)
|
||||
|
||||
if USE_UE8M0:
|
||||
packed_scale = _pack_bytes(scale_exp, num_groups)
|
||||
tl.store(
|
||||
x_sf + token_id * x_sf_stride_m + k_block_id * x_sf_stride_k,
|
||||
packed_scale,
|
||||
mask=token_mask,
|
||||
)
|
||||
else:
|
||||
sf_offset = tl.arange(0, num_groups)
|
||||
tl.store(
|
||||
x_sf
|
||||
+ token_id[:, None] * x_sf_stride_m
|
||||
+ k_block_id * x_sf_stride_k
|
||||
+ sf_offset[None, :],
|
||||
quant_scale,
|
||||
mask=token_mask[:, None],
|
||||
)
|
||||
|
||||
# DeepGEMM's SM100 shared-expert TMA loads require the activation scales
|
||||
# in an MN-major layout whose row permutation depends on the MegaMoE
|
||||
# scheduler's runtime BLOCK_M. Write that view while the packed UE8M0 scale
|
||||
# is already resident, avoiding another kernel and temporary tensor.
|
||||
if USE_UE8M0 and shared_x_sf is not None:
|
||||
shared_row = _shared_sf_row(token_id, SHARED_BLOCK_M)
|
||||
tl.store(
|
||||
shared_x_sf
|
||||
+ shared_row * shared_x_sf_stride_m
|
||||
+ k_block_id * shared_x_sf_stride_k,
|
||||
packed_scale,
|
||||
mask=token_mask,
|
||||
)
|
||||
|
||||
if k_block_id == 0:
|
||||
_store_topk(
|
||||
token_id,
|
||||
token_mask,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
topk_ids_stride_m,
|
||||
topk_ids_stride_k,
|
||||
topk_weights_stride_m,
|
||||
topk_weights_stride_k,
|
||||
is_padding_stride_m,
|
||||
topk_idx_stride_m,
|
||||
topk_idx_stride_k,
|
||||
topk_weights_out_stride_m,
|
||||
topk_weights_out_stride_k,
|
||||
top_k,
|
||||
BLOCK_M,
|
||||
BLOCK_TOPK,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_tokens", "l2_activation_scale"])
|
||||
def _prepare_megamoe_nvfp4_inputs_kernel(
|
||||
hidden_states,
|
||||
x_out,
|
||||
x_sf,
|
||||
x_scales,
|
||||
shared_x,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
w2_global_scale,
|
||||
l2_alpha,
|
||||
hidden_stride_m: tl.constexpr,
|
||||
hidden_stride_k: tl.constexpr,
|
||||
x_stride_m: tl.constexpr,
|
||||
x_sf_stride_m: tl.constexpr,
|
||||
shared_x_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_k: tl.constexpr,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
topk_weights_out_stride_k: tl.constexpr,
|
||||
num_tokens,
|
||||
l2_activation_scale,
|
||||
hidden_size: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
num_local_experts: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
BLOCK_E: tl.constexpr,
|
||||
SHARED_BLOCK_M: tl.constexpr,
|
||||
SHARED_MXFP8: tl.constexpr,
|
||||
) -> None:
|
||||
if l2_alpha is not None:
|
||||
experts = tl.arange(0, BLOCK_E)
|
||||
expert_mask = (experts < num_local_experts) & (tl.program_id(0) == 0)
|
||||
alpha = tl.load(w2_global_scale + experts, mask=expert_mask)
|
||||
tl.store(l2_alpha + experts, alpha * l2_activation_scale, mask=expert_mask)
|
||||
|
||||
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
token_mask = token_id < num_tokens
|
||||
k_offsets = tl.arange(0, BLOCK_K)
|
||||
hidden_ptrs = hidden_states + token_id[:, None] * hidden_stride_m
|
||||
|
||||
amax = tl.zeros((BLOCK_M,), tl.float32)
|
||||
for k_block_id in range(hidden_size // BLOCK_K):
|
||||
hidden = tl.load(
|
||||
hidden_ptrs + (k_block_id * BLOCK_K + k_offsets[None, :]) * hidden_stride_k,
|
||||
mask=token_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
amax = tl.maximum(amax, tl.max(tl.abs(hidden), axis=1))
|
||||
x_scale = tl.maximum(amax, 1.0e-12) * _INV_NVFP4_MAX
|
||||
tl.store(x_scales + token_id, x_scale, mask=token_mask)
|
||||
|
||||
for k_block_id in range(hidden_size // BLOCK_K):
|
||||
k_ids = k_block_id * BLOCK_K + k_offsets
|
||||
hidden = tl.load(
|
||||
hidden_ptrs + k_ids[None, :] * hidden_stride_k,
|
||||
mask=token_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# E2M1 values with one E4M3 scale per 16, relative to the token scale.
|
||||
values = tl.reshape(
|
||||
tl.div_rn(hidden.to(tl.float32), x_scale[:, None]),
|
||||
[BLOCK_M, BLOCK_K // 16, 16],
|
||||
)
|
||||
block_scale = tl.max(tl.abs(values), axis=2) * (1.0 / 6.0)
|
||||
block_scale = tl.minimum(tl.maximum(block_scale, 2.0**-9), 448.0)
|
||||
block_scale = block_scale.to(tl.float8e4nv)
|
||||
values = tl.div_rn(values, block_scale.to(tl.float32)[:, :, None])
|
||||
magnitude = tl.abs(values)
|
||||
# Round to nearest even on the grid {0, 0.5, 1, 1.5, 2, 3, 4, 6}.
|
||||
code = (
|
||||
(magnitude > 0.25).to(tl.int32)
|
||||
+ (magnitude >= 0.75).to(tl.int32)
|
||||
+ (magnitude > 1.25).to(tl.int32)
|
||||
+ (magnitude >= 1.75).to(tl.int32)
|
||||
+ (magnitude > 2.5).to(tl.int32)
|
||||
+ (magnitude >= 3.5).to(tl.int32)
|
||||
+ (magnitude > 5.0).to(tl.int32)
|
||||
)
|
||||
code = tl.where((values < 0) & (code != 0), code | 8, code)
|
||||
low, high = tl.split(tl.reshape(code, [BLOCK_M, BLOCK_K // 2, 2]))
|
||||
packed_offsets = k_block_id * (BLOCK_K // 2) + tl.arange(0, BLOCK_K // 2)
|
||||
tl.store(
|
||||
x_out + token_id[:, None] * x_stride_m + packed_offsets[None, :],
|
||||
(low | (high << 4)).to(tl.uint8),
|
||||
mask=token_mask[:, None],
|
||||
)
|
||||
sf_offsets = k_block_id * (BLOCK_K // 64) + tl.arange(0, BLOCK_K // 64)
|
||||
sf_bytes = tl.reshape(
|
||||
block_scale.to(tl.uint8, bitcast=True), [BLOCK_M, BLOCK_K // 64, 4]
|
||||
)
|
||||
tl.store(
|
||||
x_sf + token_id[:, None] * x_sf_stride_m + sf_offsets[None, :],
|
||||
_pack_bytes(sf_bytes, 4),
|
||||
mask=token_mask[:, None],
|
||||
)
|
||||
|
||||
if shared_x is not None:
|
||||
shared_ptrs = shared_x + token_id[:, None] * shared_x_stride_m
|
||||
if SHARED_MXFP8:
|
||||
fp8, _, scale_exp = _quantize_fp8_groups(
|
||||
hidden.to(tl.float32), BLOCK_M, BLOCK_K, 32, True
|
||||
)
|
||||
tl.store(shared_ptrs + k_ids[None, :], fp8, mask=token_mask[:, None])
|
||||
tl.store(
|
||||
shared_x_sf
|
||||
+ _shared_sf_row(token_id, SHARED_BLOCK_M) * shared_x_sf_stride_m
|
||||
+ k_block_id * shared_x_sf_stride_k,
|
||||
_pack_bytes(scale_exp, BLOCK_K // 32),
|
||||
mask=token_mask,
|
||||
)
|
||||
else:
|
||||
tl.store(shared_ptrs + k_ids[None, :], hidden, mask=token_mask[:, None])
|
||||
|
||||
_store_topk(
|
||||
token_id,
|
||||
token_mask,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
topk_ids_stride_m,
|
||||
topk_ids_stride_k,
|
||||
topk_weights_stride_m,
|
||||
topk_weights_stride_k,
|
||||
is_padding_stride_m,
|
||||
topk_idx_stride_m,
|
||||
topk_idx_stride_k,
|
||||
topk_weights_out_stride_m,
|
||||
topk_weights_out_stride_k,
|
||||
top_k,
|
||||
BLOCK_M,
|
||||
BLOCK_TOPK,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_hidden_quant(hidden_quant: QuantKey, block_k: int) -> tuple[int, bool]:
|
||||
"""Map a hidden-state ``QuantKey`` to the kernel's ``(GROUP_K, USE_UE8M0)``."""
|
||||
scale = hidden_quant.scale
|
||||
group_shape = scale.group_shape
|
||||
if (
|
||||
hidden_quant.dtype != FP8_DTYPE
|
||||
or not hidden_quant.symmetric
|
||||
or hidden_quant.scale2 is not None
|
||||
or scale.static
|
||||
or not group_shape.is_per_group()
|
||||
or block_k % group_shape.col != 0
|
||||
or scale.dtype not in (*_E8M0_SCALE_DTYPES, torch.float32)
|
||||
):
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires symmetric dynamic fp8 "
|
||||
f"per-group quantization with a group size dividing {block_k} and "
|
||||
f"E8M0 or fp32 scales, got {hidden_quant}."
|
||||
)
|
||||
return group_shape.col, scale.dtype in _E8M0_SCALE_DTYPES
|
||||
|
||||
|
||||
def prepare_megamoe_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
x_sf: torch.Tensor | None,
|
||||
topk_idx_out: torch.Tensor,
|
||||
topk_weights_out: torch.Tensor,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
shared_x_sf: torch.Tensor | None = None,
|
||||
shared_block_m: int | None = None,
|
||||
hidden_quant: QuantKey | None = kMxfp8Dynamic,
|
||||
) -> None:
|
||||
"""Quantize hidden states and repack top-k routing for DeepGEMM MegaMoE.
|
||||
|
||||
Args:
|
||||
hidden_states: Input activations of shape ``[num_tokens, hidden_dim]``.
|
||||
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
|
||||
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
|
||||
x: Output buffer for the dispatched hidden states.
|
||||
x_sf: Output buffer for the hidden-state scale factors; ``None`` when
|
||||
``hidden_quant`` is ``None``.
|
||||
topk_idx_out: Output buffer for the repacked top-k expert ids.
|
||||
topk_weights_out: Output buffer for the repacked top-k weights.
|
||||
is_padding: Optional per-token mask; padded tokens are not routed.
|
||||
shared_x_sf: Optional output buffer for shared-expert scale factors.
|
||||
Must be given together with ``shared_block_m``.
|
||||
shared_block_m: Block M used to lay out ``shared_x_sf``.
|
||||
hidden_quant: Hidden-state quantization scheme. E8M0 scales are packed
|
||||
four per int32 into ``x_sf``; fp32 scales are stored directly.
|
||||
``None`` copies the hidden states into ``x`` unquantized.
|
||||
|
||||
"""
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
if num_tokens == 0:
|
||||
return
|
||||
if hidden_size % 128 != 0:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires hidden_size to be a multiple of 128."
|
||||
)
|
||||
block_k = 128
|
||||
quantize = hidden_quant is not None
|
||||
if quantize != (x_sf is not None):
|
||||
raise ValueError("MegaMoE input staging needs x_sf iff hidden_quant is set.")
|
||||
group_k, use_ue8m0 = (
|
||||
(block_k, False)
|
||||
if hidden_quant is None
|
||||
else _resolve_hidden_quant(hidden_quant, block_k)
|
||||
)
|
||||
top_k = topk_ids.shape[1]
|
||||
if topk_weights.shape != topk_ids.shape:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires topk_weights and "
|
||||
"topk_ids to have the same shape."
|
||||
)
|
||||
if (shared_x_sf is None) != (shared_block_m is None):
|
||||
raise ValueError(
|
||||
"MegaMoE shared input staging requires both shared_x_sf and shared_block_m."
|
||||
)
|
||||
if shared_x_sf is not None and not use_ue8m0:
|
||||
raise ValueError(
|
||||
"MegaMoE shared input staging currently requires "
|
||||
"UE8M0-packed hidden scales."
|
||||
)
|
||||
if shared_x_sf is not None:
|
||||
assert shared_block_m is not None
|
||||
if shared_block_m <= 0:
|
||||
raise ValueError("MegaMoE shared_block_m must be positive.")
|
||||
expected_sf_k = hidden_size // 128
|
||||
if shared_x_sf.ndim != 2 or shared_x_sf.shape[1] != expected_sf_k:
|
||||
raise ValueError(
|
||||
"MegaMoE shared_x_sf must have shape "
|
||||
f"(*, {expected_sf_k}), got {tuple(shared_x_sf.shape)}."
|
||||
)
|
||||
aligned_block_m = triton.cdiv(shared_block_m, 128) * 128
|
||||
required_rows = triton.cdiv(num_tokens, shared_block_m) * aligned_block_m
|
||||
if shared_x_sf.shape[0] < required_rows:
|
||||
raise ValueError(
|
||||
"MegaMoE shared_x_sf has insufficient rows: requires "
|
||||
f"{required_rows}, got {shared_x_sf.shape[0]}."
|
||||
)
|
||||
|
||||
# On GB200, eight-row tiles win from 64 tokens; keep smaller batches untiled.
|
||||
block_m = 8 if num_tokens >= 64 else 1
|
||||
grid = (triton.cdiv(num_tokens, block_m), triton.cdiv(hidden_size, block_k))
|
||||
block_topk = triton.next_power_of_2(top_k)
|
||||
padding_stride_m = is_padding.stride(0) if is_padding is not None else 0
|
||||
_prepare_megamoe_inputs_kernel[grid](
|
||||
hidden_states,
|
||||
x,
|
||||
x_sf,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
hidden_states.stride(0),
|
||||
hidden_states.stride(1),
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
x_sf.stride(0) if x_sf is not None else 0,
|
||||
x_sf.stride(1) if x_sf is not None else 0,
|
||||
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
|
||||
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
|
||||
topk_ids.stride(0),
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
padding_stride_m,
|
||||
topk_idx_out.stride(0),
|
||||
topk_idx_out.stride(1),
|
||||
topk_weights_out.stride(0),
|
||||
topk_weights_out.stride(1),
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
top_k,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=block_k,
|
||||
GROUP_K=group_k,
|
||||
BLOCK_TOPK=block_topk,
|
||||
SHARED_BLOCK_M=shared_block_m or 1,
|
||||
QUANTIZE=quantize,
|
||||
USE_UE8M0=use_ue8m0,
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
def prepare_megamoe_nvfp4_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
x_sf: torch.Tensor,
|
||||
x_scales: torch.Tensor,
|
||||
topk_idx_out: torch.Tensor,
|
||||
topk_weights_out: torch.Tensor,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
shared_x: torch.Tensor | None = None,
|
||||
shared_x_sf: torch.Tensor | None = None,
|
||||
shared_block_m: int | None = None,
|
||||
w2_global_scale: torch.Tensor | None = None,
|
||||
l2_alpha: torch.Tensor | None = None,
|
||||
l2_activation_scale: float = 1.0,
|
||||
) -> None:
|
||||
"""Stage DeepGEMM NVFP4 MegaMoE inputs (``kNvfp4DynamicToken``).
|
||||
|
||||
Each token is scaled by its own fp32 ``x_scales`` (row amax / 2688) and
|
||||
quantized to E2M1 with one E4M3 scale per 16 values.
|
||||
|
||||
Args:
|
||||
hidden_states: BF16 activations of shape ``[num_tokens, hidden_dim]``.
|
||||
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
|
||||
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
|
||||
x: Packed E2M1 output, ``[num_tokens, hidden_dim // 2]``.
|
||||
x_sf: E4M3 scales packed four per int32, ``[num_tokens, hidden_dim // 64]``.
|
||||
x_scales: Per-token fp32 dequantization scales, ``[num_tokens]``.
|
||||
topk_idx_out: Output buffer for the repacked top-k expert ids.
|
||||
topk_weights_out: Output buffer for the repacked top-k weights.
|
||||
is_padding: Optional per-token mask; padded tokens are not routed.
|
||||
shared_x: Optional shared-expert input, fp8 (MXFP8 with ``shared_x_sf``)
|
||||
or a BF16 copy of the hidden states.
|
||||
shared_x_sf: UE8M0 scales of an fp8 ``shared_x`` in the shared-expert
|
||||
TMA layout for ``shared_block_m``.
|
||||
shared_block_m: Block M used to lay out ``shared_x_sf``.
|
||||
w2_global_scale: Per-local-expert L2 weight global scales.
|
||||
l2_alpha: Output ``w2_global_scale * l2_activation_scale``, the L2
|
||||
dequantization scale of each local expert. Written even without
|
||||
local tokens, as other ranks' tokens still reach these experts.
|
||||
l2_activation_scale: Global scale the kernel quantizes L2 inputs with.
|
||||
|
||||
"""
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
if hidden_size % 128 != 0:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires hidden_size to be a multiple of 128."
|
||||
)
|
||||
if topk_weights.shape != topk_ids.shape:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires topk_weights and "
|
||||
"topk_ids to have the same shape."
|
||||
)
|
||||
shared_mxfp8 = shared_x is not None and shared_x.dtype == FP8_DTYPE
|
||||
if shared_mxfp8 != (shared_x_sf is not None) or (shared_x_sf is None) != (
|
||||
shared_block_m is None
|
||||
):
|
||||
raise ValueError(
|
||||
"MegaMoE NVFP4 staging needs shared_x_sf and shared_block_m exactly "
|
||||
"for an fp8 shared_x."
|
||||
)
|
||||
if (w2_global_scale is None) != (l2_alpha is None):
|
||||
raise ValueError("MegaMoE NVFP4 staging needs w2_global_scale with l2_alpha.")
|
||||
for name, tensor in (("x", x), ("x_sf", x_sf), ("shared_x", shared_x)):
|
||||
if tensor is not None and tensor.stride(-1) != 1:
|
||||
raise ValueError(f"MegaMoE NVFP4 staging requires a row-major {name}.")
|
||||
|
||||
block_m = 8 if num_tokens >= 64 else 1
|
||||
top_k = topk_ids.shape[1]
|
||||
num_local_experts = 0 if l2_alpha is None else l2_alpha.numel()
|
||||
# One program even without tokens, to write l2_alpha.
|
||||
grid = (max(triton.cdiv(num_tokens, block_m), 1),)
|
||||
_prepare_megamoe_nvfp4_inputs_kernel[grid](
|
||||
hidden_states,
|
||||
x.view(torch.uint8),
|
||||
x_sf,
|
||||
x_scales,
|
||||
shared_x,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
w2_global_scale,
|
||||
l2_alpha,
|
||||
hidden_states.stride(0),
|
||||
hidden_states.stride(1),
|
||||
x.stride(0),
|
||||
x_sf.stride(0),
|
||||
shared_x.stride(0) if shared_x is not None else 0,
|
||||
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
|
||||
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
|
||||
topk_ids.stride(0),
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
is_padding.stride(0) if is_padding is not None else 0,
|
||||
topk_idx_out.stride(0),
|
||||
topk_idx_out.stride(1),
|
||||
topk_weights_out.stride(0),
|
||||
topk_weights_out.stride(1),
|
||||
num_tokens,
|
||||
l2_activation_scale,
|
||||
hidden_size,
|
||||
top_k,
|
||||
num_local_experts,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=128,
|
||||
BLOCK_TOPK=triton.next_power_of_2(top_k),
|
||||
BLOCK_E=triton.next_power_of_2(max(num_local_experts, 1)),
|
||||
SHARED_BLOCK_M=shared_block_m or 1,
|
||||
SHARED_MXFP8=shared_mxfp8,
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -1,76 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_FP8_MIN, _FP8_MAX = get_fp8_min_max()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _relu_squared_static_fp8_quant_kernel(
|
||||
x_ptr,
|
||||
scale_ptr,
|
||||
output_ptr,
|
||||
num_elements,
|
||||
FP8_MIN: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < num_elements
|
||||
x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
|
||||
input_is_nan = x != x
|
||||
relu = tl.maximum(x, 0.0)
|
||||
# Preserve the standalone ReLU2 kernel's BF16 output boundary.
|
||||
activated = (relu * relu).to(tl.bfloat16).to(tl.float32)
|
||||
# Match O2's Inductor-compiled QuantFP8 reciprocal-then-multiply.
|
||||
quantized = activated * (1.0 / tl.load(scale_ptr))
|
||||
quantized = tl.maximum(tl.minimum(quantized, FP8_MAX), FP8_MIN)
|
||||
# Match the O2 native chain by preserving NaN through the FP8 conversion.
|
||||
quantized = tl.where(input_is_nan, x, quantized)
|
||||
tl.store(output_ptr + offsets, quantized, mask=mask)
|
||||
|
||||
|
||||
def relu_squared_static_fp8_quant(
|
||||
x: torch.Tensor, linear: LinearBase
|
||||
) -> QuantizedActivation:
|
||||
"""BF16 ReLU2 followed by static per-tensor FP8 quantization."""
|
||||
assert x.dtype == torch.bfloat16
|
||||
assert x.is_contiguous()
|
||||
scale = linear.input_scale
|
||||
assert scale.dtype == torch.float32
|
||||
assert x.device == scale.device
|
||||
assert scale.numel() == 1
|
||||
|
||||
output = torch.empty_like(x, dtype=current_platform.fp8_dtype())
|
||||
if x.numel() != 0:
|
||||
block_size = min(triton.next_power_of_2(x.shape[-1]), 2048)
|
||||
num_warps = min(max(block_size // 256, 1), 4)
|
||||
grid = lambda meta: (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),)
|
||||
_relu_squared_static_fp8_quant_kernel[grid](
|
||||
x,
|
||||
scale,
|
||||
output,
|
||||
x.numel(),
|
||||
FP8_MIN=_FP8_MIN,
|
||||
FP8_MAX=_FP8_MAX,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
|
||||
return QuantizedActivation(
|
||||
data=output,
|
||||
scale=scale,
|
||||
orig_dtype=x.dtype,
|
||||
orig_shape=x.shape,
|
||||
quant_key=kFp8StaticTensorSym,
|
||||
)
|
||||
@@ -1,405 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Top-k kernels for the DSA sparse attention indexer."""
|
||||
|
||||
import functools
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
|
||||
# cooperative_topk's hard row limit: one cluster wave covers at most this many
|
||||
# rows. "auto" uses cooperative_topk for every batch within the limit and falls
|
||||
# back to persistent_topk past it; explicit ``cooperative`` requests are
|
||||
# validated against the same bound.
|
||||
AUTO_COOPERATIVE_MAX_ROWS = 64
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DeepSelect (vllm._deepselect_C)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Matches the -1 fill convention used for topk_indices_buffer elsewhere.
|
||||
IDX_OOB_FILL_VALUE = -1
|
||||
|
||||
try:
|
||||
import vllm._deepselect_C # noqa: F401 (registers torch.ops.deep_select)
|
||||
|
||||
_HAS_DEEP_SELECT_C = True
|
||||
except ImportError:
|
||||
_HAS_DEEP_SELECT_C = False
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def get_deep_select_stride_requirement() -> tuple[int, int]:
|
||||
"""Stride alignment requirement (input, output) in bytes."""
|
||||
return torch.ops.deep_select.get_alignment_requirement()
|
||||
|
||||
|
||||
def is_deep_select_supported(input: torch.Tensor, topk: int) -> bool:
|
||||
"""Whether the kernel accepts this input (dtype/stride/topk constraints)."""
|
||||
return (
|
||||
topk <= 4096
|
||||
and input.shape[1] < 2**23
|
||||
and input.dtype in (torch.float32, torch.bfloat16)
|
||||
and input.stride(1) == 1
|
||||
and input.stride(0)
|
||||
* input.element_size()
|
||||
% get_deep_select_stride_requirement()[0]
|
||||
== 0
|
||||
)
|
||||
|
||||
|
||||
def _get_empty_and_aligned_tensor(
|
||||
dim0: int, dim1: int, device: torch.device, dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""Tensor with shape (dim0, dim1) whose stride(0) is 32B-aligned."""
|
||||
output_stride_requirement = get_deep_select_stride_requirement()[1] // (
|
||||
dtype.itemsize
|
||||
)
|
||||
assert output_stride_requirement > 0
|
||||
dim1_rounded = (
|
||||
(dim1 + output_stride_requirement - 1) // output_stride_requirement
|
||||
) * output_stride_requirement
|
||||
return torch.empty((dim0, dim1_rounded), device=device, dtype=dtype)[:, :dim1]
|
||||
|
||||
|
||||
def deep_select_topk(
|
||||
input: torch.Tensor,
|
||||
topk: int,
|
||||
end: torch.Tensor | None = None,
|
||||
output_idx: torch.Tensor | None = None,
|
||||
indices_dtype: torch.dtype = torch.int32,
|
||||
) -> torch.Tensor:
|
||||
"""Select the top-k indices per row of `input` with DeepSelect.
|
||||
|
||||
Args:
|
||||
input: (num_rows, vocab_size), bf16 or fp32. stride(1) must be 1 and
|
||||
stride(0) must be 1024B-aligned.
|
||||
topk: Number of elements to select per row; must be <= 4096.
|
||||
end: Optional (num_rows,) int32 tensor with the exclusive right
|
||||
boundary of each row. Rows with `end[i] < topk` get their
|
||||
remaining indices filled with -1.
|
||||
output_idx: Optional preallocated (num_rows, topk) output tensor whose
|
||||
stride(0) is 32B-aligned (e.g. a slice of a wider buffer).
|
||||
indices_dtype: Output dtype when `output_idx` is not provided.
|
||||
|
||||
Returns:
|
||||
The (num_rows, topk) indices tensor.
|
||||
|
||||
"""
|
||||
assert input.dim() == 2 and input.stride(1) == 1
|
||||
assert (
|
||||
input.stride(0) * input.element_size() % get_deep_select_stride_requirement()[0]
|
||||
== 0
|
||||
)
|
||||
|
||||
num_rows = input.shape[0]
|
||||
if output_idx is None:
|
||||
output_idx = _get_empty_and_aligned_tensor(
|
||||
num_rows, topk, input.device, indices_dtype
|
||||
)
|
||||
else:
|
||||
assert output_idx.dtype == indices_dtype
|
||||
assert output_idx.shape[0] >= num_rows and output_idx.shape[1] >= topk
|
||||
|
||||
torch.ops.deep_select.topk(
|
||||
input,
|
||||
topk,
|
||||
None, # begin is not supported
|
||||
end,
|
||||
False, # sorted_value
|
||||
False, # sorted_index
|
||||
None, # output_value
|
||||
output_idx,
|
||||
None, # output_idx_offset
|
||||
IDX_OOB_FILL_VALUE,
|
||||
float("-inf"), # value_oob_fill_value
|
||||
False, # return_value
|
||||
False, # abort_when_nan_found: dummy/capture passes can feed NaN
|
||||
# logits from uninitialized KV cache; the reference topk kernels do
|
||||
# not trap on NaN, and trapping would abort CUDA graph capture.
|
||||
)
|
||||
return output_idx
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backend selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_indexer_topk(backend: str) -> "SparseIndexerTopk":
|
||||
return SparseIndexerTopk(backend)
|
||||
|
||||
|
||||
class SparseIndexerTopk(torch.nn.Module):
|
||||
"""The sparse indexer's decode top-k stage.
|
||||
|
||||
Selects among the available top-k kernels (see
|
||||
kernel_config.sparse_indexer_topk_backend) and runs the chosen one.
|
||||
"""
|
||||
|
||||
def __init__(self, backend: str | None = None) -> None:
|
||||
super().__init__()
|
||||
if backend is None:
|
||||
backend = (
|
||||
get_current_vllm_config().kernel_config.sparse_indexer_topk_backend
|
||||
)
|
||||
self._backend = backend
|
||||
self._is_cuda = current_platform.is_cuda()
|
||||
self._has_deep_select = self._is_cuda and (
|
||||
current_platform.is_device_capability_family(100)
|
||||
)
|
||||
self._has_flashinfer_topk = has_flashinfer()
|
||||
self._cooperative_capable = self._is_cuda and (
|
||||
current_platform.has_device_capability(90)
|
||||
and not current_platform.is_device_capability_family(120)
|
||||
)
|
||||
self._is_rocm = current_platform.is_rocm()
|
||||
self._aiter_enabled = False
|
||||
self._aiter_capable = False
|
||||
if self._is_rocm:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
self._aiter_enabled = bool(rocm_aiter_ops.is_enabled())
|
||||
self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled())
|
||||
|
||||
def resolve_backend(
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> str:
|
||||
"""Resolve the decode top-k implementation from the configured
|
||||
backend ("auto" = the pre-existing chain, or a validated explicit
|
||||
value)."""
|
||||
if self._backend == "auto":
|
||||
return self._resolve_auto(logits, topk_tokens, num_rows)
|
||||
|
||||
failures: list[str] = []
|
||||
if self._backend == "cooperative":
|
||||
failures = self._cooperative_constraints(logits, topk_tokens, num_rows)
|
||||
elif self._backend == "persistent":
|
||||
if not self._is_cuda:
|
||||
failures.append("requires a CUDA platform")
|
||||
if topk_tokens not in (512, 1024, 2048):
|
||||
failures.append(
|
||||
f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
|
||||
)
|
||||
elif self._backend == "deep_select":
|
||||
if not self._is_cuda:
|
||||
failures.append("requires a CUDA platform")
|
||||
elif not self._has_deep_select:
|
||||
failures.append("requires SM100a/SM103a (10.x device family)")
|
||||
elif not _HAS_DEEP_SELECT_C:
|
||||
failures.append("vllm._deepselect_C could not be imported")
|
||||
elif not is_deep_select_supported(logits, topk_tokens):
|
||||
failures.append(
|
||||
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
|
||||
f" stride={logits.stride()}, topk={topk_tokens}"
|
||||
)
|
||||
elif self._backend == "aiter":
|
||||
if not self._is_rocm:
|
||||
failures.append("requires a ROCm platform")
|
||||
elif not self._aiter_enabled:
|
||||
failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
|
||||
elif not self._aiter_capable:
|
||||
failures.append(
|
||||
"AITER has no tuned indexer top-k kernels for this GPU arch"
|
||||
)
|
||||
if topk_tokens not in (512, 1024, 2048, 4096):
|
||||
failures.append(
|
||||
f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
|
||||
)
|
||||
elif self._backend == "flashinfer":
|
||||
if not self._is_cuda:
|
||||
failures.append("requires a CUDA platform")
|
||||
if not self._has_flashinfer_topk:
|
||||
failures.append(
|
||||
"flashinfer.topk.top_k_ragged_transform is not importable"
|
||||
)
|
||||
if logits.dtype != torch.float32 or logits.stride(1) != 1:
|
||||
failures.append(
|
||||
f"requires fp32 logits with stride(1) == 1, got "
|
||||
f"dtype={logits.dtype}, stride={logits.stride()}"
|
||||
)
|
||||
if failures:
|
||||
raise RuntimeError(
|
||||
f"sparse_indexer_topk_backend='{self._backend}' was requested, but: "
|
||||
+ "; ".join(failures)
|
||||
)
|
||||
return self._backend
|
||||
|
||||
def _resolve_auto(
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> str:
|
||||
"""The priority chain: cooperative -> persistent -> per_row.
|
||||
aiter/deep_select/flashinfer/torch are opt-in only.
|
||||
|
||||
cooperative_topk is preferred whenever it is applicable, i.e. within
|
||||
its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
|
||||
persistent_topk.
|
||||
|
||||
aiter not auto-resolved as it does not outperform the per-row on every shape,
|
||||
hence we delegate to the caller to decide when to use it.
|
||||
"""
|
||||
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
|
||||
return "cooperative"
|
||||
if self._is_cuda and topk_tokens in (512, 1024, 2048):
|
||||
return "persistent"
|
||||
return "per_row"
|
||||
|
||||
def _cooperative_constraints(
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> list[str]:
|
||||
"""Unmet constraints of cooperative_topk (empty when applicable)."""
|
||||
failures = []
|
||||
if not self._is_cuda:
|
||||
failures.append("requires a CUDA platform")
|
||||
if topk_tokens not in (512, 1024, 2048):
|
||||
failures.append(
|
||||
f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
|
||||
)
|
||||
if num_rows > AUTO_COOPERATIVE_MAX_ROWS:
|
||||
failures.append(
|
||||
f"num_rows must be <= {AUTO_COOPERATIVE_MAX_ROWS}, got {num_rows}"
|
||||
)
|
||||
if logits.stride(0) % 4 != 0:
|
||||
failures.append(
|
||||
f"logits.stride(0) must be divisible by 4, got {logits.stride(0)}"
|
||||
)
|
||||
if self._is_cuda and not self._cooperative_capable:
|
||||
failures.append("requires SM90+ and is not supported on the SM12x family")
|
||||
return failures
|
||||
|
||||
@staticmethod
|
||||
def _row_ends(seq_lens: torch.Tensor, next_n: int, num_rows: int) -> torch.Tensor:
|
||||
"""Per-row exclusive end offsets (int32, (num_rows,)) for top-k
|
||||
kernels that take ragged lengths (DeepSelect, FlashInfer, torch
|
||||
reference).
|
||||
|
||||
seq_lens is (B, next_n) per-row effective lens for native spec decode
|
||||
and (B, 1) otherwise, in which case per-row lens are derived the same
|
||||
way as the other decode top-k kernels.
|
||||
"""
|
||||
if seq_lens.numel() == num_rows:
|
||||
row_ends = seq_lens.reshape(-1)
|
||||
else:
|
||||
next_n_offsets = torch.arange(
|
||||
next_n, dtype=torch.int32, device=seq_lens.device
|
||||
)
|
||||
row_ends = (
|
||||
(seq_lens.reshape(-1, 1) - next_n + 1 + next_n_offsets)
|
||||
.clamp_(min=0)
|
||||
.reshape(-1)
|
||||
)
|
||||
assert row_ends.dtype == torch.int32
|
||||
return row_ends
|
||||
|
||||
def forward(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
next_n: int,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
max_seq_len: int,
|
||||
) -> None:
|
||||
"""Run the resolved decode top-k implementation, writing into
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
|
||||
"""
|
||||
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
|
||||
if backend == "deep_select":
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
deep_select_topk(logits, topk_tokens, end=row_ends, output_idx=topk_indices)
|
||||
elif backend == "cooperative":
|
||||
(topk_workspace,) = current_workspace_manager().get_simultaneous(
|
||||
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
|
||||
)
|
||||
torch.ops._C.cooperative_topk(
|
||||
logits,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
topk_workspace,
|
||||
topk_tokens,
|
||||
max_seq_len,
|
||||
)
|
||||
elif backend == "persistent":
|
||||
(topk_workspace,) = current_workspace_manager().get_simultaneous(
|
||||
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
|
||||
)
|
||||
# persistent_topk's last argument is the host-side "max seq_len
|
||||
# across rows" bound (it gates the sampled_topk path and clamps
|
||||
# per-row lengths), not the padded row width: logits is sized to
|
||||
# max_model_len, so passing its width would incorrectly enable the
|
||||
# long-row kernels on short-context batches.
|
||||
torch.ops._C.persistent_topk(
|
||||
logits,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
topk_workspace,
|
||||
topk_tokens,
|
||||
max_seq_len,
|
||||
)
|
||||
elif backend == "flashinfer":
|
||||
# Deferred: importing flashinfer initializes CUDA at import time.
|
||||
from flashinfer.topk import top_k_ragged_transform
|
||||
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
offsets = torch.zeros(
|
||||
logits.shape[0], dtype=torch.int32, device=logits.device
|
||||
)
|
||||
# top_k_ragged_transform selects within [0, row_ends[i]) per
|
||||
# row and -1-fills past the row length.
|
||||
indices = top_k_ragged_transform(logits, offsets, row_ends, topk_tokens)
|
||||
topk_indices.copy_(indices)
|
||||
elif backend == "torch":
|
||||
# Debug reference: mask everything past each row's end, then topk.
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
cols = torch.arange(logits.shape[1], device=logits.device)
|
||||
masked = logits.masked_fill(
|
||||
cols.unsqueeze(0) >= row_ends.unsqueeze(1), float("-inf")
|
||||
)
|
||||
indices = masked.topk(topk_tokens, dim=-1).indices
|
||||
in_range = torch.arange(topk_tokens, device=logits.device).unsqueeze(
|
||||
0
|
||||
) < row_ends.unsqueeze(1)
|
||||
indices = torch.where(in_range, indices, -1)
|
||||
topk_indices.copy_(indices)
|
||||
elif backend == "aiter":
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# AITER's decode kernel only implements the 1D/next_n == 1 form,
|
||||
# which is equivalent once the ragged ends are expanded.
|
||||
rocm_aiter_ops.indexer_top_k_decode(
|
||||
logits,
|
||||
1,
|
||||
self._row_ends(seq_lens, next_n, logits.shape[0]),
|
||||
topk_indices,
|
||||
topk_tokens,
|
||||
)
|
||||
else:
|
||||
assert backend == "per_row", f"unknown topk backend: {backend}"
|
||||
self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)
|
||||
|
||||
@staticmethod
|
||||
def _per_row(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
next_n: int,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
) -> None:
|
||||
ops.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
logits.shape[0],
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
@@ -1,741 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_diag_kernel(
|
||||
Q,
|
||||
K,
|
||||
V,
|
||||
Out,
|
||||
S,
|
||||
b: tl.constexpr,
|
||||
h: tl.constexpr,
|
||||
n,
|
||||
d: tl.constexpr,
|
||||
e: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
NUM_BLOCK,
|
||||
CBLOCK: tl.constexpr,
|
||||
):
|
||||
# This kernel computes the diagonal blocks of the attention matrix
|
||||
# Each diagonal block represents attention
|
||||
# where queries attend to keys in the same block
|
||||
off = tl.program_id(0)
|
||||
off_bh = off // NUM_BLOCK # batch-head index
|
||||
off_block = off % NUM_BLOCK # block index within the sequence
|
||||
off_cblock = tl.program_id(1) # sub-block index within a block
|
||||
|
||||
off_h = off_bh % h # head index
|
||||
|
||||
# Calculate base offsets for the current batch and head
|
||||
qk_offset = off_bh * n * d
|
||||
v_offset = off_bh * n * e
|
||||
o_offset = off_bh * n * e
|
||||
|
||||
# Calculate offsets for the current block
|
||||
block_offset = off_block * BLOCK
|
||||
qk_block_offset = block_offset * d
|
||||
v_block_offset = block_offset * e
|
||||
o_block_offset = block_offset * e
|
||||
|
||||
# Calculate offsets for the current sub-block
|
||||
cblock_offset = off_cblock * CBLOCK
|
||||
q_cblock_offset = cblock_offset * d
|
||||
o_cblock_offset = cblock_offset * e
|
||||
|
||||
# Calculate pointers to the query, key, value, and output tensors
|
||||
Q_block_ptr = (
|
||||
Q
|
||||
+ qk_offset
|
||||
+ qk_block_offset
|
||||
+ q_cblock_offset
|
||||
+ tl.arange(0, CBLOCK)[:, None] * d
|
||||
+ tl.arange(0, d)[None, :]
|
||||
)
|
||||
K_trans_block_ptr = (
|
||||
K
|
||||
+ qk_offset
|
||||
+ qk_block_offset
|
||||
+ tl.arange(0, CBLOCK)[None, :] * d
|
||||
+ tl.arange(0, d)[:, None]
|
||||
)
|
||||
V_block_ptr = (
|
||||
V
|
||||
+ v_offset
|
||||
+ v_block_offset
|
||||
+ tl.arange(0, CBLOCK)[:, None] * e
|
||||
+ tl.arange(0, e)[None, :]
|
||||
)
|
||||
O_block_ptr = (
|
||||
Out
|
||||
+ o_offset
|
||||
+ o_block_offset
|
||||
+ o_cblock_offset
|
||||
+ tl.arange(0, CBLOCK)[:, None] * e
|
||||
+ tl.arange(0, e)[None, :]
|
||||
)
|
||||
|
||||
# Load the decay rate for the current head
|
||||
S_block_ptr = S + off_h
|
||||
s = tl.load(S_block_ptr)
|
||||
|
||||
i = off_cblock
|
||||
q_index = tl.arange(0, CBLOCK) + i * CBLOCK
|
||||
|
||||
# Load query values
|
||||
q = tl.load(Q_block_ptr, mask=block_offset + q_index[:, None] < n, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
|
||||
# Initialize output accumulator
|
||||
qkv = tl.zeros([CBLOCK, e], dtype=tl.float32)
|
||||
|
||||
# Process all sub-blocks up to and
|
||||
# including the current one (causal attention)
|
||||
for j in range(i + 1):
|
||||
kv_index = tl.arange(0, CBLOCK) + j * CBLOCK
|
||||
diff = q_index[:, None] - kv_index[None, :]
|
||||
s_index = s * diff
|
||||
# Apply causal mask: only attend to positions before the current one
|
||||
s_index = tl.where(diff >= 0, -s_index, float("-inf"))
|
||||
decay = tl.exp(s_index)
|
||||
|
||||
# Load key and value
|
||||
k_trans = tl.load(
|
||||
K_trans_block_ptr,
|
||||
mask=block_offset + kv_index[None, :] < n,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
v = tl.load(
|
||||
V_block_ptr,
|
||||
mask=block_offset + kv_index[:, None] < n,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
# Compute attention scores and apply decay
|
||||
qk = tl.dot(q, k_trans) * decay
|
||||
|
||||
# Compute weighted values and accumulate
|
||||
qkv += tl.dot(qk, v)
|
||||
|
||||
# Move to the next sub-block
|
||||
K_trans_block_ptr += CBLOCK * d
|
||||
V_block_ptr += CBLOCK * e
|
||||
|
||||
# Store the result
|
||||
tl.store(
|
||||
O_block_ptr,
|
||||
qkv.to(O_block_ptr.dtype.element_ty),
|
||||
mask=block_offset + q_index[:, None] < n,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_kv_parallel(
|
||||
K,
|
||||
V,
|
||||
K_decay,
|
||||
KV,
|
||||
b: tl.constexpr,
|
||||
h: tl.constexpr,
|
||||
n,
|
||||
d: tl.constexpr,
|
||||
e: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
NUM_BLOCK,
|
||||
D_FBLOCK: tl.constexpr,
|
||||
E_FBLOCK: tl.constexpr,
|
||||
NUM_FBLOCK: tl.constexpr,
|
||||
CBLOCK: tl.constexpr,
|
||||
NUM_CBLOCK: tl.constexpr,
|
||||
):
|
||||
# This kernel computes the key-value outer
|
||||
# products for each block in parallel
|
||||
off_bh = tl.program_id(0) # batch-head index
|
||||
off_block = tl.program_id(1) # block index
|
||||
|
||||
off_h = off_bh % h # head index
|
||||
|
||||
block_offset = off_block * BLOCK
|
||||
|
||||
# Calculate offsets for the current block
|
||||
k_block_offset = block_offset * d
|
||||
v_block_offset = block_offset * e
|
||||
kv_block_offset = off_block * d * e
|
||||
|
||||
# Calculate base offsets for the current batch and head
|
||||
k_offset = off_bh * n * d
|
||||
v_offset = off_bh * n * e
|
||||
kv_offset = off_bh * NUM_BLOCK * d * e
|
||||
|
||||
# Calculate pointers to the key, value, and key-value tensors
|
||||
K_trans_block_ptr = (
|
||||
K
|
||||
+ k_offset
|
||||
+ k_block_offset
|
||||
+ tl.arange(0, CBLOCK)[None, :] * d
|
||||
+ tl.arange(0, D_FBLOCK)[:, None]
|
||||
)
|
||||
V_block_ptr = (
|
||||
V
|
||||
+ v_offset
|
||||
+ v_block_offset
|
||||
+ tl.arange(0, CBLOCK)[:, None] * e
|
||||
+ tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
KV_block_ptr = (
|
||||
KV
|
||||
+ kv_offset
|
||||
+ kv_block_offset
|
||||
+ tl.arange(0, D_FBLOCK)[:, None] * e
|
||||
+ tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
|
||||
# Load the decay factors for the current head and block
|
||||
k_decay_ptr = K_decay + off_h * BLOCK + tl.arange(0, CBLOCK)
|
||||
|
||||
kv_index = tl.arange(0, CBLOCK)
|
||||
|
||||
# Initialize the key-value outer product accumulator
|
||||
kv = tl.zeros([D_FBLOCK, E_FBLOCK], dtype=tl.float32)
|
||||
|
||||
# Handle the last block which might be smaller than BLOCK
|
||||
split_n = n - (NUM_BLOCK - 1) * BLOCK if off_block == NUM_BLOCK - 1 else BLOCK
|
||||
left_shift = tl.cdiv(split_n, CBLOCK) * CBLOCK - split_n
|
||||
num_blocks = min(tl.cdiv(split_n, CBLOCK), NUM_CBLOCK)
|
||||
k_decay_ptr += (NUM_CBLOCK - num_blocks) * CBLOCK
|
||||
|
||||
# Process all sub-blocks in the current block
|
||||
for j in range(num_blocks):
|
||||
left_bound = (1 - j) * left_shift
|
||||
# Load key and value, handling boundary conditions
|
||||
k_trans = tl.load(
|
||||
K_trans_block_ptr - left_shift * d,
|
||||
mask=kv_index[None, :] >= left_bound,
|
||||
other=0.0,
|
||||
)
|
||||
v = tl.load(
|
||||
V_block_ptr - left_shift * e,
|
||||
mask=kv_index[:, None] >= left_bound,
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# Load decay factor and compute weighted key-value outer product
|
||||
k_decay = tl.load(k_decay_ptr)
|
||||
|
||||
# NOTE: Need to add the extra dim here due to AMD MLIR lowering error.
|
||||
# Please don't move it back until issue is resolved.
|
||||
# Issue: https://github.com/ROCm/triton/issues/907
|
||||
k_decay = k_decay[None, :]
|
||||
|
||||
kv += tl.dot(k_trans * k_decay, v)
|
||||
|
||||
# Move to the next sub-block
|
||||
K_trans_block_ptr += CBLOCK * d
|
||||
V_block_ptr += CBLOCK * e
|
||||
k_decay_ptr += CBLOCK
|
||||
|
||||
# Store the result
|
||||
tl.store(KV_block_ptr, kv.to(KV_block_ptr.dtype.element_ty))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_kv_reduce(
|
||||
S,
|
||||
KV,
|
||||
KV_HISTORY,
|
||||
b: tl.constexpr,
|
||||
h: tl.constexpr,
|
||||
n,
|
||||
d: tl.constexpr,
|
||||
e: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
NUM_BLOCK,
|
||||
D_FBLOCK: tl.constexpr,
|
||||
E_FBLOCK: tl.constexpr,
|
||||
):
|
||||
# This kernel reduces the key-value outer products
|
||||
# across blocks and updates the KV history
|
||||
off_bh = tl.program_id(0) # batch-head index
|
||||
off_h = off_bh % h # head index
|
||||
|
||||
kv_offset = off_bh * NUM_BLOCK * d * e
|
||||
|
||||
# Calculate pointer to the key-value tensor
|
||||
KV_block_ptr = (
|
||||
KV
|
||||
+ kv_offset
|
||||
+ tl.arange(0, D_FBLOCK)[:, None] * e
|
||||
+ tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
|
||||
# Load the decay rate for the current head
|
||||
s_ptrs = S + off_h
|
||||
s = tl.load(s_ptrs)
|
||||
|
||||
# Calculate pointer to the key-value history tensor
|
||||
kv_history_offset = off_bh * d * e
|
||||
KV_HISTORY_block_ptr = (
|
||||
KV_HISTORY
|
||||
+ kv_history_offset
|
||||
+ tl.arange(0, D_FBLOCK)[:, None] * e
|
||||
+ tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
|
||||
# Load the previous key-value history
|
||||
kv_pre = tl.load(KV_HISTORY_block_ptr).to(tl.float32)
|
||||
|
||||
# Process all blocks in reverse order to compute the prefix sum
|
||||
for i in range(NUM_BLOCK):
|
||||
block_size = min(n - i * BLOCK, BLOCK)
|
||||
# Compute decay factor for the current block
|
||||
block_decay = tl.exp(-s.to(tl.float32) * block_size)
|
||||
|
||||
# Load the current key-value outer product
|
||||
kv_cur = tl.load(KV_block_ptr).to(tl.float32)
|
||||
# Store the previous key-value history to the current block
|
||||
tl.store(KV_block_ptr, kv_pre.to(KV_block_ptr.dtype.element_ty))
|
||||
|
||||
# Update the key-value history with the current block
|
||||
kv_pre = block_decay * kv_pre + kv_cur
|
||||
KV_block_ptr += d * e
|
||||
|
||||
# Store the updated key-value history
|
||||
tl.store(KV_HISTORY_block_ptr, kv_pre)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_none_diag_kernel(
|
||||
Q,
|
||||
Out,
|
||||
S,
|
||||
KV,
|
||||
b: tl.constexpr,
|
||||
h: tl.constexpr,
|
||||
n,
|
||||
d: tl.constexpr,
|
||||
e: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
NUM_BLOCK,
|
||||
E_FBLOCK: tl.constexpr,
|
||||
CBLOCK: tl.constexpr,
|
||||
NUM_CBLOCK: tl.constexpr,
|
||||
):
|
||||
# This kernel computes the non-diagonal blocks of the attention matrix
|
||||
# Each non-diagonal block represents attention
|
||||
# where queries attend to keys in different blocks
|
||||
off_bh = tl.program_id(0) # batch-head index
|
||||
off_h = off_bh % h # head index
|
||||
|
||||
off_nc = tl.program_id(1)
|
||||
off_n = off_nc // NUM_CBLOCK # block index
|
||||
off_c = off_nc % NUM_CBLOCK # sub-block index
|
||||
off_e = tl.program_id(2) # output feature block index
|
||||
|
||||
n_offset = off_n * BLOCK
|
||||
c_offset = off_c * CBLOCK
|
||||
e_offset = off_e * E_FBLOCK
|
||||
block_offset = n_offset + c_offset
|
||||
|
||||
# Calculate offsets for the current batch, head, and block
|
||||
q_offset = off_bh * n * d + (n_offset + c_offset) * d
|
||||
o_offset = off_bh * n * e + (n_offset + c_offset) * e + e_offset
|
||||
kv_offset = off_bh * NUM_BLOCK * d * e + off_n * d * e + e_offset
|
||||
|
||||
# Calculate pointers to the query, output, and key-value tensors
|
||||
Q_block_ptr = (
|
||||
Q + q_offset + tl.arange(0, CBLOCK)[:, None] * d + tl.arange(0, d)[None, :]
|
||||
)
|
||||
O_block_ptr = (
|
||||
Out
|
||||
+ o_offset
|
||||
+ tl.arange(0, CBLOCK)[:, None] * e
|
||||
+ tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
KV_block_ptr = (
|
||||
KV + kv_offset + tl.arange(0, d)[:, None] * e + tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
|
||||
# Load the decay rate for the current head
|
||||
S_block_ptr = S + off_h
|
||||
s = tl.load(S_block_ptr)
|
||||
|
||||
c_array = tl.arange(0, CBLOCK)
|
||||
|
||||
# Load the key-value outer product for the current block
|
||||
kv = tl.load(KV_block_ptr).to(tl.float32)
|
||||
q_index = block_offset + tl.arange(0, CBLOCK)
|
||||
|
||||
# Load query values
|
||||
q = tl.load(Q_block_ptr, mask=q_index[:, None] < n, other=0.0).to(tl.float32)
|
||||
|
||||
# Compute decay factors for the current sub-block
|
||||
q_decay = tl.exp(-s.to(tl.float32) * (off_c * CBLOCK + c_array[:, None]))
|
||||
|
||||
# Compute non-diagonal attention output
|
||||
qkv_none_diag = tl.dot(q, kv) * q_decay
|
||||
|
||||
# Load diagonal attention output (computed by _fwd_diag_kernel)
|
||||
qkv_diag = tl.load(O_block_ptr, mask=q_index[:, None] < n, other=0.0).to(tl.float32)
|
||||
|
||||
# Combine diagonal and non-diagonal attention outputs
|
||||
qkv = qkv_diag + qkv_none_diag
|
||||
|
||||
# Store the result
|
||||
tl.store(
|
||||
O_block_ptr, qkv.to(O_block_ptr.dtype.element_ty), mask=q_index[:, None] < n
|
||||
)
|
||||
|
||||
|
||||
class _attention(torch.autograd.Function):
|
||||
@staticmethod
|
||||
def forward(ctx, q, k, v, s, kv_history):
|
||||
# Forward pass of the lightning attention algorithm
|
||||
q = q.contiguous()
|
||||
k = k.contiguous()
|
||||
v = v.contiguous()
|
||||
s = s.contiguous()
|
||||
|
||||
# Check CUDA compute capability (Ampere+ required for flash attention
|
||||
# path). Other accelerators (ROCm, XPU) rely on their own Triton
|
||||
# backend support and skip this check.
|
||||
if current_platform.is_cuda():
|
||||
capability = torch.cuda.get_device_capability()
|
||||
if capability[0] < 8:
|
||||
raise RuntimeError(
|
||||
"Flash attention currently only supported",
|
||||
"for compute capability >= 80",
|
||||
)
|
||||
|
||||
# Get input dimensions
|
||||
b, h, n, d = q.shape
|
||||
e = v.shape[-1]
|
||||
|
||||
# Initialize output tensor
|
||||
o = torch.empty((b, h, n, e), dtype=q.dtype, device=q.device)
|
||||
|
||||
# Set block sizes
|
||||
BLOCK = 256
|
||||
NUM_BLOCK = triton.cdiv(n, BLOCK)
|
||||
|
||||
CBLOCK = 32
|
||||
NUM_CBLOCK = BLOCK // CBLOCK
|
||||
assert BLOCK % CBLOCK == 0, "BLOCK must be a multiple of CBLOCK"
|
||||
|
||||
# Compute decay factors for keys
|
||||
array = torch.arange(0, BLOCK, device=q.device) + 1
|
||||
k_decay = torch.exp(-s * (BLOCK - array.reshape(1, -1)))
|
||||
|
||||
# Step 1: Compute diagonal blocks of attention
|
||||
grid = (b * h * NUM_BLOCK, NUM_CBLOCK)
|
||||
_fwd_diag_kernel[grid](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
o,
|
||||
s,
|
||||
b,
|
||||
h,
|
||||
n,
|
||||
d,
|
||||
e,
|
||||
BLOCK=BLOCK,
|
||||
NUM_BLOCK=NUM_BLOCK,
|
||||
CBLOCK=CBLOCK,
|
||||
)
|
||||
|
||||
# Set feature block sizes
|
||||
NUM_FBLOCK = 1
|
||||
D_FBLOCK = d // NUM_FBLOCK
|
||||
assert d % NUM_FBLOCK == 0
|
||||
E_FBLOCK = e // NUM_FBLOCK
|
||||
assert e % NUM_FBLOCK == 0
|
||||
|
||||
CBLOCK = 64
|
||||
NUM_CBLOCK = BLOCK // CBLOCK
|
||||
assert BLOCK % CBLOCK == 0, "BLOCK must be a multiple of CBLOCK"
|
||||
|
||||
# Step 2: Compute key-value outer products for each block in parallel
|
||||
kv = torch.empty((b, h, NUM_BLOCK, d, e), dtype=torch.float32, device=q.device)
|
||||
grid = (b * h, NUM_BLOCK)
|
||||
_fwd_kv_parallel[grid](
|
||||
k,
|
||||
v,
|
||||
k_decay,
|
||||
kv,
|
||||
b,
|
||||
h,
|
||||
n,
|
||||
d,
|
||||
e,
|
||||
BLOCK=BLOCK,
|
||||
NUM_BLOCK=NUM_BLOCK,
|
||||
D_FBLOCK=D_FBLOCK,
|
||||
E_FBLOCK=E_FBLOCK,
|
||||
NUM_FBLOCK=NUM_FBLOCK,
|
||||
CBLOCK=CBLOCK,
|
||||
NUM_CBLOCK=NUM_CBLOCK,
|
||||
)
|
||||
|
||||
# Step 3: Reduce key-value outer products
|
||||
# across blocks and update KV history
|
||||
grid = (b * h, NUM_FBLOCK)
|
||||
_fwd_kv_reduce[grid](
|
||||
s,
|
||||
kv,
|
||||
kv_history,
|
||||
b,
|
||||
h,
|
||||
n,
|
||||
d,
|
||||
e,
|
||||
BLOCK=BLOCK,
|
||||
NUM_BLOCK=NUM_BLOCK,
|
||||
D_FBLOCK=D_FBLOCK,
|
||||
E_FBLOCK=E_FBLOCK,
|
||||
)
|
||||
|
||||
# Step 4: Compute non-diagonal blocks of attention
|
||||
grid = (b * h, NUM_BLOCK * NUM_CBLOCK)
|
||||
_fwd_none_diag_kernel[grid](
|
||||
q,
|
||||
o,
|
||||
s,
|
||||
kv,
|
||||
b,
|
||||
h,
|
||||
n,
|
||||
d,
|
||||
e,
|
||||
BLOCK=BLOCK,
|
||||
NUM_BLOCK=NUM_BLOCK,
|
||||
E_FBLOCK=E_FBLOCK,
|
||||
CBLOCK=CBLOCK,
|
||||
NUM_CBLOCK=NUM_CBLOCK,
|
||||
)
|
||||
|
||||
# Save tensors for backward pass
|
||||
ctx.save_for_backward(q, k, v, s, kv)
|
||||
ctx.BLOCK = BLOCK
|
||||
|
||||
return o, torch.cat([kv, kv_history.unsqueeze(2)], dim=2)
|
||||
|
||||
|
||||
# Apply the lightning attention function
|
||||
lightning_attention_ = _attention.apply
|
||||
|
||||
|
||||
def lightning_attention(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
ed: torch.Tensor,
|
||||
block_size: int = 256,
|
||||
kv_history: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Apply lightning attention algorithm
|
||||
to compute attention efficiently.
|
||||
|
||||
Args:
|
||||
q: Query tensor of shape [batch, heads, seq_len, dim]
|
||||
k: Key tensor of shape [batch, heads, seq_len, dim]
|
||||
v: Value tensor of shape [batch, heads, seq_len, dim_v]
|
||||
ed: Decay rate tensor of shape [heads]
|
||||
block_size: Size of blocks for block-sparse attention
|
||||
kv_history: Optional key-value history from previous computations
|
||||
|
||||
Returns:
|
||||
output: Attention output
|
||||
kv: Updated key-value history
|
||||
|
||||
"""
|
||||
d = q.shape[-1]
|
||||
e = v.shape[-1]
|
||||
|
||||
if ed.dim() == 1:
|
||||
ed = ed.view(1, -1, 1, 1)
|
||||
|
||||
# Split the computation into chunks for better parallelism
|
||||
m = 128 if d >= 128 else 64
|
||||
assert d % m == 0, f"Dimension d ({d}) must be divisible by m ({m})"
|
||||
arr = [m * i for i in range(d // m + 1)]
|
||||
if arr[-1] != d:
|
||||
arr.append(d)
|
||||
n = len(arr)
|
||||
output = 0
|
||||
|
||||
# Initialize or clone key-value history
|
||||
if kv_history is None:
|
||||
kv_history = torch.zeros(
|
||||
(q.shape[0], q.shape[1], d, e), dtype=torch.float32, device=q.device
|
||||
)
|
||||
else:
|
||||
kv_history = kv_history.clone().contiguous()
|
||||
|
||||
# Process each chunk and accumulate results
|
||||
for i in range(n - 1):
|
||||
s = arr[i]
|
||||
e = arr[i + 1]
|
||||
q1 = q[..., s:e]
|
||||
k1 = k[..., s:e]
|
||||
o, kv = lightning_attention_(q1, k1, v, ed, kv_history)
|
||||
output = output + o
|
||||
return output, kv
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _linear_attn_decode_kernel(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
v_ptr,
|
||||
kv_cache_ptr,
|
||||
slope_rate,
|
||||
slot_idx,
|
||||
output_ptr,
|
||||
D: tl.constexpr,
|
||||
qkv_b_stride,
|
||||
qkv_h_stride,
|
||||
cache_b_stride,
|
||||
cache_h_stride,
|
||||
cache_d0_stride,
|
||||
cache_d1_stride,
|
||||
pad_slot_id: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""Kernel for linear attention decoding with KV cache.
|
||||
|
||||
This kernel computes attention for a single token using the KV cache.
|
||||
"""
|
||||
pid_b = tl.program_id(0) # batch index
|
||||
pid_h = tl.program_id(1) # head index
|
||||
pid_d = tl.program_id(2) # dimension block index
|
||||
|
||||
# Load slot index for the current batch
|
||||
slot_id = tl.load(slot_idx + pid_b).to(tl.int64)
|
||||
|
||||
# Skip if slot_id is PAD_SLOT_ID (padding)
|
||||
if slot_id == pad_slot_id:
|
||||
return
|
||||
|
||||
batch_id = pid_b
|
||||
head_id = pid_h
|
||||
|
||||
# Load decay rate for the current head
|
||||
ratio = tl.load(slope_rate + pid_h)
|
||||
|
||||
# Calculate offsets for dimensions
|
||||
qk_d_offsets = tl.arange(0, D)
|
||||
v_d_offsets = tl.arange(0, BLOCK_SIZE) + pid_d * BLOCK_SIZE
|
||||
cache_d_offsets = (
|
||||
qk_d_offsets[:, None] * cache_d0_stride + v_d_offsets[None, :] * cache_d1_stride
|
||||
)
|
||||
|
||||
# Calculate offsets for the current batch and head
|
||||
q_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
|
||||
k_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
|
||||
v_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
|
||||
|
||||
cache_offset = slot_id * cache_b_stride + head_id * cache_h_stride
|
||||
|
||||
# Create masks for loading tensors
|
||||
qk_mask = qk_d_offsets < D
|
||||
v_mask = v_d_offsets < D
|
||||
|
||||
# Load query, key, and value tensors
|
||||
q = tl.load(q_ptr + q_offset + qk_d_offsets, mask=qk_mask, other=0.0)
|
||||
k = tl.load(k_ptr + k_offset + qk_d_offsets, mask=qk_mask, other=0.0)
|
||||
v = tl.load(v_ptr + v_offset + v_d_offsets, mask=v_mask, other=0.0)
|
||||
|
||||
# Compute key-value outer product
|
||||
kv_outer = k[:, None] * v[None, :]
|
||||
kv_mask = qk_mask[:, None] & v_mask[None, :]
|
||||
|
||||
# Apply decay to previous KV cache
|
||||
ratio = tl.exp(-ratio)
|
||||
kv_ptr = kv_cache_ptr + cache_offset + cache_d_offsets
|
||||
kv_cache_old = tl.load(kv_ptr, mask=kv_mask, other=0.0)
|
||||
kv_outer = kv_outer + ratio * kv_cache_old
|
||||
|
||||
# Compute attention output
|
||||
output = q[:, None].to(tl.float32) * kv_outer
|
||||
output = tl.sum(output, axis=0)
|
||||
|
||||
# Update KV cache and store output
|
||||
tl.store(kv_ptr, kv_outer, mask=kv_mask)
|
||||
tl.store(output_ptr + q_offset + v_d_offsets, output, mask=v_mask)
|
||||
|
||||
|
||||
def linear_decode_forward_triton(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_caches: torch.Tensor,
|
||||
slope_rate: torch.Tensor,
|
||||
slot_idx: torch.Tensor,
|
||||
BLOCK_SIZE: int = 32,
|
||||
) -> torch.Tensor:
|
||||
"""Perform linear attention decoding using Triton kernels.
|
||||
|
||||
Args:
|
||||
q: Query tensor of shape [B, H, 1, D]
|
||||
k: Key tensor of shape [B, H, 1, D]
|
||||
v: Value tensor of shape [B, H, 1, D]
|
||||
kv_caches: Key-value cache tensor
|
||||
slope_rate: Decay rate tensor
|
||||
slot_idx: Slot indices for batches
|
||||
BLOCK_SIZE: Size of blocks for processing
|
||||
|
||||
Returns:
|
||||
output: Attention output tensor
|
||||
|
||||
"""
|
||||
B, H, _, D = q.shape
|
||||
assert k.shape == (B, H, 1, D)
|
||||
assert v.shape == (B, H, 1, D)
|
||||
|
||||
# Initialize output tensor
|
||||
output = torch.empty_like(q)
|
||||
|
||||
# Set grid dimensions for the kernel
|
||||
grid = (B, H, D // BLOCK_SIZE)
|
||||
|
||||
# Calculate strides for tensors
|
||||
qkv_b_stride = q.stride(0)
|
||||
qkv_h_stride = q.stride(1)
|
||||
|
||||
cache_b_stride = kv_caches.stride(0)
|
||||
cache_h_stride = kv_caches.stride(1)
|
||||
cache_d0_stride = kv_caches.stride(2)
|
||||
cache_d1_stride = kv_caches.stride(3)
|
||||
|
||||
# Launch the kernel
|
||||
_linear_attn_decode_kernel[grid](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_caches,
|
||||
slope_rate,
|
||||
slot_idx,
|
||||
output,
|
||||
D,
|
||||
qkv_b_stride,
|
||||
qkv_h_stride,
|
||||
cache_b_stride,
|
||||
cache_h_stride,
|
||||
cache_d0_stride,
|
||||
cache_d1_stride,
|
||||
pad_slot_id=PAD_SLOT_ID,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
# Reshape output and return
|
||||
output = rearrange(output, "b h n d -> b n (h d)")
|
||||
return output.squeeze(1).contiguous()
|
||||
@@ -1,382 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata
|
||||
|
||||
|
||||
# --8<-- [start:short_conv]
|
||||
@PluggableLayer.register("short_conv")
|
||||
class ShortConv(MambaBase, PluggableLayer):
|
||||
# --8<-- [end:short_conv]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
dim: int,
|
||||
layer_idx: int,
|
||||
model_config: ModelConfig | None = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.layer_idx = layer_idx
|
||||
self.conv_dim = dim
|
||||
self.L_cache = config.conv_L_cache
|
||||
self.bias = config.conv_bias
|
||||
|
||||
self.conv = ColumnParallelLinear(
|
||||
input_size=self.L_cache,
|
||||
output_size=dim,
|
||||
bias=self.bias,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
# unsqueeze to fit conv1d weights shape into the linear weights shape.
|
||||
# Can't do this in `weight_loader` since it already exists in
|
||||
# `ColumnParallelLinear` and `set_weight_attrs`
|
||||
# doesn't allow to override it
|
||||
self.conv.weight.data = self.conv.weight.data.unsqueeze(1)
|
||||
|
||||
self.in_proj = MergedColumnParallelLinear(
|
||||
input_size=dim,
|
||||
output_sizes=[dim] * 3,
|
||||
bias=self.bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.in_proj",
|
||||
)
|
||||
self.out_proj = RowParallelLinear(
|
||||
input_size=dim,
|
||||
output_size=dim,
|
||||
bias=self.bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.num_spec = vllm_config.num_speculative_tokens
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
self.kv_cache = (torch.tensor([]),)
|
||||
|
||||
self.model_config = model_config
|
||||
self.cache_config = cache_config
|
||||
self.prefix = prefix
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
# Reference torch causal conv1d; runs on all CPU platforms. AMX kernels
|
||||
# for causal conv can be plugged in here later.
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_fn_cpu as causal_conv1d_torch,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_update_cpu,
|
||||
causal_conv1d_update_torch,
|
||||
)
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
assert attn_metadata is None or isinstance(
|
||||
attn_metadata, ShortConvAttentionMetadata
|
||||
)
|
||||
|
||||
BCx, _ = self.in_proj(hidden_states)
|
||||
B, C, x = BCx.chunk(3, dim=-1)
|
||||
|
||||
# (dim, kernel_size) — same reshape as forward_cuda
|
||||
conv_weights = self.conv.weight.view(
|
||||
self.conv.weight.size(0), self.conv.weight.size(2)
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
# Profile run — output value doesn't matter
|
||||
Bx = (B * x).contiguous()
|
||||
output_tensor, _ = self.out_proj(C * Bx)
|
||||
output[: hidden_states.shape[0]] = output_tensor
|
||||
return
|
||||
|
||||
conv_state = (
|
||||
self.kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self.kv_cache[0].transpose(-1, -2)
|
||||
) # (num_blocks, dim, state_len)
|
||||
|
||||
num_prefills = attn_metadata.num_prefills
|
||||
num_decodes = attn_metadata.num_decode_tokens
|
||||
num_prefill_tokens = attn_metadata.num_prefill_tokens
|
||||
has_prefill = num_prefills > 0
|
||||
has_decode = num_decodes > 0
|
||||
num_actual_tokens = num_decodes + num_prefill_tokens
|
||||
|
||||
B_d, B_p = torch.split(
|
||||
B[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
|
||||
)
|
||||
C_d, C_p = torch.split(
|
||||
C[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
|
||||
)
|
||||
x_d, x_p = torch.split(
|
||||
x[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
|
||||
)
|
||||
|
||||
conv_output_list = []
|
||||
|
||||
if has_prefill:
|
||||
assert attn_metadata.state_indices_tensor_p is not None
|
||||
Bx_p = (B_p * x_p).transpose(0, 1) # (dim, num_prefill_tokens)
|
||||
out_p = causal_conv1d_torch(
|
||||
Bx_p,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
conv_state,
|
||||
attn_metadata.query_start_loc_p,
|
||||
attn_metadata.state_indices_tensor_p.flatten(),
|
||||
attn_metadata.has_initial_states_p,
|
||||
activation=None,
|
||||
).transpose(0, 1)[:num_prefill_tokens] # (num_prefill_tokens, dim)
|
||||
conv_output_list.append(C_p * out_p)
|
||||
|
||||
if has_decode:
|
||||
assert attn_metadata.state_indices_tensor_d is not None
|
||||
state_indices_d = attn_metadata.state_indices_tensor_d.flatten()
|
||||
Bx_d = B_d * x_d # (num_decodes, dim)
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
conv_state_view = conv_state[state_indices_d].contiguous()
|
||||
out_d = causal_conv1d_update_torch(
|
||||
Bx_d.unsqueeze(-1),
|
||||
conv_state_view,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
).squeeze(-1)
|
||||
conv_state[state_indices_d] = conv_state_view
|
||||
else:
|
||||
out_d = causal_conv1d_update_cpu(
|
||||
Bx_d,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
conv_state_indices=state_indices_d,
|
||||
)
|
||||
conv_output_list.insert(0, C_d * out_d)
|
||||
|
||||
hidden_states_out = torch.vstack(conv_output_list)
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden_states_out)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
torch.ops.vllm.short_conv(
|
||||
hidden_states,
|
||||
output,
|
||||
self.prefix,
|
||||
)
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
forward_context = get_forward_context()
|
||||
# ShortConvAttentionMetadata contains metadata necessary for the
|
||||
# short_conv triton kernels to operate in continuous batching and in
|
||||
# chunked prefill modes; they are computed at top-level model forward
|
||||
# since they stay the same and reused for all mamba layers in the same
|
||||
# iteration.
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
|
||||
conv_state = (
|
||||
self.kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self.kv_cache[0].transpose(-1, -2)
|
||||
)
|
||||
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
|
||||
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
|
||||
has_initial_states_p = attn_metadata.has_initial_states_p
|
||||
query_start_loc_p = attn_metadata.query_start_loc_p
|
||||
num_accepted_tokens = attn_metadata.num_accepted_tokens
|
||||
query_start_loc_d = attn_metadata.query_start_loc_d
|
||||
|
||||
BCx, _ = self.in_proj(hidden_states)
|
||||
|
||||
B, C, x = BCx.chunk(3, dim=-1)
|
||||
|
||||
conv_weights = self.conv.weight.view(
|
||||
self.conv.weight.size(0), self.conv.weight.size(2)
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
# V1 profile run
|
||||
Bx = (B * x).contiguous()
|
||||
hidden_states = C * Bx
|
||||
contextualized_states, _ = self.out_proj(hidden_states)
|
||||
return contextualized_states
|
||||
|
||||
num_prefills = attn_metadata.num_prefills # request count
|
||||
num_decodes = attn_metadata.num_decode_tokens # token count (=request)
|
||||
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
|
||||
has_prefill = num_prefills > 0
|
||||
has_decode = num_decodes > 0
|
||||
num_actual_tokens = num_decodes + num_prefill_tokens
|
||||
|
||||
# NOTE: V1 puts decode before prefill
|
||||
# Separate prefill and decode by splitting varlen input
|
||||
# Split along token dimension
|
||||
B_d, B_p = torch.split(
|
||||
B[:num_actual_tokens],
|
||||
[num_decodes, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
C_d, C_p = torch.split(
|
||||
C[:num_actual_tokens],
|
||||
[num_decodes, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
x_d, x_p = torch.split(
|
||||
x[:num_actual_tokens],
|
||||
[num_decodes, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
conv_output_list = []
|
||||
|
||||
if has_prefill:
|
||||
Bx_p = (B_p * x_p).transpose(0, 1)
|
||||
Bx = causal_conv1d_fn(
|
||||
Bx_p,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
conv_states=conv_state,
|
||||
has_initial_state=has_initial_states_p,
|
||||
cache_indices=state_indices_tensor_p,
|
||||
metadata=attn_metadata,
|
||||
query_start_loc=query_start_loc_p,
|
||||
).transpose(0, 1)[:num_prefill_tokens]
|
||||
|
||||
y = C_p * Bx
|
||||
conv_output_list.append(y)
|
||||
|
||||
if has_decode:
|
||||
Bx_d = (B_d * x_d).contiguous()
|
||||
if num_accepted_tokens is not None:
|
||||
# Speculative decode: the verify step feeds >1 query token per
|
||||
# decode request, so use the spec-aware conv update path
|
||||
# (mirrors mamba_mixer2). state_indices_tensor_d is
|
||||
# (num_decodes, 1 + num_spec_tokens) here.
|
||||
assert state_indices_tensor_d is not None
|
||||
Bx = causal_conv1d_update(
|
||||
Bx_d,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
conv_state_indices=state_indices_tensor_d,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=query_start_loc_d,
|
||||
max_query_len=state_indices_tensor_d.size(-1),
|
||||
)
|
||||
else:
|
||||
# Non-spec decode
|
||||
Bx = causal_conv1d_update(
|
||||
Bx_d,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
conv_state_indices=state_indices_tensor_d,
|
||||
)
|
||||
y = C_d * Bx
|
||||
conv_output_list.insert(0, y)
|
||||
|
||||
# Merge prefill and decode outputs before passing to gated MLP
|
||||
hidden_states = torch.vstack(conv_output_list)
|
||||
|
||||
# Final linear projection
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden_states)
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
return MambaStateDtypeCalculator.short_conv_state_dtype(
|
||||
self.model_config.dtype,
|
||||
self.cache_config.mamba_cache_dtype,
|
||||
)
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.short_conv_state_shape(
|
||||
tp_world_size=get_tensor_model_parallel_world_size(),
|
||||
intermediate_size=self.conv_dim,
|
||||
conv_kernel=self.L_cache,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.SHORT_CONV
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def short_conv(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
if not current_platform.is_cpu():
|
||||
self.forward_cuda(hidden_states=hidden_states, output=output)
|
||||
else:
|
||||
self.forward_native(hidden_states=hidden_states, output=output)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="short_conv",
|
||||
op_func=short_conv,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
@@ -1,10 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.minimax_rms_norm.rms_norm_tp import (
|
||||
MiniMaxText01RMSNormTP,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MiniMaxText01RMSNormTP",
|
||||
]
|
||||
@@ -1,316 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import array
|
||||
import contextlib
|
||||
import ctypes
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import torch
|
||||
|
||||
try:
|
||||
from cuda.bindings import runtime as cudart
|
||||
except ImportError:
|
||||
from cuda import cudart
|
||||
|
||||
_ALIGN = 1 << 21 # 2 MiB — CUDA IPC allocation alignment
|
||||
_IPC_HANDLE_SIZE = 64 # sizeof(cudaIpcMemHandle_t)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CUDA helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _check(error):
|
||||
"""Raise on CUDA runtime error."""
|
||||
success = getattr(cudart.cudaError_t, "cudaSuccess", None) or cudart.cudaError_t(0)
|
||||
if error != success:
|
||||
raise RuntimeError(f"CUDA runtime error: {error}")
|
||||
|
||||
|
||||
# cuda-bindings >= 13.4 no longer exposes cudaIpcMemHandle_t.reserved, so
|
||||
# copy the raw handle bytes through getPtr() instead. This is safe because the
|
||||
# CUDA ABI fixes the handle as an opaque 64-byte struct (char reserved[64]).
|
||||
def _ipc_handle_to_bytes(handle) -> bytes:
|
||||
return ctypes.string_at(handle.getPtr(), _IPC_HANDLE_SIZE)
|
||||
|
||||
|
||||
def _ipc_handle_from_bytes(raw: bytes):
|
||||
assert len(raw) == _IPC_HANDLE_SIZE, len(raw)
|
||||
handle = cudart.cudaIpcMemHandle_t()
|
||||
ctypes.memmove(handle.getPtr(), raw, _IPC_HANDLE_SIZE)
|
||||
return handle
|
||||
|
||||
|
||||
def _cuda_malloc(size: int):
|
||||
aligned = ((size + _ALIGN - 1) >> 21) << 21
|
||||
err, ptr = cudart.cudaMalloc(aligned)
|
||||
_check(err)
|
||||
return ptr, aligned
|
||||
|
||||
|
||||
def _cuda_free(ptr: int):
|
||||
if ptr:
|
||||
_check(cudart.cudaFree(ptr)[0])
|
||||
|
||||
|
||||
def _cuda_memset_zero(ptr: int, size: int):
|
||||
_check(cudart.cudaMemset(ptr, 0, size)[0])
|
||||
|
||||
|
||||
def _cuda_memcpy_d2d(dst: int, src: int, size: int):
|
||||
_check(
|
||||
cudart.cudaMemcpy(
|
||||
dst, src, size, cudart.cudaMemcpyKind.cudaMemcpyDeviceToDevice
|
||||
)[0]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IPC buffer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class IpcBuffer:
|
||||
"""Allocates CUDA device memory and exchanges IPC handles with all ranks
|
||||
so that every rank holds a valid device pointer to every other rank's buffer.
|
||||
"""
|
||||
|
||||
def __init__(self, rank: int, world_size: int, size: int, process_group=None):
|
||||
self.rank = rank
|
||||
self.world_size = world_size
|
||||
self.peer_ptrs: list[int] = [0] * world_size
|
||||
self.local_ptr: int = 0
|
||||
self._alive = False
|
||||
|
||||
if size <= 0:
|
||||
return
|
||||
|
||||
self.local_ptr, _ = _cuda_malloc(size)
|
||||
_cuda_memset_zero(self.local_ptr, size)
|
||||
self._alive = True
|
||||
|
||||
# --- exchange IPC handles via torch.distributed ---
|
||||
err, local_handle = cudart.cudaIpcGetMemHandle(self.local_ptr)
|
||||
_check(err)
|
||||
|
||||
all_handles: list[bytes | None] = [None] * world_size
|
||||
torch.distributed.all_gather_object(
|
||||
all_handles, _ipc_handle_to_bytes(local_handle), group=process_group
|
||||
)
|
||||
|
||||
for r in range(world_size):
|
||||
if r == rank:
|
||||
self.peer_ptrs[r] = self.local_ptr
|
||||
else:
|
||||
raw = all_handles[r]
|
||||
assert raw is not None
|
||||
handle = _ipc_handle_from_bytes(raw)
|
||||
err, ptr = cudart.cudaIpcOpenMemHandle(
|
||||
handle, cudart.cudaIpcMemLazyEnablePeerAccess
|
||||
)
|
||||
_check(err)
|
||||
self.peer_ptrs[r] = ptr
|
||||
|
||||
def serialize(self) -> list[int]:
|
||||
"""Return peer pointers as a list of int64 values (one per rank)."""
|
||||
raw = b""
|
||||
for ptr in self.peer_ptrs:
|
||||
raw += struct.pack("P", ptr)
|
||||
return array.array("Q", raw).tolist()
|
||||
|
||||
def cleanup(self):
|
||||
if not self._alive:
|
||||
return
|
||||
self._alive = False
|
||||
for r in range(self.world_size):
|
||||
if self.peer_ptrs[r] == 0:
|
||||
continue
|
||||
if r == self.rank:
|
||||
_cuda_free(self.peer_ptrs[r])
|
||||
else:
|
||||
with contextlib.suppress(RuntimeError):
|
||||
_check(cudart.cudaIpcCloseMemHandle(self.peer_ptrs[r])[0])
|
||||
self.peer_ptrs[r] = 0
|
||||
self.local_ptr = 0
|
||||
|
||||
def __del__(self):
|
||||
if not sys.is_finalizing():
|
||||
self.cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lamport negative-zero initialization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _lamport_fill_neg_zero(device_ptr: int, size_bytes: int):
|
||||
"""Fill device memory with IEEE-754 negative zero (-0.0f = 0x80000000).
|
||||
This is the "slot empty" sentinel for the Lamport protocol: the kernel
|
||||
spin-waits until a value is *not* negative zero.
|
||||
"""
|
||||
if size_bytes == 0 or device_ptr == 0:
|
||||
return
|
||||
n_floats = size_bytes // 4
|
||||
# torch preserves -0.0 in IEEE-754
|
||||
fill = torch.full((n_floats,), -0.0, dtype=torch.float32, device="cuda")
|
||||
_cuda_memcpy_d2d(device_ptr, fill.data_ptr(), size_bytes)
|
||||
del fill
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LamportWorkspace — the main class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LamportWorkspace:
|
||||
"""Self-contained workspace for Lamport-based cross-GPU AllReduce.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
rank : int
|
||||
Local rank (0-based).
|
||||
world_size : int
|
||||
Total number of ranks in the TP group.
|
||||
comm_size : int
|
||||
Size in bytes of *one* Lamport buffer slot. The total IPC allocation
|
||||
per rank is ``3 * comm_size`` (triple-buffering). Must be large enough
|
||||
to hold the per-slot data written by the kernel. Use
|
||||
``compute_comm_size_for_minimax()`` for a safe default.
|
||||
process_group : optional
|
||||
``torch.distributed`` process group for IPC handle exchange.
|
||||
``None`` uses the default group.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, rank: int, world_size: int, comm_size: int, process_group=None):
|
||||
assert world_size >= 2, "Lamport workspace requires at least 2 ranks"
|
||||
assert comm_size > 0, "comm_size must be positive"
|
||||
|
||||
self.rank = rank
|
||||
self.world_size = world_size
|
||||
self.comm_size = comm_size
|
||||
|
||||
# 1) Lamport triple-buffer (the only IPC memory the kernel reads/writes)
|
||||
lamport_total = 3 * comm_size
|
||||
self._lamport = IpcBuffer(rank, world_size, lamport_total, process_group)
|
||||
_lamport_fill_neg_zero(self._lamport.local_ptr, lamport_total)
|
||||
|
||||
# 2) flag_buffer on device: int32[3] = {counter, unused, lamport_flag}
|
||||
# counter — used for block-level sync inside the kernel
|
||||
# unused — reserved (index 1)
|
||||
# lamport_flag — triple-buffer rotation index (0 → 1 → 2 → 0 …)
|
||||
self._flag_buf = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
|
||||
# 3) layout_buffer on device: int64[2] = {clear_size, comm_size}
|
||||
# clear_size — bytes to clear from *previous* slot (set by kernel)
|
||||
# comm_size — size of one triple-buffer slot
|
||||
self._layout_buf = torch.tensor(
|
||||
[0, comm_size], dtype=torch.int64, device="cuda"
|
||||
)
|
||||
|
||||
# 4) Assemble device-side void* pointer array
|
||||
N = world_size
|
||||
ptrs: list[int] = []
|
||||
ptrs += [0] * N # [0 .. N-1] ipc_buffers (placeholder)
|
||||
ptrs += [0] * N # [N .. 2N-1] ipc_barriers (placeholder)
|
||||
ptrs += self._lamport.serialize() # [2N .. 3N-1] lamport peer ptrs
|
||||
ptrs.append(self._flag_buf.data_ptr()) # [3N] flag_buffer
|
||||
ptrs.append(self._layout_buf.data_ptr()) # [3N+1] layout_buffer
|
||||
|
||||
self._workspace = torch.tensor(ptrs, dtype=torch.int64, device="cuda")
|
||||
|
||||
@property
|
||||
def workspace(self) -> torch.Tensor:
|
||||
"""Device tensor (int64) that can be passed to the kernel
|
||||
as ``void** workspace``."""
|
||||
return self._workspace
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def compute_comm_size_for_minimax(
|
||||
max_tokens: int,
|
||||
world_size: int,
|
||||
fused_qk: bool = True,
|
||||
) -> int:
|
||||
"""Return a safe ``comm_size`` (in bytes) for MiniMaxReduceRMSKernel.
|
||||
|
||||
The kernel stores per-token variance scalars in the Lamport buffer:
|
||||
- single-matrix path: ``world_size × max_tokens × 4`` bytes per slot
|
||||
- fused Q+K path: ``world_size × 2 × ceil(max_tokens/4) × 16`` bytes per slot
|
||||
|
||||
The returned value is rounded up to 2 MiB alignment.
|
||||
"""
|
||||
if fused_qk:
|
||||
groups = (max_tokens + 3) // 4
|
||||
slot_bytes = world_size * 2 * groups * 16 # 16 = sizeof(float4)
|
||||
else:
|
||||
slot_bytes = world_size * max_tokens * 4 # 4 = sizeof(float)
|
||||
return ((slot_bytes + _ALIGN - 1) >> 21) << 21
|
||||
|
||||
def cleanup(self):
|
||||
if hasattr(self, "_lamport"):
|
||||
self._lamport.cleanup()
|
||||
|
||||
def __del__(self):
|
||||
if not sys.is_finalizing():
|
||||
self.cleanup()
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"LamportWorkspace(rank={self.rank}, world_size={self.world_size}, "
|
||||
f"comm_size={self.comm_size})"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cached convenience function (mirrors TRT-LLM's get_allreduce_workspace)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_cache_lock = threading.Lock()
|
||||
_workspace_cache: dict = {}
|
||||
|
||||
|
||||
def get_allreduce_workspace(
|
||||
rank: int,
|
||||
world_size: int,
|
||||
comm_size: int | None = None,
|
||||
max_tokens: int = 16384,
|
||||
process_group=None,
|
||||
) -> torch.Tensor:
|
||||
"""Return a cached workspace tensor for the given (rank, world_size) pair.
|
||||
|
||||
On first call the workspace is allocated and IPC handles are exchanged;
|
||||
subsequent calls with the same arguments return the cached tensor.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
rank, world_size : int
|
||||
TP rank and TP size.
|
||||
comm_size : int, optional
|
||||
Explicit slot size in bytes. If ``None``, computed automatically
|
||||
from ``max_tokens`` and ``world_size`` (fused Q+K path).
|
||||
max_tokens : int
|
||||
Maximum number of tokens per batch (used when ``comm_size is None``).
|
||||
process_group : optional
|
||||
``torch.distributed`` process group.
|
||||
|
||||
"""
|
||||
if comm_size is None:
|
||||
comm_size = LamportWorkspace.compute_comm_size_for_minimax(
|
||||
max_tokens, world_size, fused_qk=True
|
||||
)
|
||||
pg_id = id(process_group) if process_group is not None else 0
|
||||
key = (rank, world_size, comm_size, pg_id)
|
||||
with _cache_lock:
|
||||
if key not in _workspace_cache:
|
||||
ws = LamportWorkspace(rank, world_size, comm_size, process_group)
|
||||
_workspace_cache[key] = ws
|
||||
return _workspace_cache[key].workspace
|
||||
@@ -1,421 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Max number of tokens supported by the Lamport fused allreduce+RMSNorm kernel.
|
||||
# Larger batches fall back to the eager allreduce + RMSNorm path.
|
||||
MINIMAX_QK_NORM_MAX_TOKEN_NUM = 2048
|
||||
|
||||
_MINIMAX_FUSED_AR_RMS_QK = getattr(torch.ops._C, "minimax_allreduce_rms_qk", None)
|
||||
|
||||
|
||||
def _all_reduce_variance(var: torch.Tensor) -> torch.Tensor:
|
||||
"""All-reduce a per-token variance tensor across the TP group.
|
||||
|
||||
Variance is accumulated in fp32 for numerical stability. The FlashInfer
|
||||
fused all-reduce caches a single global workspace keyed to the model's
|
||||
16-bit activation dtype (``use_fp32_lamport=False``); routing an fp32
|
||||
reduction through it would read against a mismatched workspace and corrupt
|
||||
the result. FlashInfer's fast-path only triggers for 2D inputs, so reducing
|
||||
a flattened (1D) view keeps these fp32 reductions on custom all-reduce /
|
||||
pynccl, both of which handle fp32 correctly.
|
||||
"""
|
||||
return tensor_model_parallel_all_reduce(var.flatten()).view_as(var)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _minimax_qk_var_kernel(
|
||||
qkv_ptr, # [num_tokens, hidden], 16-bit activations
|
||||
var_ptr, # [num_tokens, 2], fp32
|
||||
row_stride, # element stride between tokens in qkv
|
||||
q_size: tl.constexpr, # constant per deployment -> loops unroll, mask elides
|
||||
kv_size: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""TP-pre stage: per-token mean-of-squares for the q and k segments.
|
||||
|
||||
Accumulates in fp32 while reading the 16-bit qkv in place, so no fp32
|
||||
copy of q/k is materialized. ``var[:, 0]`` is the q variance and
|
||||
``var[:, 1]`` the k variance; both are the local-shard means, ready for
|
||||
the all-reduce that follows.
|
||||
"""
|
||||
token = tl.program_id(0)
|
||||
base = qkv_ptr + token * row_stride
|
||||
|
||||
q_acc = 0.0
|
||||
for off in range(0, q_size, BLOCK):
|
||||
idx = off + tl.arange(0, BLOCK)
|
||||
mask = idx < q_size
|
||||
x = tl.load(base + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
q_acc += tl.sum(x * x, axis=0)
|
||||
|
||||
k_acc = 0.0
|
||||
for off in range(0, kv_size, BLOCK):
|
||||
idx = off + tl.arange(0, BLOCK)
|
||||
mask = idx < kv_size
|
||||
x = tl.load(base + q_size + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
k_acc += tl.sum(x * x, axis=0)
|
||||
|
||||
tl.store(var_ptr + token * 2 + 0, q_acc / q_size)
|
||||
tl.store(var_ptr + token * 2 + 1, k_acc / kv_size)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _minimax_rms_apply_kernel(
|
||||
qkv_ptr, # [num_tokens, hidden]
|
||||
var_ptr, # [num_tokens, 2], fp32, all-reduced sum of per-shard means
|
||||
q_w_ptr, # [q_size], q per-channel weight
|
||||
k_w_ptr, # [kv_size], k per-channel weight
|
||||
q_out_ptr, # [num_tokens, q_size], contiguous
|
||||
k_out_ptr, # [num_tokens, kv_size], contiguous
|
||||
row_stride, # element stride between tokens in qkv
|
||||
q_size: tl.constexpr, # constant per deployment -> loops unroll, mask elides
|
||||
kv_size: tl.constexpr,
|
||||
tp_world: tl.constexpr, # folds the post-all-reduce /tp_world into rsqrt
|
||||
eps: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""TP-post stage: ``x * rsqrt(var / tp_world + eps) * weight``.
|
||||
|
||||
A single program normalizes both the q and k segments of one token, so q
|
||||
and k share one launch instead of two. The all-reduce yields the sum of
|
||||
per-shard means, so the ``/ tp_world`` that recovers the global
|
||||
mean-of-squares is folded into the ``rsqrt`` here rather than run as a
|
||||
separate elementwise pass over the ``[num_tokens, 2]`` variance tensor.
|
||||
"""
|
||||
token = tl.program_id(0)
|
||||
base = qkv_ptr + token * row_stride
|
||||
|
||||
q_inv = tl.rsqrt(tl.load(var_ptr + token * 2 + 0) / tp_world + eps)
|
||||
q_out_row = q_out_ptr + token * q_size
|
||||
for off in range(0, q_size, BLOCK):
|
||||
idx = off + tl.arange(0, BLOCK)
|
||||
mask = idx < q_size
|
||||
x = tl.load(base + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
w = tl.load(q_w_ptr + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
y = x * q_inv * w
|
||||
tl.store(q_out_row + idx, y.to(q_out_ptr.dtype.element_ty), mask=mask)
|
||||
|
||||
k_inv = tl.rsqrt(tl.load(var_ptr + token * 2 + 1) / tp_world + eps)
|
||||
k_out_row = k_out_ptr + token * kv_size
|
||||
for off in range(0, kv_size, BLOCK):
|
||||
idx = off + tl.arange(0, BLOCK)
|
||||
mask = idx < kv_size
|
||||
x = tl.load(base + q_size + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
w = tl.load(k_w_ptr + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
y = x * k_inv * w
|
||||
tl.store(k_out_row + idx, y.to(k_out_ptr.dtype.element_ty), mask=mask)
|
||||
|
||||
|
||||
def _minimax_qk_norm_tp_eager(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
tp_world: int,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pure-torch reference path used when Triton is unavailable."""
|
||||
q, k, _ = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
orig_dtype = q.dtype
|
||||
q = q.to(torch.float32)
|
||||
k = k.to(torch.float32)
|
||||
q_var = q.pow(2).mean(dim=-1, keepdim=True)
|
||||
k_var = k.pow(2).mean(dim=-1, keepdim=True)
|
||||
|
||||
qk_var = torch.cat([q_var, k_var], dim=-1)
|
||||
qk_var = _all_reduce_variance(qk_var) / tp_world
|
||||
q_var, k_var = qk_var.chunk(2, dim=-1)
|
||||
q = q * torch.rsqrt(q_var + eps) * q_weight
|
||||
k = k * torch.rsqrt(k_var + eps) * k_weight
|
||||
return q.to(orig_dtype), k.to(orig_dtype)
|
||||
|
||||
|
||||
def _minimax_qk_norm_tp_fallback(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
tp_rank: int,
|
||||
tp_world: int,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""All-reduce + QK RMSNorm without the Lamport fused kernel.
|
||||
|
||||
The all-reduce is a TP communication barrier and cannot live inside a
|
||||
single kernel, so the eager-torch path is split into two Triton kernels
|
||||
around it: a variance reduction before the all-reduce and a normalize
|
||||
after. Compared to the ``torch.compile`` path this avoids materializing
|
||||
fp32 copies of q/k and the ``cat``/``chunk`` temporaries.
|
||||
"""
|
||||
if not HAS_TRITON:
|
||||
return _minimax_qk_norm_tp_eager(
|
||||
qkv, q_weight, k_weight, q_size, kv_size, tp_world, eps
|
||||
)
|
||||
|
||||
num_tokens = qkv.shape[0]
|
||||
row_stride = qkv.stride(0)
|
||||
BLOCK = 1024
|
||||
grid = (num_tokens,)
|
||||
|
||||
qk_var = torch.empty(num_tokens, 2, dtype=torch.float32, device=qkv.device)
|
||||
_minimax_qk_var_kernel[grid](
|
||||
qkv, qk_var, row_stride, q_size=q_size, kv_size=kv_size, BLOCK=BLOCK
|
||||
)
|
||||
|
||||
# All-reduce sums the per-shard means; the /tp_world that turns this back
|
||||
# into the global mean is folded into the apply kernel's rsqrt below.
|
||||
qk_var = _all_reduce_variance(qk_var)
|
||||
|
||||
q_out = torch.empty(num_tokens, q_size, dtype=qkv.dtype, device=qkv.device)
|
||||
k_out = torch.empty(num_tokens, kv_size, dtype=qkv.dtype, device=qkv.device)
|
||||
_minimax_rms_apply_kernel[grid](
|
||||
qkv,
|
||||
qk_var,
|
||||
q_weight,
|
||||
k_weight,
|
||||
q_out,
|
||||
k_out,
|
||||
row_stride,
|
||||
q_size=q_size,
|
||||
kv_size=kv_size,
|
||||
tp_world=tp_world,
|
||||
eps=eps,
|
||||
BLOCK=BLOCK,
|
||||
)
|
||||
return q_out, k_out
|
||||
|
||||
|
||||
def _minimax_qk_norm_fusion(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
tp_rank: int,
|
||||
tp_world: int,
|
||||
eps: float,
|
||||
workspace: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert qkv.ndim == 2
|
||||
num_tokens = qkv.shape[0]
|
||||
if (
|
||||
workspace is not None
|
||||
and tp_world > 1
|
||||
and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM
|
||||
and _MINIMAX_FUSED_AR_RMS_QK is not None
|
||||
):
|
||||
return _MINIMAX_FUSED_AR_RMS_QK(
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
workspace,
|
||||
q_size,
|
||||
kv_size,
|
||||
tp_rank,
|
||||
tp_world,
|
||||
eps,
|
||||
)
|
||||
# ROCm AITER fused QK-norm + AllReduce; skips unaligned shapes.
|
||||
if tp_world > 1 and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM:
|
||||
pack_size = 16 // qkv.element_size()
|
||||
warp_work_size = 32 * pack_size
|
||||
if q_size % warp_work_size == 0 and kv_size % warp_work_size == 0:
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
if aiter_ar is not None:
|
||||
q_out, k_out, _ = aiter_ar.aiter_ca.custom_fused_qknorm_ar(
|
||||
qkv, q_weight, k_weight, eps
|
||||
)
|
||||
return q_out, k_out
|
||||
return _minimax_qk_norm_tp_fallback(
|
||||
qkv, q_weight, k_weight, q_size, kv_size, tp_rank, tp_world, eps
|
||||
)
|
||||
|
||||
|
||||
def _minimax_qk_norm_fusion_fake(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
tp_rank: int,
|
||||
tp_world: int,
|
||||
eps: float,
|
||||
workspace: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert qkv.ndim == 2
|
||||
num_tokens = qkv.shape[0]
|
||||
return (
|
||||
torch.empty([num_tokens, q_size], dtype=qkv.dtype, device=qkv.device),
|
||||
torch.empty([num_tokens, kv_size], dtype=qkv.dtype, device=qkv.device),
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="minimax_qk_norm_fusion",
|
||||
op_func=_minimax_qk_norm_fusion,
|
||||
fake_impl=_minimax_qk_norm_fusion_fake,
|
||||
mutates_args=[],
|
||||
)
|
||||
|
||||
|
||||
@CustomOp.register("minimax_text01_rmsnorm_tp")
|
||||
class MiniMaxText01RMSNormTP(CustomOp):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
eps: float = 1e-6,
|
||||
*,
|
||||
weight_shard_world_size: int | None = None,
|
||||
weight_shard_rank: int | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_world = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
self.weight_shard_world = weight_shard_world_size or self.tp_world
|
||||
self.weight_shard_rank = (
|
||||
self.tp_rank if weight_shard_rank is None else weight_shard_rank
|
||||
)
|
||||
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size // self.weight_shard_world))
|
||||
self.weight.weight_loader = partial(
|
||||
self.weight_loader,
|
||||
shard_world_size=self.weight_shard_world,
|
||||
shard_rank=self.weight_shard_rank,
|
||||
)
|
||||
self.variance_epsilon = eps
|
||||
|
||||
self.workspace = None
|
||||
if _MINIMAX_FUSED_AR_RMS_QK is not None and self.tp_world > 1:
|
||||
from .lamport_workspace import (
|
||||
get_allreduce_workspace,
|
||||
)
|
||||
|
||||
# The Lamport workspace exchanges CUDA IPC handles and enables peer
|
||||
# access between GPUs. This requires P2P (IPC peer access) to be
|
||||
# available; on topologies where it is not (e.g. consumer PCIe cards
|
||||
# with P2P disabled in the driver), allocation raises. Fall back to
|
||||
# the eager allreduce + RMSNorm path instead of failing model load.
|
||||
try:
|
||||
self.workspace = get_allreduce_workspace(
|
||||
rank=self.tp_rank,
|
||||
world_size=self.tp_world,
|
||||
max_tokens=MINIMAX_QK_NORM_MAX_TOKEN_NUM,
|
||||
process_group=get_tp_group().cpu_group,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning_once(
|
||||
"Failed to initialize MiniMax fused allreduce+RMSNorm "
|
||||
"Lamport workspace: %s. This is expected on GPUs without "
|
||||
"P2P (IPC peer access) support. Falling back to the eager "
|
||||
"allreduce + RMSNorm path.",
|
||||
e,
|
||||
)
|
||||
self.workspace = None
|
||||
|
||||
@staticmethod
|
||||
def weight_loader(
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_world_size: int | None = None,
|
||||
shard_rank: int | None = None,
|
||||
) -> None:
|
||||
if shard_world_size is None:
|
||||
shard_world_size = get_tensor_model_parallel_world_size()
|
||||
if shard_rank is None:
|
||||
shard_rank = get_tensor_model_parallel_rank()
|
||||
|
||||
shard_size = loaded_weight.shape[0] // shard_world_size
|
||||
shard = slice(shard_rank * shard_size, (shard_rank + 1) * shard_size)
|
||||
param.data.copy_(loaded_weight[shard])
|
||||
|
||||
def _forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
orig_dtype = x.dtype
|
||||
x = x.to(torch.float32)
|
||||
variance = x.pow(2).mean(dim=-1, keepdim=True, dtype=torch.float32)
|
||||
if self.tp_world > 1:
|
||||
variance = _all_reduce_variance(variance) / self.tp_world
|
||||
x = x * torch.rsqrt(variance + self.variance_epsilon)
|
||||
x = (x * self.weight).to(orig_dtype)
|
||||
return x
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
assert residual is None, "RMSNorm does not support residual connection."
|
||||
return self._forward(x)
|
||||
|
||||
@staticmethod
|
||||
def forward_qk(
|
||||
q_norm: "MiniMaxText01RMSNormTP",
|
||||
k_norm: "MiniMaxText01RMSNormTP",
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
orig_dtype = q.dtype
|
||||
q = q.to(torch.float32)
|
||||
k = k.to(torch.float32)
|
||||
q_var = q.pow(2).mean(dim=-1, keepdim=True)
|
||||
k_var = k.pow(2).mean(dim=-1, keepdim=True)
|
||||
|
||||
q = q * torch.rsqrt(q_var + q_norm.variance_epsilon) * q_norm.weight
|
||||
k = k * torch.rsqrt(k_var + k_norm.variance_epsilon) * k_norm.weight
|
||||
q = q.to(orig_dtype)
|
||||
k = k.to(orig_dtype)
|
||||
return q, k
|
||||
|
||||
@staticmethod
|
||||
def forward_qkv(
|
||||
q_norm: "MiniMaxText01RMSNormTP",
|
||||
k_norm: "MiniMaxText01RMSNormTP",
|
||||
qkv: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
assert qkv.ndim == 2
|
||||
|
||||
assert q_norm.variance_epsilon == k_norm.variance_epsilon
|
||||
# Case 0: tp_size=1
|
||||
if get_tensor_model_parallel_world_size() == 1:
|
||||
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
q, k = MiniMaxText01RMSNormTP.forward_qk(q_norm, k_norm, q, k)
|
||||
return q, k, v
|
||||
# Case : tp_size>1
|
||||
q, k = torch.ops.vllm.minimax_qk_norm_fusion(
|
||||
qkv,
|
||||
q_norm.weight,
|
||||
k_norm.weight,
|
||||
q_size,
|
||||
kv_size,
|
||||
q_norm.tp_rank,
|
||||
q_norm.tp_world,
|
||||
q_norm.variance_epsilon,
|
||||
q_norm.workspace,
|
||||
)
|
||||
_, _, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
return q, k, v
|
||||
Reference in New Issue
Block a user