[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
@@ -1534,7 +1534,7 @@ class MLACommonBackend(AttentionBackend):
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [320, 576]
|
||||
return [320, 512, 576]
|
||||
|
||||
@classmethod
|
||||
def is_mla(cls) -> bool:
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _scatter_states_kernel(
|
||||
state_ptr,
|
||||
src_ptr,
|
||||
indices_ptr,
|
||||
stride_state_batch,
|
||||
stride_src_batch,
|
||||
stride_indices,
|
||||
row_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
block_idx = tl.program_id(0)
|
||||
batch_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < row_size
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
|
||||
values = tl.load(src_ptr + batch_idx * stride_src_batch + offsets, mask=mask)
|
||||
tl.store(state_ptr + state_idx * stride_state_batch + offsets, values, mask=mask)
|
||||
|
||||
|
||||
def scatter_states(
|
||||
state: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Scatter ``src`` rows into ``state`` at ``indices`` (in place).
|
||||
|
||||
Equivalent to ``state[indices] = src`` but non-atomic and bandwidth-bound,
|
||||
since mamba cache slots are unique per sequence. ``gather_initial_states``
|
||||
is the read-side counterpart.
|
||||
"""
|
||||
assert state.ndim >= 2
|
||||
assert state.is_cuda
|
||||
assert src.ndim == state.ndim
|
||||
assert indices.ndim == 1
|
||||
assert indices.device == state.device
|
||||
assert src.shape[1:] == state.shape[1:]
|
||||
assert src.shape[0] == indices.shape[0]
|
||||
assert indices.dtype in (torch.int32, torch.int64)
|
||||
|
||||
row_size = state[0].numel()
|
||||
assert state[0].is_contiguous()
|
||||
assert src[0].is_contiguous()
|
||||
block_size = min(triton.next_power_of_2(row_size), 1024)
|
||||
grid = (triton.cdiv(row_size, block_size), indices.numel())
|
||||
_scatter_states_kernel[grid](
|
||||
state,
|
||||
src,
|
||||
indices,
|
||||
state.stride(0),
|
||||
src.stride(0),
|
||||
indices.stride(0),
|
||||
row_size=row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import inspect
|
||||
|
||||
import torch
|
||||
|
||||
# this import will also register the custom ops
|
||||
@@ -29,6 +31,67 @@ HAS_TILELANG_MHC = _has_tilelang_mhc()
|
||||
HAS_AITER_MHC = is_aiter_found_and_supported()
|
||||
|
||||
|
||||
def _has_aiter_mhc_fused() -> bool:
|
||||
if not HAS_AITER_MHC:
|
||||
return False
|
||||
try:
|
||||
from aiter.ops.mhc import mhc_fused_post_pre
|
||||
except Exception:
|
||||
return False
|
||||
return callable(mhc_fused_post_pre)
|
||||
|
||||
|
||||
def _aiter_mhc_op_accepts_norm(op_name: str) -> bool:
|
||||
if not HAS_AITER_MHC:
|
||||
return False
|
||||
try:
|
||||
from aiter.ops import mhc as aiter_mhc
|
||||
|
||||
for candidate_name in (op_name, f"{op_name}_fake"):
|
||||
op = getattr(aiter_mhc, candidate_name, None)
|
||||
if op is None:
|
||||
continue
|
||||
parameters = inspect.signature(op).parameters
|
||||
if "norm_weight" in parameters and "norm_eps" in parameters:
|
||||
return True
|
||||
except (AttributeError, ImportError, TypeError, ValueError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
HAS_AITER_MHC_FUSED = _has_aiter_mhc_fused()
|
||||
HAS_AITER_MHC_PRE_NORM = _aiter_mhc_op_accepts_norm("mhc_pre")
|
||||
HAS_AITER_MHC_FUSED_NORM = _aiter_mhc_op_accepts_norm("mhc_fused_post_pre")
|
||||
|
||||
|
||||
def _aiter_mhc_supported(
|
||||
residual: torch.Tensor,
|
||||
norm_weight: torch.Tensor | None,
|
||||
*,
|
||||
supports_norm: bool,
|
||||
) -> bool:
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult = residual.shape[-2]
|
||||
return (
|
||||
HAS_AITER_MHC
|
||||
and hidden_size % 256 == 0
|
||||
and hc_mult == 4
|
||||
and (norm_weight is None or supports_norm)
|
||||
)
|
||||
|
||||
|
||||
def _apply_mhc_norm(
|
||||
layer_input: torch.Tensor,
|
||||
norm_weight: torch.Tensor | None,
|
||||
norm_eps: float,
|
||||
) -> torch.Tensor:
|
||||
if norm_weight is None:
|
||||
return layer_input
|
||||
from vllm import ir
|
||||
|
||||
return ir.ops.rms_norm(layer_input, norm_weight, norm_eps)
|
||||
|
||||
|
||||
# --8<-- [start:mhc_pre]
|
||||
@CustomOp.register("mhc_pre")
|
||||
class MHCPreOp(CustomOp):
|
||||
@@ -89,9 +152,11 @@ class MHCPreOp(CustomOp):
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult = residual.shape[-2]
|
||||
if HAS_AITER_MHC and hidden_size % 256 == 0 and hc_mult == 4:
|
||||
if _aiter_mhc_supported(
|
||||
residual,
|
||||
norm_weight,
|
||||
supports_norm=HAS_AITER_MHC_PRE_NORM,
|
||||
):
|
||||
return torch.ops.vllm.mhc_pre_aiter(
|
||||
residual,
|
||||
fn,
|
||||
@@ -122,7 +187,7 @@ class MHCPreOp(CustomOp):
|
||||
norm_eps,
|
||||
)
|
||||
else:
|
||||
return self.forward_native(
|
||||
post_mix, comb_mix, layer_input = self.forward_native(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
@@ -136,6 +201,11 @@ class MHCPreOp(CustomOp):
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
return (
|
||||
post_mix,
|
||||
comb_mix,
|
||||
_apply_mhc_norm(layer_input, norm_weight, norm_eps),
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
@@ -225,9 +295,7 @@ class MHCPostOp(CustomOp):
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult = residual.shape[-2]
|
||||
if HAS_AITER_MHC and hidden_size % 256 == 0 and hc_mult == 4:
|
||||
if _aiter_mhc_supported(residual, None, supports_norm=True):
|
||||
return torch.ops.vllm.mhc_post_aiter(
|
||||
x,
|
||||
residual,
|
||||
@@ -449,9 +517,11 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult = residual.shape[-2]
|
||||
if HAS_AITER_MHC and hidden_size % 256 == 0 and hc_mult == 4:
|
||||
if HAS_AITER_MHC_FUSED and _aiter_mhc_supported(
|
||||
residual,
|
||||
norm_weight,
|
||||
supports_norm=HAS_AITER_MHC_FUSED_NORM,
|
||||
):
|
||||
return torch.ops.vllm.mhc_fused_post_pre_aiter(
|
||||
x,
|
||||
residual,
|
||||
@@ -489,7 +559,7 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
return self.forward_native(
|
||||
residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur = self.forward_native(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
@@ -507,6 +577,12 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
return (
|
||||
residual_cur,
|
||||
post_mix_cur,
|
||||
comb_mix_cur,
|
||||
_apply_mhc_norm(layer_input_cur, norm_weight, norm_eps),
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
@@ -577,3 +653,13 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
|
||||
def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
"""[s, hidden_size] -> [s, n * hidden_size] by replication."""
|
||||
return x.unsqueeze(1).expand(-1, n, -1).contiguous()
|
||||
|
||||
|
||||
def hc_contract(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
"""[s, n * hidden_size] -> [s, hidden_size] by averaging."""
|
||||
return x.mean(dim=1)
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.config import CacheConfig
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -17,7 +18,7 @@ class MLAModules:
|
||||
|
||||
kv_a_layernorm: torch.nn.Module
|
||||
kv_b_proj: torch.nn.Module
|
||||
rotary_emb: torch.nn.Module
|
||||
rotary_emb: torch.nn.Module | None
|
||||
o_proj: torch.nn.Module
|
||||
fused_qkv_a_proj: torch.nn.Module | None
|
||||
kv_a_proj_with_mqa: torch.nn.Module | None
|
||||
@@ -69,6 +70,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
skip_topk: bool = False,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
allow_short_prefill_indexer_scoring_skip: bool = False,
|
||||
fuse_qkv_rmsnorm: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -98,6 +100,9 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
# the topk_tokens buffer written by a previous layer in the same pass.
|
||||
# Refer: https://arxiv.org/abs/2603.12201 for more details.
|
||||
self.skip_topk = skip_topk
|
||||
# When True, fuse the q_a and kv_a RMSNorms into a single kernel launch
|
||||
# (MLA layers with q-LoRA). Opt-in; default False preserves other models.
|
||||
self.fuse_qkv_rmsnorm = fuse_qkv_rmsnorm
|
||||
# qrep is active when the query projection is a DCP-group-sharded layer
|
||||
# that materializes the full group head set locally.
|
||||
q_proj_layer = self.q_b_proj if self.q_lora_rank is not None else self.q_proj
|
||||
@@ -172,8 +177,9 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
|
||||
dim=-1,
|
||||
)
|
||||
q_c = self.q_a_layernorm(q_c)
|
||||
q_proj_layer = self.q_b_proj
|
||||
if not self.fuse_qkv_rmsnorm:
|
||||
q_c = self.q_a_layernorm(q_c)
|
||||
q_proj_input = q_c
|
||||
else:
|
||||
assert self.kv_a_proj_with_mqa is not None, (
|
||||
@@ -187,7 +193,20 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
q_proj_input = hidden_states
|
||||
|
||||
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
|
||||
kv_c_normed = self.kv_a_layernorm(kv_c)
|
||||
if self.fuse_qkv_rmsnorm and q_c is not None:
|
||||
# Fuse q_a + kv_a RMSNorm into one kernel launch (q_c is still the
|
||||
# pre-norm projection output here).
|
||||
assert self.q_a_layernorm is not None
|
||||
q_proj_input, kv_c_normed = fused_q_kv_rmsnorm(
|
||||
q_c,
|
||||
kv_c,
|
||||
self.q_a_layernorm.weight.data,
|
||||
self.kv_a_layernorm.weight.data,
|
||||
self.q_a_layernorm.variance_epsilon,
|
||||
)
|
||||
q_c = q_proj_input
|
||||
else:
|
||||
kv_c_normed = self.kv_a_layernorm(kv_c)
|
||||
# Add head dim of 1 to k_pe
|
||||
k_pe = k_pe.unsqueeze(1)
|
||||
|
||||
|
||||
@@ -361,30 +361,27 @@ def _shuffle_deepseek_fp8_moe_weights(
|
||||
Returns 4D weight tensors in BlockMajorK layout
|
||||
(E, K/block_k, Mn, block_k)
|
||||
"""
|
||||
from flashinfer import shuffle_matrix_a
|
||||
from flashinfer.fused_moe import convert_to_block_layout
|
||||
from flashinfer.utils import get_shuffle_matrix_a_row_indices
|
||||
|
||||
epilogue_tile_m = 64
|
||||
block_k = 128
|
||||
num_experts = w13.shape[0]
|
||||
|
||||
M13, K13 = w13.shape[1], w13.shape[2]
|
||||
M2, K2 = w2.shape[1], w2.shape[2]
|
||||
w13_out = torch.empty(
|
||||
num_experts, K13 // block_k, M13, block_k, dtype=torch.uint8, device=w13.device
|
||||
)
|
||||
w2_out = torch.empty(
|
||||
num_experts, K2 // block_k, M2, block_k, dtype=torch.uint8, device=w2.device
|
||||
)
|
||||
def shuffle_to_block_major_k(w: torch.Tensor) -> torch.Tensor:
|
||||
# shuffle_matrix_a's row permutation depends only on (M,
|
||||
# epilogue_tile_m), so it is computed once and applied to every expert
|
||||
# in a single gather instead of once per expert. Gathering through the
|
||||
# BlockMajorK-permuted view also folds convert_to_block_layout into
|
||||
# that same kernel. Per-expert loops here cost minutes for a MoE this
|
||||
# wide (~24k tiny launches plus a host round-trip each).
|
||||
num_experts, m, k = w.shape
|
||||
rows = get_shuffle_matrix_a_row_indices(
|
||||
w[0].view(torch.uint8), epilogue_tile_m
|
||||
).to(w.device)
|
||||
blocked = w.view(torch.uint8).view(num_experts, m, k // block_k, block_k)
|
||||
out = blocked.permute(0, 2, 1, 3)[:, :, rows, :].contiguous()
|
||||
return out.view(torch.float8_e4m3fn)
|
||||
|
||||
for i in range(num_experts):
|
||||
t13 = shuffle_matrix_a(w13[i].view(torch.uint8), epilogue_tile_m)
|
||||
w13_out[i] = convert_to_block_layout(t13, block_k)
|
||||
|
||||
t2 = shuffle_matrix_a(w2[i].view(torch.uint8), epilogue_tile_m)
|
||||
w2_out[i] = convert_to_block_layout(t2, block_k)
|
||||
|
||||
return w13_out.view(torch.float8_e4m3fn), w2_out.view(torch.float8_e4m3fn)
|
||||
return shuffle_to_block_major_k(w13), shuffle_to_block_major_k(w2)
|
||||
|
||||
|
||||
def _shuffle_mxfp8_moe_weights(
|
||||
@@ -396,70 +393,54 @@ def _shuffle_mxfp8_moe_weights(
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Preprocess MXFP8 weights and scales for the FlashInfer TRT-LLM kernel.
|
||||
|
||||
Following flashinfer/tests/moe/test_trtllm_gen_fused_moe.py:
|
||||
1. reorder_rows_for_gated_act_gemm (interleave gate/up rows)
|
||||
2. shuffle_matrix_a (weight data layout shuffle)
|
||||
3. shuffle_matrix_sf_a (scale factor layout shuffle)
|
||||
All three transforms (gate/up row reorder, ``shuffle_matrix_a`` weight
|
||||
shuffle, ``shuffle_matrix_sf_a`` scale shuffle) are fixed row/index
|
||||
permutations that depend only on the per-expert matrix shape, so the
|
||||
permutation is computed once and applied to every expert in a single
|
||||
gather instead of once per expert. ``block_scale_interleave`` accepts a
|
||||
batched ``(E, M, K)`` scale tensor directly. Output is bit-identical to
|
||||
the per-expert loop but ~20x faster (a 288-expert MoE otherwise costs
|
||||
seconds per layer at load).
|
||||
"""
|
||||
from flashinfer import (
|
||||
reorder_rows_for_gated_act_gemm,
|
||||
shuffle_matrix_a,
|
||||
shuffle_matrix_sf_a,
|
||||
from flashinfer.fused_moe.core import (
|
||||
get_reorder_rows_for_gated_act_gemm_row_indices,
|
||||
)
|
||||
from flashinfer.quantization.fp4_quantization import block_scale_interleave
|
||||
from flashinfer.utils import get_shuffle_matrix_a_row_indices
|
||||
|
||||
epilogue_tile_m = 128
|
||||
num_experts = w13.shape[0]
|
||||
intermediate_size = w13.shape[1] // 2
|
||||
hidden_size = w13.shape[2]
|
||||
|
||||
w13_interleaved: list[torch.Tensor] = []
|
||||
w13_scale_interleaved: list[torch.Tensor] = []
|
||||
for i in range(num_experts):
|
||||
if is_gated:
|
||||
w13_interleaved.append(
|
||||
reorder_rows_for_gated_act_gemm(
|
||||
w13[i].reshape(2 * intermediate_size, -1)
|
||||
)
|
||||
)
|
||||
w13_scale_interleaved.append(
|
||||
reorder_rows_for_gated_act_gemm(
|
||||
w13_scale[i].reshape(2 * intermediate_size, -1)
|
||||
)
|
||||
)
|
||||
else:
|
||||
w13_interleaved.append(w13[i])
|
||||
w13_scale_interleaved.append(w13_scale[i])
|
||||
w13_u = w13.view(torch.uint8)
|
||||
w2_u = w2.view(torch.uint8)
|
||||
|
||||
w13_shuffled: list[torch.Tensor] = []
|
||||
w2_shuffled: list[torch.Tensor] = []
|
||||
w13_scale_shuffled: list[torch.Tensor] = []
|
||||
w2_scale_shuffled: list[torch.Tensor] = []
|
||||
for i in range(num_experts):
|
||||
w13_shuffled.append(
|
||||
shuffle_matrix_a(w13_interleaved[i].view(torch.uint8), epilogue_tile_m)
|
||||
)
|
||||
w2_shuffled.append(shuffle_matrix_a(w2[i].view(torch.uint8), epilogue_tile_m))
|
||||
w13_scale_shuffled.append(
|
||||
shuffle_matrix_sf_a(
|
||||
w13_scale_interleaved[i]
|
||||
.view(torch.uint8)
|
||||
.reshape(2 * intermediate_size, -1),
|
||||
epilogue_tile_m,
|
||||
)
|
||||
)
|
||||
w2_scale_shuffled.append(
|
||||
shuffle_matrix_sf_a(
|
||||
w2_scale[i].view(torch.uint8).reshape(hidden_size, -1),
|
||||
epilogue_tile_m,
|
||||
)
|
||||
# 1. Interleave gate/up rows (gated activation GEMM layout).
|
||||
if is_gated:
|
||||
gate_idx = get_reorder_rows_for_gated_act_gemm_row_indices(
|
||||
w13_u[0].reshape(w13_u.shape[1], -1)
|
||||
)
|
||||
w13_u = w13_u[:, gate_idx]
|
||||
w13_scale = w13_scale[:, gate_idx]
|
||||
|
||||
w13_out = torch.stack(w13_shuffled).view(torch.float8_e4m3fn)
|
||||
w2_out = torch.stack(w2_shuffled).view(torch.float8_e4m3fn)
|
||||
w13_scale_out = torch.stack(w13_scale_shuffled).reshape(w13_scale.shape)
|
||||
w2_scale_out = torch.stack(w2_scale_shuffled).reshape(w2_scale.shape)
|
||||
def shuffle_weights(t: torch.Tensor) -> torch.Tensor:
|
||||
# Row permutation depends only on (M, epilogue_tile_m).
|
||||
idx = get_shuffle_matrix_a_row_indices(t[0], epilogue_tile_m).to(t.device)
|
||||
return t[:, idx].view(torch.float8_e4m3fn)
|
||||
|
||||
return w13_out, w2_out, w13_scale_out, w2_scale_out
|
||||
def shuffle_scales(s: torch.Tensor) -> torch.Tensor:
|
||||
# shuffle_matrix_sf_a == row-gather (same indices as the weight shuffle)
|
||||
# followed by the 128x4 block-scale interleave, which is batch-capable.
|
||||
idx = get_shuffle_matrix_a_row_indices(
|
||||
s[0].view(torch.uint8).reshape(s.shape[1], -1), epilogue_tile_m
|
||||
).to(s.device)
|
||||
interleaved = block_scale_interleave(s[:, idx])
|
||||
return interleaved.reshape(s.shape)
|
||||
|
||||
return (
|
||||
shuffle_weights(w13_u),
|
||||
shuffle_weights(w2_u),
|
||||
shuffle_scales(w13_scale),
|
||||
shuffle_scales(w2_scale),
|
||||
)
|
||||
|
||||
|
||||
def prepare_fp8_moe_layer_for_fi(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -120,6 +120,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
|
||||
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
|
||||
"GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"),
|
||||
"Glm5NextForCausalLM": ("vllm.models.glm5next", "Glm5NextForCausalLM"),
|
||||
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
|
||||
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
|
||||
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
|
||||
@@ -413,6 +414,10 @@ _MULTIMODAL_MODELS = {
|
||||
"Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
|
||||
"Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
|
||||
"GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"),
|
||||
"Glm5NextForConditionalGeneration": (
|
||||
"vllm.models.glm5next",
|
||||
"Glm5NextForConditionalGeneration",
|
||||
),
|
||||
"GraniteSpeechForConditionalGeneration": (
|
||||
"granite_speech",
|
||||
"GraniteSpeechForConditionalGeneration",
|
||||
@@ -666,6 +671,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
|
||||
"Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
|
||||
"GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
|
||||
"Glm5NextMTPModel": ("vllm.models.glm5next", "Glm5NextMTP"),
|
||||
"MedusaModel": ("medusa", "Medusa"),
|
||||
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
|
||||
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
|
||||
|
||||
@@ -41,6 +41,9 @@ from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
|
||||
from vllm.model_executor.warmup.replayssm_warmup import (
|
||||
replayssm_autotune_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.spec_decode_rejection_warmup import (
|
||||
spec_decode_rejection_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
@@ -163,6 +166,7 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
||||
kimi_k3_triton_warmup(worker)
|
||||
fa4_cutedsl_warmup(worker)
|
||||
spec_decode_rejection_warmup(worker)
|
||||
qwen4_exp_qsa_triton_warmup(worker)
|
||||
|
||||
if current_platform.has_device_capability(90):
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up spec-decode rejection-sampler Triton kernels.
|
||||
|
||||
The rejection sampler kernels (``_compute_local_logits_stats_kernel``,
|
||||
``_rejection_kernel``, ``_resample_kernel``) are JIT-compiled by Triton on
|
||||
first use. Without warmup, the first spec-decode request pays a multi-second
|
||||
compilation cost. This pre-compiles them with dummy data matching the
|
||||
server's vocab size and speculative config.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def spec_decode_rejection_warmup(worker: Worker) -> None:
|
||||
spec_config = worker.vllm_config.speculative_config
|
||||
if spec_config is None:
|
||||
return
|
||||
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
|
||||
rejection_sample,
|
||||
)
|
||||
|
||||
model_config = worker.vllm_config.model_config
|
||||
vocab_size = model_config.get_vocab_size()
|
||||
num_spec = spec_config.num_speculative_tokens
|
||||
if num_spec <= 0 or vocab_size <= 0:
|
||||
return
|
||||
|
||||
# Mirror the constexpr-relevant flags the runtime uses.
|
||||
rejection_method = getattr(spec_config, "rejection_sample_method", None)
|
||||
use_block_verification = rejection_method == "block"
|
||||
use_synthetic = rejection_method == "synthetic"
|
||||
|
||||
device = torch.device("cuda")
|
||||
num_reqs = 1
|
||||
tokens_per_req = num_spec + 1
|
||||
num_logits = num_reqs * tokens_per_req
|
||||
|
||||
# Triton JIT-specializes on tensor dtypes. The target logits may be fp32
|
||||
# (apply_sampling_params copies to fp32 when processing is needed) or the
|
||||
# model dtype (pass-through otherwise), while draft logits are always the
|
||||
# model dtype. Warm every (target, draft) combination the runtime can hit.
|
||||
model_dtype = model_config.dtype
|
||||
warmup_dtype_pairs = {
|
||||
(model_dtype, model_dtype),
|
||||
(torch.float32, torch.float32),
|
||||
(torch.float32, model_dtype),
|
||||
(model_dtype, torch.float32),
|
||||
}
|
||||
|
||||
logger.info(
|
||||
"Warming up spec-decode rejection sampler kernels "
|
||||
"(vocab=%d, num_spec=%d, dtype_pairs=%s, block_verify=%s).",
|
||||
vocab_size,
|
||||
num_spec,
|
||||
[(str(t), str(d)) for t, d in warmup_dtype_pairs],
|
||||
use_block_verification,
|
||||
)
|
||||
for tgt_dtype, draft_dtype in warmup_dtype_pairs:
|
||||
target_logits = torch.zeros(
|
||||
(num_logits, vocab_size), dtype=tgt_dtype, device=device
|
||||
)
|
||||
draft_logits = torch.zeros(
|
||||
(num_reqs, num_spec, vocab_size), dtype=draft_dtype, device=device
|
||||
)
|
||||
synthetic_rates = (
|
||||
torch.full((num_spec,), 0.5, dtype=torch.float32, device=device)
|
||||
if use_synthetic
|
||||
else None
|
||||
)
|
||||
try:
|
||||
rejection_sample(
|
||||
target_logits=target_logits,
|
||||
draft_logits=draft_logits,
|
||||
draft_sampled=torch.zeros(num_logits, dtype=torch.int64, device=device),
|
||||
cu_num_logits=torch.tensor(
|
||||
[0, num_logits], dtype=torch.int32, device=device
|
||||
),
|
||||
pos=torch.zeros(num_logits, dtype=torch.int64, device=device),
|
||||
idx_mapping=torch.zeros(num_reqs, dtype=torch.int32, device=device),
|
||||
expanded_idx_mapping=torch.zeros(
|
||||
num_logits, dtype=torch.int32, device=device
|
||||
),
|
||||
expanded_local_pos=torch.arange(
|
||||
num_logits, dtype=torch.int32, device=device
|
||||
),
|
||||
temperature=torch.zeros(num_reqs, dtype=torch.float32, device=device),
|
||||
seed=torch.full((num_reqs,), 42, dtype=torch.int64, device=device),
|
||||
num_speculative_steps=num_spec,
|
||||
synthetic_conditional_rates=synthetic_rates,
|
||||
use_fp64=False,
|
||||
use_block_verification=use_block_verification,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Skipping spec-decode rejection sampler warmup.", exc_info=True
|
||||
)
|
||||
return
|
||||
Reference in New Issue
Block a user