[Feat] DeepSeek V4 Rebased (#40860)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Signed-off-by: qizixi <zixi@inferact.ai> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <yongye@inferact.ai> Co-authored-by: Simon Mo <simon@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Zhewen Li <jerven.vllm@gmail.com> Co-authored-by: Zijing Liu <liuzijing2014@gmail.com> Co-authored-by: khluu <khluu000@gmail.com> Co-authored-by: qizixi <zixi@inferact.ai> Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
co-authored by
Yongye Zhu
Yongye Zhu
Simon Mo
Bugen Zhao
Giancarlo Delfin
Jee Jee Li
Nick Hill
Roger Wang
Roy Wang
Woosuk Kwon
youkaichao
Zhewen Li
Zijing Liu
khluu
qizixi
Zhewen Li
parent
32e45636e3
commit
4d51588e23
@@ -100,6 +100,8 @@ class DeepGemmFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
else params.weight_scale,
|
||||
quant_block_shape=tuple(layer.weight_block_size),
|
||||
use_e8m0=self.use_deep_gemm_e8m0,
|
||||
is_bmm=getattr(layer, "is_bmm", False),
|
||||
bmm_batch_size=getattr(layer, "bmm_batch_size", 0),
|
||||
)
|
||||
replace_parameter(layer, params.WEIGHT, dg_weight)
|
||||
replace_parameter(layer, scale_attr, dg_weight_scale)
|
||||
|
||||
@@ -1422,6 +1422,20 @@ class MLADims:
|
||||
def get_mla_dims(model_config: ModelConfig) -> MLADims:
|
||||
hf_text_config = model_config.hf_text_config
|
||||
|
||||
# Check if this is a DeepseekV4 config (uses unified head_dim + rope_head_dim)
|
||||
if hasattr(hf_text_config, "compress_ratios"):
|
||||
# DeepseekV4 style config: unified head_dim with rope_head_dim
|
||||
head_dim = hf_text_config.head_dim
|
||||
rope_head_dim = hf_text_config.qk_rope_head_dim
|
||||
return MLADims(
|
||||
q_lora_rank=hf_text_config.q_lora_rank,
|
||||
kv_lora_rank=head_dim,
|
||||
qk_nope_head_dim=head_dim - rope_head_dim,
|
||||
qk_rope_head_dim=rope_head_dim,
|
||||
v_head_dim=head_dim,
|
||||
)
|
||||
|
||||
# DeepseekV2/V3 style config
|
||||
return MLADims(
|
||||
q_lora_rank=getattr(hf_text_config, "q_lora_rank", None),
|
||||
kv_lora_rank=hf_text_config.kv_lora_rank,
|
||||
@@ -2191,6 +2205,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
qk_head_dim: int,
|
||||
v_head_dim: int,
|
||||
kv_b_proj: ColumnParallelLinear,
|
||||
# DSV3.2 MLA Specific Arguments
|
||||
indexer: object | None = None,
|
||||
q_pad_num_heads: int | None = None,
|
||||
) -> None:
|
||||
@@ -2213,6 +2228,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
self.indexer = indexer
|
||||
self.q_pad_num_heads = q_pad_num_heads
|
||||
self.supports_quant_query_input = True
|
||||
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
|
||||
|
||||
# Use flashinfer's optimized concat_mla_k kernel when available.
|
||||
# The kernel is optimized for DeepSeek V3 dimensions:
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.utils import cublas_gemm_bf16_bf16_fp32
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.ops.deepseek_v4_ops.fused_compress_quant_cache import (
|
||||
_fused_kv_compress_norm_rope_insert_indexer_attn,
|
||||
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn,
|
||||
_fused_kv_compress_norm_rope_insert_sparse_attn,
|
||||
)
|
||||
from vllm.v1.attention.ops.deepseek_v4_ops.fused_indexer_q import (
|
||||
MXFP4_BLOCK_SIZE,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
)
|
||||
|
||||
|
||||
class CompressorBackend(AttentionBackend):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CompressorBackend"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [MultipleOf(1)]
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [512, 1024]
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["CompressorMetadataBuilder"]:
|
||||
return CompressorMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
assert num_kv_heads == 1
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
if include_num_layers_dimension:
|
||||
return (0, 1, 2, 3)
|
||||
return (0, 1, 2)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompressorMetadata:
|
||||
block_table: torch.Tensor
|
||||
slot_mapping: torch.Tensor
|
||||
block_size: int
|
||||
|
||||
token_to_req_indices: torch.Tensor | None = None # [num_tokens]
|
||||
|
||||
|
||||
class CompressorMetadataBuilder(AttentionMetadataBuilder):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
assert isinstance(self.kv_cache_spec, SlidingWindowMLASpec | MLAAttentionSpec)
|
||||
mla_spec = cast(SlidingWindowMLASpec | MLAAttentionSpec, self.kv_cache_spec)
|
||||
self.block_size = mla_spec.block_size
|
||||
|
||||
self.token_to_req_indices = torch.zeros(
|
||||
self.vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> CompressorMetadata:
|
||||
query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
query_lens = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
x = torch.repeat_interleave(torch.arange(num_reqs), query_lens).pin_memory()
|
||||
token_to_req_indices = self.token_to_req_indices[: x.shape[0]]
|
||||
token_to_req_indices.copy_(x, non_blocking=True)
|
||||
return CompressorMetadata(
|
||||
block_table=common_attn_metadata.block_table_tensor.clamp_(min=0),
|
||||
slot_mapping=common_attn_metadata.slot_mapping,
|
||||
block_size=self.block_size,
|
||||
token_to_req_indices=token_to_req_indices,
|
||||
)
|
||||
|
||||
|
||||
class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
|
||||
def __init__(
|
||||
self,
|
||||
state_dim: int,
|
||||
dtype: torch.dtype,
|
||||
compress_ratio: int,
|
||||
prefix: str,
|
||||
):
|
||||
super().__init__()
|
||||
self.state_dim = state_dim
|
||||
self.dtype = dtype
|
||||
self.prefix = prefix
|
||||
self.kv_cache = torch.tensor([])
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
assert self.dtype == torch.float32
|
||||
assert compress_ratio in [4, 128]
|
||||
coff = 1 + (compress_ratio == 4)
|
||||
self.sliding_window = coff * compress_ratio
|
||||
# Block size is constrained by tensor sharing between compressor states
|
||||
# and KV blocks. Since compressor states share the same physical tensor
|
||||
# as KV blocks, they must use the same page size.
|
||||
# The KV block shape [256//4, head_dim] = [64, 584] determines:
|
||||
# - C4 compressor block shape [4, 2*512*2*4] -> block_size = 4
|
||||
# - C128 compressor block shape [8, 512*2*4] -> block_size = 8
|
||||
# TODO(yifan): make block size automatically determined and configurable.
|
||||
if compress_ratio == 4:
|
||||
self.block_size = 4
|
||||
elif compress_ratio == 128:
|
||||
self.block_size = 8
|
||||
else:
|
||||
raise ValueError(f"Invalid compress ratio: {compress_ratio}")
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
return SlidingWindowMLASpec( # only has one vector instead of K + V
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.state_dim,
|
||||
dtype=self.dtype,
|
||||
sliding_window=self.sliding_window,
|
||||
alignment=576, # NOTE: FlashMLA requires 576B alignment
|
||||
)
|
||||
|
||||
def forward(self): ...
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return CompressorBackend
|
||||
|
||||
|
||||
class DeepseekCompressor(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
compress_ratio: int,
|
||||
hidden_size: int,
|
||||
head_dim: int,
|
||||
rotate: bool = False,
|
||||
prefix: str = "",
|
||||
k_cache_prefix="",
|
||||
use_fp4_cache: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.compress_ratio = compress_ratio
|
||||
self.hidden_size = hidden_size
|
||||
self.head_dim = head_dim
|
||||
self.rotate = rotate
|
||||
self.prefix = prefix
|
||||
self.k_cache_prefix = k_cache_prefix
|
||||
self.use_fp4_cache = use_fp4_cache
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.rope_head_dim = config.qk_rope_head_dim
|
||||
self.nope_head_dim = self.head_dim - self.rope_head_dim
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.device = current_platform.device_type
|
||||
self.max_num_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
|
||||
self.overlap = compress_ratio == 4
|
||||
self.coff = 1 + self.overlap
|
||||
|
||||
state_dtype = torch.float32
|
||||
self.ape = nn.Parameter(
|
||||
torch.empty(
|
||||
(compress_ratio, self.coff * self.head_dim),
|
||||
dtype=state_dtype,
|
||||
device=self.device,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
self.fused_wkv_wgate = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
[self.coff * self.head_dim, self.coff * self.head_dim],
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=None,
|
||||
disable_tp=True,
|
||||
prefix=f"{prefix}.fused_wkv_wgate",
|
||||
)
|
||||
self.norm = RMSNorm(self.head_dim, self.rms_norm_eps)
|
||||
|
||||
self.state_cache = CompressorStateCache(
|
||||
state_dim=2 * self.coff * self.head_dim, # kv_state + score_state
|
||||
dtype=state_dtype,
|
||||
compress_ratio=compress_ratio,
|
||||
prefix=f"{prefix}.state_cache",
|
||||
)
|
||||
|
||||
# Save reference to static_forward_context for forward-time KV cache lookup.
|
||||
# get_current_vllm_config() is only available during __init__, not forward.
|
||||
self._static_forward_context = (
|
||||
vllm_config.compilation_config.static_forward_context
|
||||
)
|
||||
|
||||
if self.head_dim == 512:
|
||||
assert not use_fp4_cache, (
|
||||
"MXFP4 cache is only supported for indexer (head=128)"
|
||||
)
|
||||
self._fused_kernel = _fused_kv_compress_norm_rope_insert_sparse_attn
|
||||
self._quant_block = 64
|
||||
self._token_stride = self.nope_head_dim + self.rope_head_dim * 2
|
||||
self._scale_dim = self.nope_head_dim // 64 + 1 # 7 real + 1 pad
|
||||
self._num_warps = 4
|
||||
elif self.head_dim == 128:
|
||||
if use_fp4_cache:
|
||||
self._fused_kernel = (
|
||||
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn
|
||||
)
|
||||
self._quant_block = MXFP4_BLOCK_SIZE
|
||||
self._token_stride = self.head_dim // 2
|
||||
self._scale_dim = self.head_dim // MXFP4_BLOCK_SIZE
|
||||
else:
|
||||
self._fused_kernel = _fused_kv_compress_norm_rope_insert_indexer_attn
|
||||
self._quant_block = 128
|
||||
self._token_stride = self.head_dim
|
||||
self._scale_dim = 4 # single float32 scale
|
||||
self._num_warps = 1
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported head_dim for fused quant+cache: {self.head_dim}"
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
# [num_tokens, hidden_size]
|
||||
x: torch.Tensor,
|
||||
# [num_tokens]
|
||||
positions: torch.Tensor,
|
||||
rotary_emb,
|
||||
) -> None:
|
||||
num_tokens, _ = x.shape
|
||||
# bf16 weights/activations but fp32 output for numerical stability of
|
||||
# the downstream compressor math.
|
||||
kv_score = cublas_gemm_bf16_bf16_fp32(x, self.fused_wkv_wgate.weight)
|
||||
# Each of shape [num_tokens, coff * self.head_dim]
|
||||
# input bf16, output are fp32
|
||||
kv, score = kv_score.split(
|
||||
[self.coff * self.head_dim, self.coff * self.head_dim], dim=-1
|
||||
)
|
||||
|
||||
# Get the metadata and handle dummy profiling run.
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return
|
||||
|
||||
state_metadata = cast(
|
||||
CompressorMetadata, attn_metadata[self.state_cache.prefix]
|
||||
)
|
||||
token_to_req_indices = state_metadata.token_to_req_indices
|
||||
slot_mapping = state_metadata.slot_mapping
|
||||
num_actual = slot_mapping.shape[0]
|
||||
block_table = state_metadata.block_table
|
||||
block_size = state_metadata.block_size
|
||||
|
||||
# [num_blocks, block_size, kv_dim+score_dim], where kv_dim == score_dim
|
||||
state_cache = self.state_cache.kv_cache
|
||||
# kv_state stored in first half, score_state stored in second half
|
||||
state_width = state_cache.shape[-1] // 2
|
||||
|
||||
# Store the KV and score (with fused APE addition) in the state.
|
||||
# NOTE: PDL is disabled — both this kernel and _fused_kernel below
|
||||
# depend on preceding kernel outputs (kv/score from the cublas GEMM;
|
||||
# state_cache from this kernel) but neither emits/waits on PDL grid
|
||||
# dependency primitives, so launch_pdl=True caused a read-after-write
|
||||
# race and non-deterministic output.
|
||||
_save_partial_states_kernel[(num_actual,)](
|
||||
kv,
|
||||
kv.stride(0),
|
||||
score,
|
||||
score.stride(0),
|
||||
self.ape,
|
||||
self.ape.stride(0),
|
||||
positions,
|
||||
state_cache,
|
||||
state_cache.stride(0),
|
||||
state_cache.stride(1),
|
||||
slot_mapping,
|
||||
block_size,
|
||||
HEAD_SIZE=kv.shape[-1],
|
||||
TRITON_BLOCK_SIZE=triton.next_power_of_2(kv.shape[-1]),
|
||||
STATE_WIDTH=state_width,
|
||||
COMPRESS_RATIO=self.compress_ratio,
|
||||
launch_pdl=False,
|
||||
)
|
||||
|
||||
# Fused: compress → RMSNorm → RoPE → FP8 quant → KV cache write.
|
||||
# RoPE requirements (kernel applies forward GPT-J style rotation):
|
||||
# - is_neox_style=False (interleaved pairs, NOT split-half)
|
||||
# - cos_sin_cache layout: [max_pos, rope_head_dim] with first half cos,
|
||||
# second half sin (per-pair, length rope_head_dim // 2 each)
|
||||
# - applied to LAST rope_head_dim elements of head_dim
|
||||
# - position used: (positions // compress_ratio) * compress_ratio
|
||||
cos_sin_cache = rotary_emb.cos_sin_cache
|
||||
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
|
||||
kv_cache = self._static_forward_context[self.k_cache_prefix].kv_cache
|
||||
|
||||
self._fused_kernel[(num_actual,)](
|
||||
# state cache
|
||||
state_cache,
|
||||
state_cache.stride(0),
|
||||
state_cache.stride(1),
|
||||
# metadata
|
||||
token_to_req_indices,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
block_table.stride(0),
|
||||
block_size,
|
||||
# RMSNorm
|
||||
self.norm.weight,
|
||||
self.rms_norm_eps,
|
||||
# RoPE
|
||||
cos_sin_cache,
|
||||
cos_sin_cache.stride(0),
|
||||
# KV cache
|
||||
kv_cache,
|
||||
k_cache_metadata.slot_mapping,
|
||||
kv_cache.shape[1], # paged KV cache block size (tokens per block)
|
||||
# constexprs
|
||||
HEAD_SIZE=self.head_dim,
|
||||
TRITON_BLOCK_SIZE=triton.next_power_of_2(self.head_dim),
|
||||
STATE_WIDTH=state_width,
|
||||
COMPRESS_RATIO=self.compress_ratio,
|
||||
OVERLAP=self.overlap,
|
||||
ROPE_HEAD_DIM=self.rope_head_dim,
|
||||
FP8_MAX=448.0,
|
||||
QUANT_BLOCK=self._quant_block,
|
||||
TOKEN_STRIDE=self._token_stride,
|
||||
SCALE_DIM=self._scale_dim,
|
||||
KV_BLOCK_STRIDE=kv_cache.stride(0),
|
||||
num_warps=self._num_warps,
|
||||
launch_pdl=False,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _save_partial_states_kernel(
|
||||
kv_ptr,
|
||||
kv_stride,
|
||||
score_ptr,
|
||||
score_stride,
|
||||
ape_ptr,
|
||||
ape_stride,
|
||||
positions_ptr,
|
||||
state_cache_ptr,
|
||||
state_cache_stride0,
|
||||
state_cache_stride1,
|
||||
slot_mapping_ptr,
|
||||
block_size,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
TRITON_BLOCK_SIZE: tl.constexpr,
|
||||
# state_cache last dim packs [kv_state, score_state], each STATE_WIDTH wide.
|
||||
STATE_WIDTH: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
slot_id = tl.load(slot_mapping_ptr + token_idx)
|
||||
|
||||
# Skip padded / invalid tokens (slot_id == -1 is the PAD sentinel used
|
||||
# by vLLM). During CUDA graph replay the batch may contain padding
|
||||
# tokens whose slot_mapping is -1; writing to kv_state[-1] would be an
|
||||
# illegal memory access.
|
||||
if slot_id < 0:
|
||||
return
|
||||
|
||||
block_idx = slot_id // block_size
|
||||
pos_in_block = slot_id % block_size
|
||||
base_ptr = (
|
||||
state_cache_ptr
|
||||
+ block_idx * state_cache_stride0
|
||||
+ pos_in_block * state_cache_stride1
|
||||
)
|
||||
|
||||
block = tl.arange(0, TRITON_BLOCK_SIZE)
|
||||
mask = block < HEAD_SIZE
|
||||
|
||||
kv = tl.load(kv_ptr + token_idx * kv_stride + block, mask=mask)
|
||||
tl.store(base_ptr + block, kv, mask=mask)
|
||||
|
||||
# Fused: score += ape[position % compress_ratio]
|
||||
position = tl.load(positions_ptr + token_idx)
|
||||
ape_row = position % COMPRESS_RATIO
|
||||
ape = tl.load(ape_ptr + ape_row * ape_stride + block, mask=mask)
|
||||
score = tl.load(score_ptr + token_idx * score_stride + block, mask=mask)
|
||||
tl.store(
|
||||
base_ptr + STATE_WIDTH + block,
|
||||
score + ape,
|
||||
mask=mask,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -117,8 +117,10 @@ class RoutingMethodType(IntEnum):
|
||||
Custom = (6,)
|
||||
# Simulated
|
||||
Simulated = (7,)
|
||||
# Deepseek V4 -> sqrtsoftplus + Bias + Normalize
|
||||
DeepseekV4 = (8,)
|
||||
# Unspecified
|
||||
Unspecified = 8.0
|
||||
Unspecified = 9.0
|
||||
|
||||
|
||||
def get_routing_method_type(
|
||||
@@ -128,6 +130,14 @@ def get_routing_method_type(
|
||||
num_expert_group: int | None,
|
||||
has_e_score_bias: bool,
|
||||
) -> RoutingMethodType:
|
||||
if scoring_func == "sqrtsoftplus":
|
||||
# DeepSeek V4 uses sqrtsoftplus routing with optional routing bias
|
||||
# and top-k renormalization.
|
||||
if renormalize:
|
||||
return RoutingMethodType.DeepseekV4
|
||||
else:
|
||||
return RoutingMethodType.Unspecified
|
||||
|
||||
if has_e_score_bias:
|
||||
if (num_expert_group or 0) > 0 and scoring_func == "sigmoid":
|
||||
return RoutingMethodType.DeepSeekV3
|
||||
@@ -230,6 +240,13 @@ class FusedMoEQuantConfig:
|
||||
_w2: FusedMoEQuantDesc
|
||||
is_nvfp4_scale_swizzled: bool = True
|
||||
|
||||
# MXFP4-specific TRTLLM parameters for SwiGLU activation clamping.
|
||||
# These correspond to gemm1_alpha, gemm1_beta, gemm1_clamp_limit
|
||||
# in TrtLlmMxfp4ExpertsBase.
|
||||
gemm1_alpha: float | None = None
|
||||
gemm1_beta: float | None = None
|
||||
gemm1_clamp_limit: float | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
assert not self.per_act_token_quant or self.block_shape is None, (
|
||||
"illegal quantization"
|
||||
@@ -477,6 +494,9 @@ class FusedMoEQuantConfig:
|
||||
w2_zp: torch.Tensor | None = None,
|
||||
weight_dtype: torch.dtype | str | None = None,
|
||||
is_nvfp4_scale_swizzled: bool = True,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> "FusedMoEQuantConfig":
|
||||
"""
|
||||
General builder function for a FusedMoEQuantConfig.
|
||||
@@ -507,6 +527,9 @@ class FusedMoEQuantConfig:
|
||||
- w1_zp: Optional w1 zero points for int4/int8 quantization.
|
||||
- w2_zp: Optional w2 zero points for int4/int8 quantization.
|
||||
- is_nvfp4_scale_swizzled: Whether to swizzle the nvfp4 scale swizzling.
|
||||
- gemm1_alpha: Optional MXFP4 TRTLLM SwiGLU alpha parameter.
|
||||
- gemm1_beta: Optional MXFP4 TRTLLM SwiGLU beta parameter.
|
||||
- gemm1_clamp_limit: Optional MXFP4 TRTLLM SwiGLU clamp limit.
|
||||
"""
|
||||
assert not isinstance(quant_dtype, str) or quant_dtype in {
|
||||
"nvfp4",
|
||||
@@ -540,6 +563,9 @@ class FusedMoEQuantConfig:
|
||||
weight_dtype, w_shape, w2_scale, g2_alphas, w2_zp, w2_bias
|
||||
),
|
||||
is_nvfp4_scale_swizzled=is_nvfp4_scale_swizzled,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
assert quant_config.per_act_token_quant == per_act_token_quant
|
||||
assert quant_config.per_out_ch_quant == per_out_ch_quant
|
||||
@@ -650,6 +676,9 @@ def mxfp4_w4a16_moe_quant_config(
|
||||
w2_scale: Union[torch.Tensor, "PrecisionConfig"],
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for unquantized activations and mxfp4 weights.
|
||||
@@ -659,6 +688,9 @@ def mxfp4_w4a16_moe_quant_config(
|
||||
_a2=FusedMoEQuantDesc(),
|
||||
_w1=FusedMoEQuantDesc("mxfp4", None, w1_scale, None, None, w1_bias),
|
||||
_w2=FusedMoEQuantDesc("mxfp4", None, w2_scale, None, None, w2_bias),
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
|
||||
@@ -670,6 +702,9 @@ def mxfp4_mxfp8_moe_quant_config(
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
block_shape: list[int] | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for mxfp4 activations and mxfp4 weights.
|
||||
@@ -679,6 +714,9 @@ def mxfp4_mxfp8_moe_quant_config(
|
||||
_a2=FusedMoEQuantDesc("mxfp8"),
|
||||
_w1=FusedMoEQuantDesc("mxfp4", None, w1_scale, None, None, w1_bias),
|
||||
_w2=FusedMoEQuantDesc("mxfp4", None, w2_scale, None, None, w2_bias),
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
|
||||
@@ -712,6 +750,9 @@ def ocp_mx_moe_quant_config(
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
block_shape: list[int] | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for mxfp4 activations and mxfp4 weights.
|
||||
@@ -729,6 +770,9 @@ def ocp_mx_moe_quant_config(
|
||||
per_act_token_quant=False,
|
||||
per_out_ch_quant=False,
|
||||
block_shape=block_shape,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -25,15 +25,20 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8_packed_for_deepgemm,
|
||||
silu_mul_per_token_group_quant_fp8_colmajor,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
silu_mul_quant_fp8_packed_triton as fused_silu_mul_fp8_quant_packed,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Static128BlockSym,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.utils.deep_gemm import (
|
||||
DeepGemmQuantScaleFMT,
|
||||
get_mk_alignment_for_contiguous_layout,
|
||||
is_deep_gemm_supported,
|
||||
m_grouped_fp8_fp4_gemm_nt_contiguous,
|
||||
m_grouped_fp8_gemm_nt_contiguous,
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
@@ -197,8 +202,14 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
M_sum, N = input.size()
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
|
||||
# 1. DeepGemm UE8M0: use packed per-token-group quant
|
||||
# 1. DeepGemm UE8M0: fused SiLU+mul+clamp+quant+pack
|
||||
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
|
||||
if activation == MoEActivation.SILU:
|
||||
return fused_silu_mul_fp8_quant_packed(
|
||||
input=input,
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
)
|
||||
act_out = torch.empty(
|
||||
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
|
||||
)
|
||||
@@ -312,3 +323,225 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
expert_map=expert_map,
|
||||
output=output,
|
||||
)
|
||||
|
||||
|
||||
class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
"""DeepGemm-based fused MoE expert implementation for FP4 weights.
|
||||
|
||||
Uses m_grouped_fp8_fp4_gemm_nt_contiguous with FP8 activations and
|
||||
MXFP4 (FP4 E2M1 packed as uint8) weights. Requires SM100+ (Blackwell).
|
||||
"""
|
||||
|
||||
# FP8 activation block size (hardcoded since mxfp4_w4a8 quant config
|
||||
# does not set a block_shape on the activation descriptor).
|
||||
_ACT_BLOCK_K = 128
|
||||
# FP4 weight block size
|
||||
_WEIGHT_BLOCK_K = 32
|
||||
|
||||
def __init__(self, moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig):
|
||||
super().__init__(moe_config=moe_config, quant_config=quant_config)
|
||||
assert quant_config.weight_quant_dtype == "mxfp4"
|
||||
assert not quant_config.per_act_token_quant
|
||||
assert not quant_config.per_out_ch_quant
|
||||
|
||||
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return (
|
||||
is_deep_gemm_supported()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kMxfp4Static, kFp8Dynamic128Sym),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [MoEActivation.SILU, MoEActivation.SWIGLUSTEP]
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
return True
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
block_m = get_mk_alignment_for_contiguous_layout()[0]
|
||||
M_sum = compute_aligned_M(
|
||||
M, topk, local_num_experts, block_m, expert_tokens_meta
|
||||
)
|
||||
assert M_sum % block_m == 0
|
||||
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
workspace1 = (M_sum, max(activation_out_dim, K))
|
||||
workspace2 = (M_sum, max(N, K))
|
||||
output = (M, K)
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def _act_mul_quant(
|
||||
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
block_k = self._ACT_BLOCK_K
|
||||
scale_fmt = DeepGemmQuantScaleFMT.from_oracle()
|
||||
|
||||
M_sum, N = input.size()
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
|
||||
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
|
||||
assert activation == MoEActivation.SILU
|
||||
return fused_silu_mul_fp8_quant_packed(
|
||||
input=input,
|
||||
output_q=output,
|
||||
group_size=block_k,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
if activation == MoEActivation.SILU:
|
||||
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
input=input,
|
||||
output=output,
|
||||
use_ue8m0=use_ue8m0,
|
||||
)
|
||||
|
||||
act_out = torch.empty(
|
||||
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
|
||||
)
|
||||
self.activation(activation, act_out, input)
|
||||
return per_token_group_quant_fp8(
|
||||
act_out, block_k, column_major_scales=True, out_q=output
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
assert a1q_scale is not None
|
||||
assert a2_scale is None
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
|
||||
a1q = hidden_states
|
||||
_, N, _ = w1.size()
|
||||
# K comes from activations (full hidden dim), not from w1 which is
|
||||
# packed FP4 (E, N, K//2).
|
||||
K = a1q.size(1)
|
||||
|
||||
local_num_experts = w1.size(0)
|
||||
if global_num_experts == -1:
|
||||
global_num_experts = local_num_experts
|
||||
|
||||
M_sum = compute_aligned_M(
|
||||
M=topk_ids.size(0),
|
||||
num_topk=topk_ids.size(1),
|
||||
local_num_experts=local_num_experts,
|
||||
alignment=get_mk_alignment_for_contiguous_layout()[0],
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
)
|
||||
|
||||
a1q_perm = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
||||
)
|
||||
a1q, a1q_scale, expert_ids, inv_perm = deepgemm_moe_permute(
|
||||
aq=a1q,
|
||||
aq_scale=a1q_scale,
|
||||
topk_ids=topk_ids,
|
||||
local_num_experts=local_num_experts,
|
||||
expert_map=expert_map,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
aq_out=a1q_perm,
|
||||
)
|
||||
assert a1q.size(0) == M_sum
|
||||
|
||||
# FC1: FP8 activations x FP4 weights
|
||||
# DeepGEMM 2.4.2 requires FP4-packed weights as int8 (kPackedFP4).
|
||||
mm1_out = _resize_cache(workspace2, (M_sum, N))
|
||||
m_grouped_fp8_fp4_gemm_nt_contiguous(
|
||||
(a1q, a1q_scale),
|
||||
(w1.view(torch.int8), self.w1_scale),
|
||||
mm1_out,
|
||||
expert_ids,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
# SwiGLU activation + FP8 requant
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
quant_out = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
)
|
||||
|
||||
# FC2: FP8 activations x FP4 weights
|
||||
mm2_out = _resize_cache(workspace2, (M_sum, K))
|
||||
m_grouped_fp8_fp4_gemm_nt_contiguous(
|
||||
(a2q, a2q_scale),
|
||||
(w2.view(torch.int8), self.w2_scale),
|
||||
mm2_out,
|
||||
expert_ids,
|
||||
recipe_a=(1, self._ACT_BLOCK_K),
|
||||
recipe_b=(1, self._WEIGHT_BLOCK_K),
|
||||
)
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
topk_weights = torch.ones_like(topk_weights)
|
||||
|
||||
deepgemm_unpermute_and_reduce(
|
||||
a=mm2_out,
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
inv_perm=inv_perm,
|
||||
expert_map=expert_map,
|
||||
output=output,
|
||||
)
|
||||
|
||||
@@ -28,8 +28,162 @@ from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_triton_kernels
|
||||
|
||||
from ..utils import swiglu_limit_func
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _patch_make_bitmatrix_metadata() -> None:
|
||||
"""Monkey-patch make_bitmatrix_metadata to support non-power-of-2 top_k.
|
||||
|
||||
triton's tl.arange requires a power-of-2 range. The original kernel
|
||||
computes BLOCK_SIZE = BLOCK_PER_TOK * TOKS_PER_ROW (= 32 * top_k). For
|
||||
DeepSeek-V4 with top_k=6 this gives 192, which is not a power of 2 and
|
||||
causes a compile error at the first forward pass.
|
||||
|
||||
Fix: define a drop-in replacement kernel that accepts an extra constexpr
|
||||
BLOCK_SIZE_PADDED (next power of 2 >= BLOCK_SIZE) and uses it for the
|
||||
tl.arange call while keeping the actual BLOCK_SIZE as the stride between
|
||||
thread-blocks so that all flat indices into NonzeroIndx stay correct.
|
||||
Elements beyond BLOCK_SIZE are masked out (col_indx = 0xffff) and ignored.
|
||||
|
||||
This function is called once at module load time and patches the function
|
||||
inside the triton_kernels tensor module so that SparseMatrix.__post_init__
|
||||
picks up the fixed version transparently.
|
||||
"""
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
try:
|
||||
from vllm.third_party.triton_kernels.tensor_details import (
|
||||
bitmatrix as _bm,
|
||||
)
|
||||
from vllm.third_party.triton_kernels.tensor_details.bitmatrix import (
|
||||
BitmatrixMetadata,
|
||||
_keyed_add,
|
||||
cdiv,
|
||||
)
|
||||
from vllm.third_party.triton_kernels.tensor_details.bitmatrix_details.sum_bitmatrix_rows import ( # noqa: E501
|
||||
sum_bitmatrix_rows,
|
||||
)
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
@triton.jit
|
||||
def _stage2_pow2(
|
||||
ColSortedIndx,
|
||||
RowSortedIndx,
|
||||
NonzeroIndx,
|
||||
n_tokens,
|
||||
ColPartialSum,
|
||||
stride_pm,
|
||||
stride_pn,
|
||||
ColOffs,
|
||||
TOKS_PER_ROW: tl.constexpr,
|
||||
BLOCK_PER_TOK: tl.constexpr,
|
||||
BLOCK_SIZE_PADDED: tl.constexpr,
|
||||
):
|
||||
# Actual number of elements per block (may not be a power of 2).
|
||||
BLOCK_SIZE: tl.constexpr = BLOCK_PER_TOK * TOKS_PER_ROW
|
||||
tl.static_assert(BLOCK_SIZE_PADDED <= 32768)
|
||||
if isinstance(n_tokens, tl.tensor) and n_tokens.dtype.is_ptr():
|
||||
n_tokens = tl.load(n_tokens)
|
||||
nonzero_indx_size = n_tokens * TOKS_PER_ROW
|
||||
pid_m = tl.program_id(0)
|
||||
# Use BLOCK_SIZE_PADDED (a power of 2) for tl.arange, but stride by
|
||||
# the actual BLOCK_SIZE so flat positions in NonzeroIndx are correct.
|
||||
# Elements with offs_local >= BLOCK_SIZE have offs_global beyond the
|
||||
# valid range, get col_indx = 0xffff, and are filtered by the mask
|
||||
# below without producing any output.
|
||||
offs_local = tl.arange(0, BLOCK_SIZE_PADDED)
|
||||
offs_global = pid_m * BLOCK_SIZE + offs_local
|
||||
mask = offs_global < nonzero_indx_size
|
||||
col_indx = tl.load(NonzeroIndx + offs_global, mask=mask, other=-1).to(tl.uint32)
|
||||
kv_pairs = ((col_indx << 16) | offs_local).to(tl.uint32)
|
||||
kv_pairs = tl.sort(kv_pairs, 0)
|
||||
col_indx = kv_pairs >> 16
|
||||
offs_global = pid_m * BLOCK_SIZE + (kv_pairs & 0xFFFF)
|
||||
mask = col_indx != 0xFFFF
|
||||
x = kv_pairs & 0xFFFF0000 | 0x00000001
|
||||
cols_and_inclusive_run_lengths = tl.associative_scan(x, 0, _keyed_add)
|
||||
exclusive_run_lengths = (cols_and_inclusive_run_lengths - 1) & 0xFFFF
|
||||
row_sorted_indx = tl.load(
|
||||
ColPartialSum + pid_m * stride_pm + col_indx * stride_pn, mask=mask
|
||||
)
|
||||
row_sorted_indx += tl.load(ColOffs + col_indx, mask=mask)
|
||||
row_sorted_indx += exclusive_run_lengths
|
||||
tl.store(RowSortedIndx + offs_global, row_sorted_indx, mask=mask)
|
||||
tl.store(ColSortedIndx + row_sorted_indx, offs_global, mask=mask)
|
||||
|
||||
def _make_bitmatrix_metadata_pow2_safe(nonzero_indx, bitmatrix):
|
||||
assert nonzero_indx.ndim == 2
|
||||
PARTIAL_BLOCK_M = 32
|
||||
col_sum, col_partial_sum = sum_bitmatrix_rows(
|
||||
bitmatrix, partials_block_size=PARTIAL_BLOCK_M
|
||||
)
|
||||
device = bitmatrix.device
|
||||
n_indx = nonzero_indx.numel()
|
||||
n_cols = bitmatrix.shape[1]
|
||||
col_offs = torch.empty(n_cols, dtype=torch.int32, device=device)
|
||||
combined_indx = torch.empty(n_indx * 2, dtype=torch.int32, device=device)
|
||||
col_sorted_indx = combined_indx[:n_indx]
|
||||
row_sorted_indx = combined_indx[n_indx:]
|
||||
MEMSET_BLOCK = 1024
|
||||
memset_grid = (cdiv(n_indx * 2, MEMSET_BLOCK) + n_cols + 1,)
|
||||
_bm._bitmatrix_metadata_compute_stage1[memset_grid](
|
||||
combined_indx,
|
||||
n_indx * 2,
|
||||
-1,
|
||||
MEMSET_BLOCK,
|
||||
col_sum,
|
||||
col_offs,
|
||||
col_sum.shape[0],
|
||||
col_partial_sum,
|
||||
col_partial_sum.shape[0],
|
||||
col_partial_sum.stride(0),
|
||||
col_partial_sum.stride(1),
|
||||
BLOCK_M=512,
|
||||
BLOCK_N=512,
|
||||
)
|
||||
toks_per_row = nonzero_indx.shape[-1]
|
||||
block_size = PARTIAL_BLOCK_M * toks_per_row
|
||||
# Next power of 2 >= block_size (required by tl.arange).
|
||||
block_size_padded = 1 << (max(block_size, 1) - 1).bit_length()
|
||||
compute_grid = (cdiv(bitmatrix.shape_max[0], PARTIAL_BLOCK_M),)
|
||||
_stage2_pow2[compute_grid](
|
||||
col_sorted_indx,
|
||||
row_sorted_indx,
|
||||
nonzero_indx,
|
||||
bitmatrix.shape[0],
|
||||
col_partial_sum,
|
||||
col_partial_sum.stride(0),
|
||||
col_partial_sum.stride(1),
|
||||
col_offs,
|
||||
TOKS_PER_ROW=toks_per_row,
|
||||
BLOCK_PER_TOK=PARTIAL_BLOCK_M,
|
||||
BLOCK_SIZE_PADDED=block_size_padded,
|
||||
)
|
||||
return BitmatrixMetadata(
|
||||
col_sum=col_sum,
|
||||
col_sorted_indx=col_sorted_indx,
|
||||
row_sorted_indx=row_sorted_indx,
|
||||
)
|
||||
|
||||
# The most reliable patch point: SparseMatrix.__post_init__ looks up
|
||||
# make_bitmatrix_metadata via its own __globals__ dict (the tensor.py
|
||||
# module dict). Patching through __globals__ works regardless of how
|
||||
# sys.modules maps "triton_kernels.tensor" vs
|
||||
# "vllm.third_party.triton_kernels.tensor".
|
||||
from triton_kernels.tensor import SparseMatrix as _SparseMatrix
|
||||
|
||||
_SparseMatrix.__post_init__.__globals__["make_bitmatrix_metadata"] = (
|
||||
_make_bitmatrix_metadata_pow2_safe
|
||||
)
|
||||
# Also patch the bitmatrix module itself in case it is imported directly.
|
||||
_bm.make_bitmatrix_metadata = _make_bitmatrix_metadata_pow2_safe
|
||||
|
||||
|
||||
use_legacy_triton_kernels = False
|
||||
|
||||
if has_triton_kernels():
|
||||
@@ -59,6 +213,8 @@ if has_triton_kernels():
|
||||
use_legacy_triton_kernels = True
|
||||
else:
|
||||
raise
|
||||
if not use_legacy_triton_kernels:
|
||||
_patch_make_bitmatrix_metadata()
|
||||
except (AttributeError, ImportError) as e:
|
||||
logger.error(
|
||||
"Failed to import Triton kernels. Please make sure your triton "
|
||||
@@ -497,6 +653,8 @@ class BaseOAITritonExperts(mk.FusedMoEExpertsModular):
|
||||
return False
|
||||
# (9,0) <= cap < (11,0) covers CUDA SM90 (Hopper), SM100+ (Blackwell)
|
||||
# and ROCm gfx942/gfx950 (which map to 9.4/9.5).
|
||||
if not has_triton_kernels():
|
||||
return False
|
||||
return (9, 0) <= (cap.major, cap.minor) < (11, 0)
|
||||
|
||||
@staticmethod
|
||||
@@ -698,6 +856,37 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
|
||||
def moe_sum(self, input: torch.Tensor, output: torch.Tensor):
|
||||
ops.moe_sum(input, output)
|
||||
|
||||
def activation(
|
||||
self,
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
) -> None:
|
||||
quant_config = self.quant_config or FUSED_MOE_UNQUANTIZED_CONFIG
|
||||
if activation == MoEActivation.SWIGLUOAI:
|
||||
alpha = (
|
||||
quant_config.gemm1_alpha
|
||||
if quant_config.gemm1_alpha is not None
|
||||
else 1.702
|
||||
)
|
||||
limit = (
|
||||
quant_config.gemm1_clamp_limit
|
||||
if quant_config.gemm1_clamp_limit is not None
|
||||
else 7.0
|
||||
)
|
||||
torch.ops._C.swigluoai_and_mul(output, input, alpha, limit)
|
||||
elif (
|
||||
activation == MoEActivation.SILU
|
||||
and quant_config.gemm1_clamp_limit is not None
|
||||
):
|
||||
swiglu_limit_func(
|
||||
output,
|
||||
input,
|
||||
quant_config.gemm1_clamp_limit,
|
||||
)
|
||||
else:
|
||||
super().activation(activation, output, input)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
@@ -812,9 +1001,9 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
|
||||
act_input,
|
||||
)
|
||||
|
||||
# matmul_ogs grouped reduction fuse sum across multiple experts:
|
||||
# matmul_ogs grouped reduction fuses sum across multiple experts:
|
||||
# y[dst_indx // n_expts_act, :] += x
|
||||
# Need to set n_expts_act to 1 to unfuse moe_sum
|
||||
# Set n_expts_act to 1 to unfuse the sum so we can do it manually via moe_sum.
|
||||
routing_data.n_expts_act = 1
|
||||
|
||||
matmul_ogs(
|
||||
@@ -878,6 +1067,8 @@ class OAITritonMxfp4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
return False
|
||||
# (9,0) <= cap < (11,0) covers CUDA SM90 (Hopper), SM100+ (Blackwell)
|
||||
# and ROCm gfx942/gfx950 (which map to 9.4/9.5).
|
||||
if not has_triton_kernels():
|
||||
return False
|
||||
return (9, 0) <= (cap.major, cap.minor) < (11, 0)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import trtllm_moe_pack_topk_ids_weights
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
@@ -32,10 +33,8 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
**kwargs,
|
||||
):
|
||||
# NOTE: FusedMoEExperts.__init__ is called by the concrete subclass
|
||||
# (Monolithic/Modular) via MRO, not here, to avoid mypy issues with
|
||||
# multiple inheritance. This matches the NvFP4 expert pattern.
|
||||
self.moe_config = moe_config
|
||||
self.quant_config = quant_config
|
||||
|
||||
@@ -48,23 +47,34 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
self.local_num_experts = moe_config.num_local_experts
|
||||
self.ep_rank = moe_config.moe_parallel_config.ep_rank
|
||||
|
||||
# MXFP4-specific TRTLLM parameters
|
||||
# MXFP4-specific TRTLLM parameters from quant_config
|
||||
device = torch.accelerator.current_device_index()
|
||||
self.gemm1_alpha = torch.tensor(
|
||||
[1.702] * self.local_num_experts,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self.gemm1_beta = torch.tensor(
|
||||
[1.0] * self.local_num_experts,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self.gemm1_clamp_limit = torch.tensor(
|
||||
[7.0] * self.local_num_experts,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
if quant_config.gemm1_alpha is not None:
|
||||
self.gemm1_alpha = torch.tensor(
|
||||
[quant_config.gemm1_alpha] * self.local_num_experts,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
self.gemm1_alpha = None
|
||||
|
||||
if quant_config.gemm1_beta is not None:
|
||||
self.gemm1_beta = torch.tensor(
|
||||
[quant_config.gemm1_beta] * self.local_num_experts,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
self.gemm1_beta = None
|
||||
|
||||
if quant_config.gemm1_clamp_limit is not None:
|
||||
self.gemm1_clamp_limit = torch.tensor(
|
||||
[quant_config.gemm1_clamp_limit] * self.local_num_experts,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
self.gemm1_clamp_limit = None
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
@@ -97,7 +107,7 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation == MoEActivation.SWIGLUOAI
|
||||
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -190,36 +200,41 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
|
||||
output = torch.empty_like(hidden_states)
|
||||
|
||||
return trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits.to(torch.bfloat16),
|
||||
routing_bias=None,
|
||||
hidden_states=x_quant,
|
||||
hidden_states_scale=x_scale,
|
||||
gemm1_weights=w1,
|
||||
gemm1_weights_scale=self.w1_scale,
|
||||
gemm1_bias=self.w1_bias,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.w2_scale,
|
||||
gemm2_bias=self.w2_bias,
|
||||
output1_scale_scalar=None,
|
||||
output1_scale_gate_scalar=None,
|
||||
output2_scale_scalar=None,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.ep_rank * self.local_num_experts,
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
routing_method_type=self.routing_method_type,
|
||||
do_finalize=True,
|
||||
tune_max_num_tokens=max(self.max_capture_size, 1),
|
||||
output=output,
|
||||
)[0]
|
||||
from vllm.utils.flashinfer import _is_fi_autotuning, autotune
|
||||
|
||||
with autotune(_is_fi_autotuning):
|
||||
trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits.to(torch.bfloat16),
|
||||
routing_bias=None,
|
||||
hidden_states=x_quant,
|
||||
hidden_states_scale=x_scale,
|
||||
gemm1_weights=w1,
|
||||
gemm1_weights_scale=self.w1_scale,
|
||||
gemm1_bias=self.w1_bias,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.w2_scale,
|
||||
gemm2_bias=self.w2_bias,
|
||||
output1_scale_scalar=None,
|
||||
output1_scale_gate_scalar=None,
|
||||
output2_scale_scalar=None,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.ep_rank * self.local_num_experts,
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
routing_method_type=self.routing_method_type,
|
||||
do_finalize=True,
|
||||
tune_max_num_tokens=max(self.max_capture_size, 1),
|
||||
output=output,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
@@ -239,6 +254,16 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
# Modular kernel handles only the expert computation;
|
||||
# routing is done externally, so accept any routing method.
|
||||
return True
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
return True
|
||||
|
||||
@@ -282,7 +307,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
):
|
||||
topk = topk_ids.size(-1)
|
||||
local_num_experts = w1.size(0)
|
||||
intermediate_size = w2.size(1)
|
||||
intermediate_size = self.intermediate_size_per_partition
|
||||
local_expert_offset = self.moe_config.ep_rank * local_num_experts
|
||||
|
||||
# Handle input quantization
|
||||
@@ -302,9 +327,8 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
x_quant = hidden_states
|
||||
x_scale = None
|
||||
|
||||
packed_tensor = (topk_ids.to(torch.int32) << 16) | topk_weights.to(
|
||||
torch.bfloat16
|
||||
).view(torch.int16)
|
||||
# Pack topk ids and weights into format expected by the kernel.
|
||||
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
|
||||
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
@@ -333,7 +357,10 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
"local_expert_offset": local_expert_offset,
|
||||
"local_num_experts": local_num_experts,
|
||||
"routed_scaling_factor": None,
|
||||
"routing_method_type": self.routing_method_type,
|
||||
# Modular kernel receives pre-routed tokens, so routing
|
||||
# is already done. Use Renormalize as a safe default that
|
||||
# the TRTLLM C++ kernel supports.
|
||||
"routing_method_type": RoutingMethodType.Renormalize,
|
||||
"do_finalize": True,
|
||||
"output": output,
|
||||
"tune_max_num_tokens": max(self.max_capture_size, 1),
|
||||
@@ -341,12 +368,9 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
|
||||
from flashinfer import trtllm_fp4_block_scale_routed_moe
|
||||
|
||||
from vllm.utils.flashinfer import autotune
|
||||
from vllm.utils.flashinfer import _is_fi_autotuning, autotune
|
||||
|
||||
with autotune(False):
|
||||
# Enable autotune when,
|
||||
# https://github.com/flashinfer-ai/flashinfer/issues/2023 is
|
||||
# resolved.
|
||||
with autotune(_is_fi_autotuning):
|
||||
trtllm_fp4_block_scale_routed_moe(**kwargs)
|
||||
|
||||
return output
|
||||
|
||||
@@ -50,6 +50,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
|
||||
from .utils import swiglu_limit_func
|
||||
|
||||
|
||||
def _fused_marlin_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -88,6 +90,7 @@ def _fused_marlin_moe(
|
||||
output: torch.Tensor | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
is_k_full: bool = True,
|
||||
clamp_limit: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert hidden_states.ndim == 2
|
||||
M, K = hidden_states.size()
|
||||
@@ -155,11 +158,18 @@ def _fused_marlin_moe(
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
)
|
||||
activation_func(
|
||||
activation,
|
||||
intermediate_cache2,
|
||||
intermediate_cache1.view(-1, w13_num_shards * N),
|
||||
)
|
||||
if clamp_limit is not None and activation == MoEActivation.SILU:
|
||||
swiglu_limit_func(
|
||||
intermediate_cache2,
|
||||
intermediate_cache1.view(-1, w13_num_shards * N),
|
||||
clamp_limit,
|
||||
)
|
||||
else:
|
||||
activation_func(
|
||||
activation,
|
||||
intermediate_cache2,
|
||||
intermediate_cache1.view(-1, w13_num_shards * N),
|
||||
)
|
||||
|
||||
if output is None:
|
||||
output = intermediate_cache3
|
||||
@@ -247,6 +257,7 @@ def fused_marlin_moe(
|
||||
output: torch.Tensor | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
inplace: bool = False,
|
||||
clamp_limit: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function computes a Mixture of Experts (MoE) layer using two sets of
|
||||
@@ -363,6 +374,7 @@ def fused_marlin_moe(
|
||||
output=None,
|
||||
input_dtype=input_dtype,
|
||||
is_k_full=is_k_full,
|
||||
clamp_limit=clamp_limit,
|
||||
).view(-1, topk, K)
|
||||
|
||||
if output is None:
|
||||
@@ -557,6 +569,7 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
self.w2_g_idx_sort_indices = w2_g_idx_sort_indices
|
||||
self.is_k_full = is_k_full
|
||||
self.input_dtype = get_marlin_input_dtype()
|
||||
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
|
||||
|
||||
super().__init__(
|
||||
moe_config=moe_config,
|
||||
@@ -850,6 +863,7 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
||||
sort_indices2=self.w2_g_idx_sort_indices,
|
||||
is_k_full=self.is_k_full,
|
||||
input_dtype=self.input_dtype,
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
|
||||
|
||||
@@ -169,5 +169,6 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
layer: "FusedMoE", # type: ignore[name-defined] # noqa: F821
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -268,6 +268,7 @@ class FusedMoE(PluggableLayer):
|
||||
custom_routing_function: Callable | None = None,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
swiglu_limit: float | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
activation: str = "silu",
|
||||
@@ -285,6 +286,7 @@ class FusedMoE(PluggableLayer):
|
||||
routed_output_transform: torch.nn.Module | None = None,
|
||||
apply_routed_scale_to_output: bool = False,
|
||||
zero_expert_type: str | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -294,6 +296,7 @@ class FusedMoE(PluggableLayer):
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.vllm_config = vllm_config
|
||||
self.swiglu_limit = swiglu_limit
|
||||
|
||||
# FIXME (varun): We should have a better way of inferring the activation
|
||||
# datatype. This works for now as the tensor datatype entering the MoE
|
||||
@@ -455,6 +458,7 @@ class FusedMoE(PluggableLayer):
|
||||
self.e_score_correction_bias = e_score_correction_bias
|
||||
# TODO(bnell): end attributes
|
||||
|
||||
self.hash_indices_table = hash_indices_table
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
self.activation = MoEActivation.from_str(activation)
|
||||
|
||||
@@ -479,6 +483,7 @@ class FusedMoE(PluggableLayer):
|
||||
indices_type_getter=lambda: self.quant_method.topk_indices_dtype,
|
||||
zero_expert_type=zero_expert_type,
|
||||
num_logical_experts=self.logical_num_experts,
|
||||
hash_indices_table=self.hash_indices_table,
|
||||
)
|
||||
self.routing_method_type: RoutingMethodType = self.router.routing_method_type
|
||||
|
||||
@@ -1541,10 +1546,12 @@ class FusedMoE(PluggableLayer):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.runner.forward(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
input_ids,
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoEQuantDesc,
|
||||
mxfp4_mxfp8_moe_quant_config,
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
ocp_mx_moe_quant_config,
|
||||
@@ -24,6 +25,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import _swizzle_mxfp4
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
@@ -46,6 +48,8 @@ if has_triton_kernels():
|
||||
|
||||
class Mxfp4MoeBackend(Enum):
|
||||
NONE = "None"
|
||||
# DeepGEMM FP8xFP4 backend (SM100+)
|
||||
DEEPGEMM_MXFP4 = "DEEPGEMM_MXFP4"
|
||||
# FlashInfer TRTLLM backends
|
||||
FLASHINFER_TRTLLM_MXFP4_MXFP8 = "FLASHINFER_TRTLLM_MXFP4_MXFP8"
|
||||
FLASHINFER_TRTLLM_MXFP4_BF16 = "FLASHINFER_TRTLLM_MXFP4_BF16"
|
||||
@@ -81,7 +85,14 @@ TRITON_BACKENDS = (
|
||||
def backend_to_kernel_cls(
|
||||
backend: Mxfp4MoeBackend,
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
if backend in (
|
||||
if backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||
DeepGemmFP4Experts,
|
||||
)
|
||||
|
||||
return [DeepGemmFP4Experts]
|
||||
|
||||
elif backend in (
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
):
|
||||
@@ -159,11 +170,13 @@ def backend_to_kernel_cls(
|
||||
def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
|
||||
"""Map user's moe_backend string to Mxfp4MoeBackend."""
|
||||
mapping: dict[str, Mxfp4MoeBackend] = {
|
||||
"deep_gemm": Mxfp4MoeBackend.DEEPGEMM_MXFP4,
|
||||
"flashinfer_trtllm": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
"flashinfer_trtllm_afp8": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
"flashinfer_cutlass": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
"flashinfer_cutlass_afp8": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
"triton": Mxfp4MoeBackend.TRITON,
|
||||
"triton_unfused": Mxfp4MoeBackend.TRITON_UNFUSED,
|
||||
"marlin": Mxfp4MoeBackend.MARLIN,
|
||||
"aiter": Mxfp4MoeBackend.AITER,
|
||||
"xpu": Mxfp4MoeBackend.XPU,
|
||||
@@ -177,7 +190,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
|
||||
)
|
||||
|
||||
|
||||
def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
Only includes BF16 backends. MXFP8 backends are selected via env vars.
|
||||
@@ -187,7 +200,9 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
Mxfp4MoeBackend.AITER,
|
||||
Mxfp4MoeBackend.TRITON,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.TRITON_UNFUSED,
|
||||
# TRITON_UNFUSED has bug with MTP support
|
||||
# TODO re-enable after kernel is fixed
|
||||
# TRITON_UNFUSED
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN,
|
||||
Mxfp4MoeBackend.XPU,
|
||||
@@ -196,8 +211,28 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
|
||||
def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
"""
|
||||
Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
|
||||
TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the
|
||||
backend-level ``is_supported_config`` check filters by device capability).
|
||||
"""
|
||||
_AVAILABLE_BACKENDS = [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.DEEPGEMM_MXFP4,
|
||||
# TRITON_UNFUSED has bug with MTP support
|
||||
# TODO re-enable after kernel is fixed
|
||||
# TRITON_UNFUSED
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN,
|
||||
]
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
|
||||
def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
|
||||
"""Map backend to its activation key (MXFP8 or None for BF16)."""
|
||||
"""Map backend to its activation key (FP8, MXFP8, or None for BF16)."""
|
||||
if backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
return kFp8Dynamic128Sym
|
||||
if backend in (
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
@@ -290,7 +325,7 @@ def select_gpt_oss_mxfp4_moe_backend(
|
||||
)
|
||||
|
||||
# Select kernels in order of backend.
|
||||
AVAILABLE_BACKENDS = _get_priority_backends()
|
||||
AVAILABLE_BACKENDS = _get_priority_backends_for_gpt_oss()
|
||||
|
||||
# Handle explicit FlashInfer MXFP4 BF16 configuration.
|
||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"):
|
||||
@@ -387,11 +422,95 @@ def select_gpt_oss_mxfp4_moe_backend(
|
||||
return Mxfp4MoeBackend.NONE, None
|
||||
|
||||
|
||||
def select_mxfp4_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts] | None]:
|
||||
"""
|
||||
Select the MXFP4 MoE backend with MXFP8 activation as top priority.
|
||||
Falls back through BF16 and other backends.
|
||||
"""
|
||||
activation_format = (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
if config.moe_parallel_config.use_batched_activation_format
|
||||
else mk.FusedMoEActivationFormat.Standard
|
||||
)
|
||||
|
||||
def _make_log_backend(backend: Mxfp4MoeBackend):
|
||||
return f"Using '{backend.value}' Mxfp4 MoE backend."
|
||||
|
||||
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
if reason:
|
||||
return (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
f"deployment configuration since {reason}."
|
||||
)
|
||||
return (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
"deployment configuration."
|
||||
)
|
||||
|
||||
def _return_or_raise(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
reason: str | None = None
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend), scope="local")
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
# Honor explicit moe_backend (e.g. "marlin", "triton_unfused") before
|
||||
# falling back to the auto priority list.
|
||||
runner_backend = config.moe_backend
|
||||
if runner_backend != "auto":
|
||||
requested_backend = map_mxfp4_backend(runner_backend)
|
||||
if (
|
||||
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
|
||||
and requested_backend == Mxfp4MoeBackend.MARLIN
|
||||
):
|
||||
requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN
|
||||
return _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
_backend_activation_key(requested_backend),
|
||||
activation_format,
|
||||
)
|
||||
|
||||
# Iterate priority backends: TRTLLM MXFP8, then Triton.
|
||||
for backend in _get_priority_backends():
|
||||
activation_key = _backend_activation_key(backend)
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, kMxfp4Static, activation_key, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend), scope="local")
|
||||
return backend, k_cls
|
||||
else:
|
||||
logger.debug_once(_make_log_unsupported(backend, reason), scope="local")
|
||||
|
||||
raise NotImplementedError(
|
||||
"No MXFP4 MoE backend supports the deployment configuration."
|
||||
)
|
||||
|
||||
|
||||
def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
backend: Mxfp4MoeBackend, hidden_size: int, intermediate_size: int
|
||||
) -> tuple[int, int]:
|
||||
"""Round up hidden_size and intermediate_size based on backend requirements."""
|
||||
if backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN):
|
||||
if backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
# DeepGEMM requires M/N/K alignment
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
hidden_size = round_up(hidden_size, 128)
|
||||
elif backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN):
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
if current_platform.is_xpu():
|
||||
hidden_size = round_up(hidden_size, 128)
|
||||
@@ -434,6 +553,20 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
]:
|
||||
"""Convert loaded weights into backend-specific kernel format."""
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
|
||||
return (
|
||||
w13_weight.data,
|
||||
w2_weight.data,
|
||||
_upcast_e8m0_to_fp32(w13_weight_scale.data),
|
||||
_upcast_e8m0_to_fp32(w2_weight_scale.data),
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
num_experts = w13_weight.shape[0]
|
||||
intermediate_size = w13_weight.shape[1] // 2
|
||||
hidden_size = w13_weight.shape[2] * 2
|
||||
@@ -738,9 +871,10 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
elif mxfp4_backend in TRITON_BACKENDS:
|
||||
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
|
||||
|
||||
assert w13_bias is not None and w2_bias is not None
|
||||
w13_bias = w13_bias.to(torch.float32)
|
||||
w2_bias = w2_bias.to(torch.float32)
|
||||
if w13_bias is not None:
|
||||
w13_bias = w13_bias.to(torch.float32)
|
||||
if w2_bias is not None:
|
||||
w2_bias = w2_bias.to(torch.float32)
|
||||
|
||||
w13_weight, w13_flex, w13_scale = _swizzle_mxfp4(
|
||||
w13_weight,
|
||||
@@ -797,15 +931,271 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
)
|
||||
|
||||
|
||||
def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
layer: torch.nn.Module,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
w13_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
_cache_permute_indices: dict[torch.Size, torch.Tensor] | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
Union[torch.Tensor, "PrecisionConfig"],
|
||||
Union[torch.Tensor, "PrecisionConfig"],
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
"""Convert loaded weights into backend-specific kernel format.
|
||||
|
||||
Supports DeepGEMM, TRTLLM MXFP8, Triton and Marlin backends.
|
||||
"""
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
|
||||
# Weights stay as uint8 packed FP4 — no layout change needed.
|
||||
# Convert E8M0 uint8 scales to float32.
|
||||
return (
|
||||
w13_weight.data,
|
||||
w2_weight.data,
|
||||
_upcast_e8m0_to_fp32(w13_weight_scale.data),
|
||||
_upcast_e8m0_to_fp32(w2_weight_scale.data),
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
if mxfp4_backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN):
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_moe_mxfp4_layer_for_marlin,
|
||||
)
|
||||
|
||||
return prepare_moe_mxfp4_layer_for_marlin(
|
||||
layer,
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
num_experts = w13_weight.shape[0]
|
||||
intermediate_size = w13_weight.shape[1] // 2
|
||||
hidden_size = w13_weight.shape[2] * 2
|
||||
|
||||
sf_block_size = 32 # mxfp4 block size
|
||||
|
||||
if mxfp4_backend in TRTLLM_BACKENDS:
|
||||
assert _cache_permute_indices is not None
|
||||
from flashinfer.fp4_quantization import nvfp4_block_scale_interleave
|
||||
from flashinfer.fused_moe.core import get_w2_permute_indices_with_cache
|
||||
|
||||
w13_weight = w13_weight.data
|
||||
w2_weight = w2_weight.data
|
||||
w13_weight_scale = w13_weight_scale.data
|
||||
w2_weight_scale = w2_weight_scale.data
|
||||
if w13_bias is not None:
|
||||
w13_bias = w13_bias.data.to(torch.float32)
|
||||
if w2_bias is not None:
|
||||
w2_bias = w2_bias.data.to(torch.float32)
|
||||
|
||||
# Swap w1/w3 and interleave to match TRTLLM SwiGLU convention.
|
||||
# Standard loading gives contiguous [w1/gate, w3/up].
|
||||
# TRTLLM kernel expects interleaved [w3_0, w1_0, w3_1, w1_1, ...].
|
||||
w1_weight = w13_weight[:, :intermediate_size, :]
|
||||
w3_weight = w13_weight[:, intermediate_size:, :]
|
||||
w13_weight = torch.stack([w3_weight, w1_weight], dim=2).reshape(
|
||||
w13_weight.shape
|
||||
)
|
||||
|
||||
w1_scale = w13_weight_scale[:, :intermediate_size, :]
|
||||
w3_scale = w13_weight_scale[:, intermediate_size:, :]
|
||||
w13_weight_scale = torch.stack([w3_scale, w1_scale], dim=2).reshape(
|
||||
w13_weight_scale.shape
|
||||
)
|
||||
|
||||
if w13_bias is not None:
|
||||
b1 = w13_bias[:, :intermediate_size]
|
||||
b3 = w13_bias[:, intermediate_size:]
|
||||
w13_bias = torch.stack([b3, b1], dim=2).reshape(w13_bias.shape)
|
||||
|
||||
# Shuffle weights and scaling factors for transposed mma output.
|
||||
# Permute indices depend only on shape (cached by torch.Size),
|
||||
# so compute once and apply to all experts via batched indexing.
|
||||
epilogue_tile_m = 128
|
||||
|
||||
# w13 weight permute
|
||||
w13_perm = get_w2_permute_indices_with_cache(
|
||||
_cache_permute_indices,
|
||||
w13_weight[0].view(torch.uint8),
|
||||
epilogue_tile_m,
|
||||
).to(w13_weight.device)
|
||||
w13_weight = w13_weight.view(torch.uint8)[:, w13_perm].contiguous()
|
||||
|
||||
# w13 scale permute + interleave
|
||||
w13_sf_perm = get_w2_permute_indices_with_cache(
|
||||
_cache_permute_indices,
|
||||
w13_weight_scale[0].view(torch.uint8),
|
||||
epilogue_tile_m,
|
||||
num_elts_per_sf=16,
|
||||
).to(w13_weight_scale.device)
|
||||
w13_s = w13_weight_scale.view(torch.uint8)[:, w13_sf_perm].contiguous()
|
||||
E, N_s, K_s = w13_s.shape
|
||||
w13_weight_scale = (
|
||||
nvfp4_block_scale_interleave(w13_s.reshape(E * N_s, K_s))
|
||||
.reshape(num_experts, 2 * intermediate_size, hidden_size // sf_block_size)
|
||||
.view(torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
# w2 weight permute
|
||||
w2_perm = get_w2_permute_indices_with_cache(
|
||||
_cache_permute_indices,
|
||||
w2_weight[0].view(torch.uint8),
|
||||
epilogue_tile_m,
|
||||
).to(w2_weight.device)
|
||||
w2_weight = w2_weight.view(torch.uint8)[:, w2_perm].contiguous()
|
||||
|
||||
# w2 scale permute + interleave
|
||||
w2_sf_perm = get_w2_permute_indices_with_cache(
|
||||
_cache_permute_indices,
|
||||
w2_weight_scale[0].view(torch.uint8),
|
||||
epilogue_tile_m,
|
||||
num_elts_per_sf=16,
|
||||
).to(w2_weight_scale.device)
|
||||
w2_s = w2_weight_scale.view(torch.uint8)[:, w2_sf_perm].contiguous()
|
||||
E2, N2_s, K2_s = w2_s.shape
|
||||
w2_weight_scale = (
|
||||
nvfp4_block_scale_interleave(w2_s.reshape(E2 * N2_s, K2_s))
|
||||
.reshape(num_experts, hidden_size, intermediate_size // sf_block_size)
|
||||
.view(torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
# w13 bias permute
|
||||
if w13_bias is not None:
|
||||
w13_b_perm = get_w2_permute_indices_with_cache(
|
||||
_cache_permute_indices,
|
||||
w13_bias[0].reshape(-1, 1),
|
||||
epilogue_tile_m,
|
||||
).to(w13_bias.device)
|
||||
w13_bias = w13_bias.reshape(num_experts, -1, 1)[:, w13_b_perm].reshape(
|
||||
num_experts, -1
|
||||
)
|
||||
|
||||
# w2 bias permute
|
||||
if w2_bias is not None:
|
||||
w2_b_perm = get_w2_permute_indices_with_cache(
|
||||
_cache_permute_indices,
|
||||
w2_bias[0].reshape(-1, 1),
|
||||
epilogue_tile_m,
|
||||
).to(w2_bias.device)
|
||||
w2_bias = w2_bias.reshape(num_experts, -1, 1)[:, w2_b_perm].reshape(
|
||||
num_experts, -1
|
||||
)
|
||||
|
||||
return (
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
elif mxfp4_backend in TRITON_BACKENDS:
|
||||
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.TRITON:
|
||||
|
||||
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
|
||||
shape = w.shape
|
||||
n = shape[-1]
|
||||
first = w[..., : n // 2]
|
||||
second = w[..., n // 2 :]
|
||||
stacked = torch.stack((first, second), dim=-1)
|
||||
return stacked.reshape(shape)
|
||||
|
||||
w13_weight = shuffle_weight(w13_weight)
|
||||
w13_weight_scale = shuffle_weight(w13_weight_scale)
|
||||
|
||||
if w13_bias is not None:
|
||||
w13_bias = shuffle_weight(w13_bias.to(torch.float32))
|
||||
else:
|
||||
if w13_bias is not None:
|
||||
w13_bias = w13_bias.to(torch.float32)
|
||||
|
||||
if w2_bias is not None:
|
||||
w2_bias = w2_bias.to(torch.float32)
|
||||
|
||||
w13_weight, w13_flex, w13_scale = _swizzle_mxfp4(
|
||||
w13_weight,
|
||||
w13_weight_scale,
|
||||
)
|
||||
w2_weight, w2_flex, w2_scale = _swizzle_mxfp4(
|
||||
w2_weight,
|
||||
w2_weight_scale,
|
||||
)
|
||||
|
||||
w13_precision_config = PrecisionConfig(
|
||||
weight_scale=w13_scale, flex_ctx=FlexCtx(rhs_data=w13_flex)
|
||||
)
|
||||
w2_precision_config = PrecisionConfig(
|
||||
weight_scale=w2_scale, flex_ctx=FlexCtx(rhs_data=w2_flex)
|
||||
)
|
||||
|
||||
del layer.w13_weight
|
||||
del layer.w2_weight
|
||||
|
||||
return (
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_precision_config,
|
||||
w2_precision_config,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported mxfp4_backend for Mxfp4MoEMethod: {mxfp4_backend}. "
|
||||
f"Expected TRTLLM or Triton backend."
|
||||
)
|
||||
|
||||
|
||||
def make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
w1_scale: Union[torch.Tensor, "PrecisionConfig"],
|
||||
w2_scale: Union[torch.Tensor, "PrecisionConfig"],
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
swiglu_limit: float | None = None,
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
"""Create a FusedMoEQuantConfig for the given MXFP4 backend."""
|
||||
if mxfp4_backend in (
|
||||
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
)
|
||||
|
||||
# DeepGEMM FP4 uses FP8 per-token-group activation quantization
|
||||
# with block 128, matching the FP8 DeepGEMM path.
|
||||
_fp8_dtype = current_platform.fp8_dtype()
|
||||
_block_shape = GroupShape(128, 128)
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
|
||||
_a2=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
|
||||
_w1=FusedMoEQuantDesc("mxfp4", None, w1_scale, None, None, w1_bias),
|
||||
_w2=FusedMoEQuantDesc("mxfp4", None, w2_scale, None, None, w2_bias),
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
elif mxfp4_backend in (
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
):
|
||||
@@ -814,6 +1204,9 @@ def make_mxfp4_moe_quant_config(
|
||||
w2_bias=w2_bias,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
elif mxfp4_backend in (
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
@@ -829,6 +1222,9 @@ def make_mxfp4_moe_quant_config(
|
||||
w2_bias=w2_bias,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
else:
|
||||
return ocp_mx_moe_quant_config(
|
||||
@@ -837,6 +1233,9 @@ def make_mxfp4_moe_quant_config(
|
||||
w2_bias=w2_bias,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -228,6 +228,8 @@ class BaseRouter(FusedMoERouter):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
indices_type: torch.dtype | None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Compute the actual routing logic.
|
||||
@@ -249,6 +251,8 @@ class BaseRouter(FusedMoERouter):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Route the input hidden states to the top-k experts based on the
|
||||
@@ -278,7 +282,7 @@ class BaseRouter(FusedMoERouter):
|
||||
|
||||
# Step 3: Compute routing (delegated to subclass)
|
||||
topk_weights, topk_ids = self._compute_routing(
|
||||
hidden_states, router_logits, indices_type
|
||||
hidden_states, router_logits, indices_type, input_ids=input_ids
|
||||
)
|
||||
|
||||
# Capture logical ids before EPLB mapping.
|
||||
|
||||
@@ -46,6 +46,8 @@ class CustomRoutingRouter(BaseRouter):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
indices_type: torch.dtype | None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute routing using the custom routing function."""
|
||||
topk_weights, topk_ids = self.custom_routing_function(
|
||||
|
||||
@@ -31,6 +31,8 @@ class FusedMoERouter(ABC):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Route the input hidden states to the top-k experts based on the
|
||||
|
||||
@@ -4,6 +4,7 @@ import functools
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
import vllm.envs as envs
|
||||
@@ -56,6 +57,32 @@ def vllm_topk_sigmoid(
|
||||
return topk_weights, topk_indices
|
||||
|
||||
|
||||
def vllm_topk_softplus_sqrt(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
token_expert_indices: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
renormalize: bool = False,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
input_tokens: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
ops.topk_hash_softplus_sqrt(
|
||||
topk_weights,
|
||||
topk_indices,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
routed_scaling_factor,
|
||||
e_score_correction_bias,
|
||||
input_tokens,
|
||||
hash_indices_table,
|
||||
)
|
||||
|
||||
return topk_weights, topk_indices
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=8)
|
||||
def _aiter_get_num_expert_group(num_experts: int) -> int:
|
||||
_AITER_MAX_EXPERTS_PER_GROUP = 32
|
||||
@@ -72,11 +99,14 @@ def _aiter_get_num_expert_group(num_experts: int) -> int:
|
||||
def fused_topk_bias(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
scoring_func: str,
|
||||
e_score_correction_bias: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
scoring_func: str = "softmax",
|
||||
indices_type: torch.dtype | None = None,
|
||||
input_tokens: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
):
|
||||
if not rocm_aiter_ops.is_fused_moe_enabled():
|
||||
assert hidden_states.size(0) == gating_output.size(0), (
|
||||
@@ -107,6 +137,8 @@ def fused_topk_bias(
|
||||
renormalize,
|
||||
e_score_correction_bias,
|
||||
)
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights *= routed_scaling_factor
|
||||
return topk_weights, topk_ids
|
||||
elif scoring_func == "sigmoid":
|
||||
topk_weights, topk_ids = vllm_topk_sigmoid(
|
||||
@@ -117,9 +149,24 @@ def fused_topk_bias(
|
||||
renormalize,
|
||||
e_score_correction_bias,
|
||||
)
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights *= routed_scaling_factor
|
||||
return topk_weights, topk_ids
|
||||
elif scoring_func == "sqrtsoftplus":
|
||||
return vllm_topk_softplus_sqrt(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
e_score_correction_bias,
|
||||
input_tokens,
|
||||
hash_indices_table,
|
||||
routed_scaling_factor,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported scoring function: {scoring_func}")
|
||||
|
||||
elif rocm_aiter_ops.is_fused_moe_enabled() and scoring_func == "sigmoid":
|
||||
M = hidden_states.size(0)
|
||||
num_experts = gating_output.shape[-1]
|
||||
@@ -143,6 +190,8 @@ def fused_topk_bias(
|
||||
topk_group=num_expert_group,
|
||||
need_renorm=renormalize,
|
||||
)
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights *= routed_scaling_factor
|
||||
return topk_weights, topk_ids
|
||||
|
||||
n_routed_experts = gating_output.shape[-1]
|
||||
@@ -150,20 +199,31 @@ def fused_topk_bias(
|
||||
scores = gating_output.softmax(dim=-1)
|
||||
elif scoring_func == "sigmoid":
|
||||
scores = gating_output.sigmoid()
|
||||
elif scoring_func == "sqrtsoftplus":
|
||||
scores = F.softplus(gating_output).sqrt()
|
||||
else:
|
||||
raise ValueError(f"Unsupported scoring function: {scoring_func}")
|
||||
|
||||
scores_for_choice = scores.view(
|
||||
-1, n_routed_experts
|
||||
) + e_score_correction_bias.unsqueeze(0)
|
||||
|
||||
if e_score_correction_bias is not None:
|
||||
scores_for_choice = scores.view(
|
||||
-1, n_routed_experts
|
||||
) + e_score_correction_bias.unsqueeze(0)
|
||||
else:
|
||||
scores_for_choice = scores.view(-1, n_routed_experts)
|
||||
# For batch invariance, use sorted=True to ensure deterministic expert selection
|
||||
use_sorted = envs.VLLM_BATCH_INVARIANT
|
||||
topk_indices = torch.topk(scores_for_choice, k=topk, dim=-1, sorted=use_sorted)[1]
|
||||
if hash_indices_table is not None:
|
||||
topk_indices = hash_indices_table[input_tokens]
|
||||
else:
|
||||
use_sorted = envs.VLLM_BATCH_INVARIANT
|
||||
topk_indices = torch.topk(scores_for_choice, k=topk, dim=-1, sorted=use_sorted)[
|
||||
1
|
||||
]
|
||||
topk_weights = scores.gather(1, topk_indices)
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
return topk_weights.to(torch.float32), topk_indices.to(
|
||||
topk_weights = topk_weights.to(torch.float32)
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights *= routed_scaling_factor
|
||||
return topk_weights, topk_indices.to(
|
||||
torch.int32 if indices_type is None else indices_type
|
||||
)
|
||||
|
||||
@@ -176,12 +236,14 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
top_k: int,
|
||||
global_num_experts: int,
|
||||
eplb_state: EplbLayerState,
|
||||
e_score_correction_bias: torch.Tensor,
|
||||
scoring_func: str,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
renormalize: bool = True,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
enable_eplb: bool = False,
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
*,
|
||||
scoring_func: str = "sigmoid",
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
@@ -194,6 +256,8 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
self.renormalize = renormalize
|
||||
self.scoring_func = scoring_func
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.scoring_func = scoring_func
|
||||
self._hash_indices_table = hash_indices_table
|
||||
|
||||
@property
|
||||
def routing_method_type(self) -> RoutingMethodType:
|
||||
@@ -210,19 +274,23 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
indices_type: torch.dtype | None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute routing using fused top-k with bias."""
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
e_score_correction_bias=self.e_score_correction_bias.data,
|
||||
scoring_func=self.scoring_func,
|
||||
e_score_correction_bias=self.e_score_correction_bias.data
|
||||
if self.e_score_correction_bias is not None
|
||||
else None,
|
||||
topk=self.top_k,
|
||||
renormalize=self.renormalize,
|
||||
scoring_func=self.scoring_func,
|
||||
indices_type=indices_type,
|
||||
input_tokens=input_ids,
|
||||
hash_indices_table=self._hash_indices_table,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
if self.routed_scaling_factor != 1.0:
|
||||
topk_weights *= self.routed_scaling_factor
|
||||
|
||||
return topk_weights, topk_ids
|
||||
|
||||
@@ -151,6 +151,8 @@ class FusedTopKRouter(BaseRouter):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
indices_type: torch.dtype | None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute routing using standard fused top-k."""
|
||||
topk_weights, topk_ids, token_expert_indices = fused_topk(
|
||||
|
||||
@@ -292,6 +292,8 @@ class GroupedTopKRouter(BaseRouter):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
indices_type: torch.dtype | None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute routing using grouped top-k."""
|
||||
|
||||
@@ -308,6 +310,7 @@ class GroupedTopKRouter(BaseRouter):
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
scoring_func=self.scoring_func,
|
||||
e_score_correction_bias=self.e_score_correction_bias.data,
|
||||
topk=self.top_k,
|
||||
renormalize=self.renormalize,
|
||||
|
||||
@@ -55,6 +55,7 @@ def create_fused_moe_router(
|
||||
# zero expert parameters
|
||||
zero_expert_type: str | None = None,
|
||||
num_logical_experts: int | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> FusedMoERouter:
|
||||
"""
|
||||
Factory function to create the appropriate FusedMoERouter subclass based on
|
||||
@@ -99,6 +100,9 @@ def create_fused_moe_router(
|
||||
num_logical_experts: Number of real (non-zero) experts. Required when
|
||||
zero_expert_type is not None.
|
||||
|
||||
Hash Indices Table:
|
||||
Used to map input_ids to experts, need for Deepseek V4
|
||||
|
||||
Returns:
|
||||
An instance of the appropriate FusedMoERouter subclass
|
||||
"""
|
||||
@@ -179,17 +183,20 @@ def create_fused_moe_router(
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
|
||||
if e_score_correction_bias is not None:
|
||||
assert scoring_func in ["sigmoid", "softmax", "sqrtsoftplus"]
|
||||
|
||||
if e_score_correction_bias is not None or hash_indices_table is not None:
|
||||
return FusedTopKBiasRouter(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
enable_eplb=enable_eplb,
|
||||
indices_type_getter=indices_type_getter,
|
||||
scoring_func=scoring_func,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
return FusedTopKRouter(
|
||||
|
||||
@@ -334,6 +334,8 @@ class RoutingSimulatorRouter(BaseRouter):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
indices_type: torch.dtype | None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Use routing simulator to compute routing."""
|
||||
routing_strategy = envs.VLLM_MOE_ROUTING_SIMULATION_STRATEGY
|
||||
|
||||
@@ -72,6 +72,8 @@ class ZeroExpertRouter(BaseRouter):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
indices_type: torch.dtype | None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute routing with full bias, compute zero expert output,
|
||||
mask zero expert IDs."""
|
||||
|
||||
@@ -91,6 +91,7 @@ def _moe_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
input_ids: torch.Tensor | None,
|
||||
layer_name: _layer_name_type,
|
||||
) -> torch.Tensor:
|
||||
layer = get_layer_from_name(_resolve_layer_name(layer_name))
|
||||
@@ -99,6 +100,7 @@ def _moe_forward(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
shared_experts_input,
|
||||
input_ids,
|
||||
)
|
||||
|
||||
|
||||
@@ -106,6 +108,7 @@ def _moe_forward_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
input_ids: torch.Tensor | None,
|
||||
layer_name: _layer_name_type,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(hidden_states)
|
||||
@@ -115,6 +118,7 @@ def _moe_forward_shared(
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
input_ids: torch.Tensor | None,
|
||||
layer_name: _layer_name_type,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
layer = get_layer_from_name(_resolve_layer_name(layer_name))
|
||||
@@ -123,6 +127,7 @@ def _moe_forward_shared(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
shared_experts_input,
|
||||
input_ids,
|
||||
)
|
||||
|
||||
|
||||
@@ -130,6 +135,7 @@ def _moe_forward_shared_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
input_ids: torch.Tensor | None,
|
||||
layer_name: _layer_name_type,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Output shapes:
|
||||
@@ -433,6 +439,7 @@ class MoERunner(MoERunnerInterface):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""Run expert routing and the fused MoE kernel via the quant method.
|
||||
|
||||
@@ -449,11 +456,13 @@ class MoERunner(MoERunnerInterface):
|
||||
layer=layer,
|
||||
x=hidden_states,
|
||||
router_logits=router_logits,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
else:
|
||||
topk_weights, topk_ids = self.router.select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
# Passing shared_experts_input in case SharedExpertsOrder is
|
||||
@@ -523,6 +532,7 @@ class MoERunner(MoERunnerInterface):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Invoke the fused moe layer.
|
||||
|
||||
@@ -565,6 +575,7 @@ class MoERunner(MoERunnerInterface):
|
||||
hidden_states,
|
||||
router_logits,
|
||||
shared_experts_input,
|
||||
input_ids,
|
||||
self._encode_layer_name(),
|
||||
)
|
||||
|
||||
@@ -672,6 +683,7 @@ class MoERunner(MoERunnerInterface):
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Entry point called by the custom op to run the MoE computation.
|
||||
|
||||
@@ -712,6 +724,7 @@ class MoERunner(MoERunnerInterface):
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
shared_experts_input=shared_experts_input,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
return self._maybe_combine(
|
||||
|
||||
@@ -26,6 +26,7 @@ class MoERunnerInterface(ABC):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -309,6 +309,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
layer: "FusedMoE", # type: ignore[name-defined] # noqa: F821
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
|
||||
|
||||
@@ -4,6 +4,7 @@ import functools
|
||||
from math import prod
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
@@ -384,3 +385,20 @@ def trtllm_moe_pack_topk_ids_weights(
|
||||
return (topk_ids.to(torch.int32) << 16) | topk_weights.to(torch.bfloat16).view(
|
||||
torch.int16
|
||||
)
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
def swiglu_limit_func(
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor, # first half is gate, second half is up
|
||||
swiglu_limit: float = 0.0,
|
||||
) -> None:
|
||||
d = input.shape[1] // 2
|
||||
gate = input[:, :d]
|
||||
up = input[:, d:]
|
||||
|
||||
if swiglu_limit > 0:
|
||||
gate = torch.clamp(gate, max=swiglu_limit)
|
||||
up = torch.clamp(up, min=-swiglu_limit, max=swiglu_limit)
|
||||
|
||||
output.copy_(F.silu(gate) * up)
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from functools import cache
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_tilelang
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
# tilelang is only available on CUDA platforms
|
||||
if TYPE_CHECKING or current_platform.is_cuda_alike():
|
||||
if not has_tilelang():
|
||||
raise ImportError(
|
||||
"tilelang is required for mhc but is not installed. Install it with "
|
||||
"`pip install tilelang`."
|
||||
)
|
||||
import tilelang
|
||||
import tilelang.language as T
|
||||
else:
|
||||
tilelang = None # type: ignore[assignment]
|
||||
T = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@cache
|
||||
def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int:
|
||||
device_props = torch.cuda.get_device_properties(0)
|
||||
n_sms = device_props.multi_processor_count
|
||||
split_k = n_sms // grid_size
|
||||
if k is not None:
|
||||
# avoid split_k for small k
|
||||
num_block_k = cdiv(k, block_k)
|
||||
split_k = min(split_k, num_block_k // 4)
|
||||
split_k = max(split_k, 1)
|
||||
return split_k
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs={
|
||||
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
|
||||
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
|
||||
tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10,
|
||||
},
|
||||
)
|
||||
def mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 16,
|
||||
hc_mult: int = 4,
|
||||
):
|
||||
"""Deeply fused kernels, everything other than gemm & sqrsum in mHC pre block."""
|
||||
num_tokens = T.dynamic("num_tokens")
|
||||
hc_mult3 = hc_mult * (2 + hc_mult)
|
||||
hidden_block = math.gcd(512, hidden_size)
|
||||
|
||||
gemm_out_mul: T.Tensor[[n_splits, num_tokens, hc_mult3], T.float32] # type: ignore[no-redef, valid-type]
|
||||
gemm_out_sqrsum: T.Tensor[[n_splits, num_tokens], T.float32] # type: ignore[no-redef, valid-type]
|
||||
hc_scale: T.Tensor[[3], T.float32] # type: ignore[no-redef, valid-type]
|
||||
hc_base: T.Tensor[[hc_mult3], T.float32] # type: ignore[no-redef, valid-type]
|
||||
residual: T.Tensor[[num_tokens, hc_mult, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
|
||||
# outputs
|
||||
post_mix: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
|
||||
comb_mix: T.Tensor[[num_tokens, hc_mult * hc_mult], T.float32] # type: ignore[no-redef, valid-type]
|
||||
layer_input: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
|
||||
|
||||
with T.Kernel(num_tokens, threads=96) as i:
|
||||
T.pdl_sync()
|
||||
##################################################################
|
||||
# _pre_norm_fn_fwd_norm
|
||||
rms = T.alloc_fragment(1, T.float32)
|
||||
mixes = T.alloc_fragment(hc_mult3, T.float32)
|
||||
T.clear(mixes)
|
||||
rms[0] = 0
|
||||
for i_split in T.serial(n_splits):
|
||||
rms[0] += gemm_out_sqrsum[i_split, i]
|
||||
rms[0] = T.rsqrt(rms[0] / (hc_mult * hidden_size) + rms_eps)
|
||||
for j in T.Parallel(hc_mult3):
|
||||
mixes[j] = 0
|
||||
for i_split in T.serial(n_splits):
|
||||
mixes[j] += gemm_out_mul[i_split, i, j]
|
||||
mixes[j] *= rms[0]
|
||||
mixes_shared = T.alloc_shared(hc_mult3, T.float32)
|
||||
T.copy(mixes, mixes_shared)
|
||||
|
||||
if T.get_thread_binding() < 32:
|
||||
##################################################################
|
||||
# _pre_split_mixes_fwd (post & comb)
|
||||
cm = T.alloc_fragment((hc_mult, hc_mult), T.float32)
|
||||
for j in T.Parallel(hc_mult):
|
||||
post_mix[i, j] = (
|
||||
T.sigmoid(
|
||||
mixes_shared[j + hc_mult] * hc_scale[1] + hc_base[j + hc_mult]
|
||||
)
|
||||
* hc_post_mult_value
|
||||
)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = (
|
||||
mixes_shared[j * hc_mult + k + hc_mult * 2] * hc_scale[2]
|
||||
+ hc_base[j * hc_mult + k + hc_mult * 2]
|
||||
)
|
||||
|
||||
##################################################################
|
||||
# _sinkhorn_fwd
|
||||
row_sum = T.alloc_fragment(hc_mult, T.float32)
|
||||
col_sum = T.alloc_fragment(hc_mult, T.float32)
|
||||
|
||||
# comb = comb.softmax(-1) + eps
|
||||
row_max = T.alloc_fragment(hc_mult, T.float32)
|
||||
T.reduce_max(cm, row_max, dim=1)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = T.exp(cm[j, k] - row_max[j])
|
||||
T.reduce_sum(cm, row_sum, dim=1)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = cm[j, k] / row_sum[j] + hc_sinkhorn_eps
|
||||
|
||||
# comb = comb / (comb.sum(-2) + eps)
|
||||
T.reduce_sum(cm, col_sum, dim=0)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = cm[j, k] / (col_sum[k] + hc_sinkhorn_eps)
|
||||
|
||||
for _ in T.serial(sinkhorn_repeat - 1):
|
||||
# comb = comb / (comb.sum(-1) + eps)
|
||||
T.reduce_sum(cm, row_sum, dim=1)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = cm[j, k] / (row_sum[j] + hc_sinkhorn_eps)
|
||||
|
||||
# comb = comb / (comb.sum(-2) + eps)
|
||||
T.reduce_sum(cm, col_sum, dim=0)
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
cm[j, k] = cm[j, k] / (col_sum[k] + hc_sinkhorn_eps)
|
||||
|
||||
# save comb_mix to global memory
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
comb_mix[i, j * hc_mult + k] = cm[j, k]
|
||||
else:
|
||||
##################################################################
|
||||
# _pre_split_mixes_fwd (pre)
|
||||
pre_mix_shared = T.alloc_shared(hc_mult, T.float32)
|
||||
for j in T.Parallel(hc_mult):
|
||||
pre_mix_shared[j] = (
|
||||
T.sigmoid(
|
||||
mixes_shared[j] * hc_scale[0] + hc_base[j],
|
||||
)
|
||||
+ hc_pre_eps
|
||||
)
|
||||
###################################################################
|
||||
# _pre_apply_mix_fwd
|
||||
for i0_h in T.Pipelined(hidden_size // hidden_block, num_stages=2):
|
||||
xs = T.alloc_shared((hc_mult, hidden_block), T.float32)
|
||||
xl = T.alloc_fragment((hc_mult, hidden_block), T.float32)
|
||||
T.copy(residual[i, 0, i0_h * hidden_block], xs)
|
||||
T.copy(xs, xl)
|
||||
|
||||
ol = T.alloc_fragment(hidden_block, T.float32)
|
||||
T.clear(ol)
|
||||
|
||||
for i_hc in T.serial(hc_mult):
|
||||
pre = pre_mix_shared[i_hc]
|
||||
for i1_h in T.Parallel(hidden_block):
|
||||
ol[i1_h] += pre * xl[i_hc, i1_h]
|
||||
|
||||
T.copy(ol, layer_input[i, i0_h * hidden_block])
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
def mhc_pre(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
"""
|
||||
|
||||
# Validate shapes
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
assert fn.shape[0] == hc_mult3
|
||||
assert fn.shape[1] == hc_hidden_size
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
fn_flat = fn
|
||||
|
||||
# these number are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m))
|
||||
|
||||
post_mix = torch.empty(
|
||||
num_tokens,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
num_tokens,
|
||||
hc_mult2,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
gemm_out_mul = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
hc_mult3,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
gemm_out_sqrsum = torch.empty(
|
||||
n_splits,
|
||||
num_tokens,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
tf32_hc_prenorm_gemm(
|
||||
residual_flat.view(num_tokens, hc_mult * hidden_size),
|
||||
fn_flat,
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
n_splits,
|
||||
)
|
||||
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
post_mix,
|
||||
comb_mix,
|
||||
layer_input,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
post_mix = post_mix.view(*outer_shape, hc_mult, 1)
|
||||
comb_mix = comb_mix.view(*outer_shape, hc_mult, hc_mult)
|
||||
layer_input = layer_input.view(*outer_shape, hidden_size)
|
||||
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
def _mhc_pre_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
# Create empty tensors with correct shapes for meta device / shape inference
|
||||
post_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs={
|
||||
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
|
||||
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
|
||||
tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10,
|
||||
},
|
||||
)
|
||||
def mhc_post_tilelang(
|
||||
a,
|
||||
b,
|
||||
c,
|
||||
d,
|
||||
x,
|
||||
hc: int,
|
||||
hidden: int,
|
||||
n_thr: int = 128,
|
||||
h_blk: int = 1024,
|
||||
) -> tilelang.JITKernel:
|
||||
# rename for shorter code
|
||||
n = T.dynamic("num_tokens")
|
||||
h = hidden
|
||||
|
||||
h_blk = math.gcd(hidden, h_blk)
|
||||
a: T.Tensor((n, hc, hc), T.float32) # type: ignore[no-redef, valid-type]
|
||||
b: T.Tensor((n, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type]
|
||||
c: T.Tensor((n, hc), T.float32) # type: ignore[no-redef, valid-type]
|
||||
d: T.Tensor((n, h), T.bfloat16) # type: ignore[no-redef, valid-type]
|
||||
x: T.Tensor((n, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type]
|
||||
with T.Kernel(n, threads=n_thr) as i_n:
|
||||
x_shared = T.alloc_shared((hc, h_blk), T.bfloat16)
|
||||
b_shared = T.alloc_shared((hc, h_blk), T.bfloat16)
|
||||
d_shared = T.alloc_shared(h_blk, T.bfloat16)
|
||||
|
||||
x_local = T.alloc_fragment((hc, h_blk), T.float32)
|
||||
b_local = T.alloc_fragment((hc, h_blk), T.float32)
|
||||
d_local = T.alloc_fragment(h_blk, T.float32)
|
||||
|
||||
a_local = T.alloc_fragment((hc, hc), T.float32)
|
||||
c_local = T.alloc_fragment(hc, T.float32)
|
||||
T.pdl_sync()
|
||||
T.copy(a[i_n, 0, 0], a_local)
|
||||
T.copy(c[i_n, 0], c_local)
|
||||
|
||||
for i0_h in T.Pipelined(T.ceildiv(h, h_blk), num_stages=2):
|
||||
T.copy(b[i_n, 0, i0_h * h_blk], b_shared)
|
||||
T.copy(d[i_n, i0_h * h_blk], d_shared)
|
||||
|
||||
T.copy(b_shared, b_local)
|
||||
T.copy(d_shared, d_local)
|
||||
for i_hco, i1_h in T.Parallel(hc, h_blk):
|
||||
x_local[i_hco, i1_h] = c_local[i_hco] * d_local[i1_h]
|
||||
for i_hci in T.serial(hc):
|
||||
x_local[i_hco, i1_h] += a_local[i_hci, i_hco] * b_local[i_hci, i1_h]
|
||||
T.copy(x_local, x_shared)
|
||||
|
||||
T.copy(x_shared, x[i_n, 0, i0_h * h_blk])
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
def mhc_post(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
out = torch.empty_like(residual)
|
||||
mhc_post_tilelang(
|
||||
comb_res_mix,
|
||||
residual,
|
||||
post_layer_mix.squeeze(-1),
|
||||
x,
|
||||
out,
|
||||
residual.shape[-2],
|
||||
residual.shape[-1],
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _mhc_post_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(residual)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre",
|
||||
op_func=mhc_pre,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_post",
|
||||
op_func=mhc_post,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_post_fake,
|
||||
)
|
||||
@@ -32,6 +32,7 @@ QuantizationMethods = Literal[
|
||||
"inc",
|
||||
"mxfp4",
|
||||
"gpt_oss_mxfp4",
|
||||
"deepseek_v4_fp8",
|
||||
"cpu_awq",
|
||||
"online",
|
||||
# Below are values of the OnlineQuantScheme enum, specified as strings to
|
||||
@@ -112,6 +113,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
# lazy import to avoid triggering `torch.compile` too early
|
||||
from vllm.config.quantization import OnlineQuantScheme
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.models.deepseek_v4 import DeepseekV4FP8Config
|
||||
|
||||
from .awq import AWQConfig
|
||||
from .awq_marlin import AWQMarlinConfig
|
||||
@@ -163,6 +165,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"inc": INCConfig,
|
||||
"mxfp4": Mxfp4Config,
|
||||
"gpt_oss_mxfp4": GptOssMxfp4Config,
|
||||
"deepseek_v4_fp8": DeepseekV4FP8Config,
|
||||
"cpu_awq": CPUAWQConfig,
|
||||
"humming": HummingConfig,
|
||||
"online": OnlineQuantizationConfig,
|
||||
|
||||
+1
@@ -265,6 +265,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
|
||||
+1
@@ -305,6 +305,7 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert not layer.enable_eplb, "EPLB not supported for W4A8-int MoE yet."
|
||||
assert layer.activation in (
|
||||
|
||||
+1
@@ -367,6 +367,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
|
||||
+1
@@ -168,6 +168,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
|
||||
+1
@@ -517,6 +517,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.kernel_backend == "Flashinfer"
|
||||
return flashinfer_trtllm_mxint4_moe(
|
||||
|
||||
@@ -269,6 +269,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
|
||||
def __init__(self, quant_config: Fp8Config):
|
||||
self.quant_config = quant_config
|
||||
self.is_scale_e8m0 = getattr(quant_config, "is_scale_e8m0", False)
|
||||
self.cutlass_block_fp8_supported = cutlass_block_fp8_supported()
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
@@ -362,6 +363,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
input_size_per_partition,
|
||||
self.weight_block_size,
|
||||
weight_loader,
|
||||
scale_dtype=(torch.float8_e8m0fnu if self.is_scale_e8m0 else None),
|
||||
)
|
||||
# The weight_scale_inv name is intentional for deepseekv3
|
||||
layer.register_parameter("weight_scale_inv", scale)
|
||||
@@ -866,6 +868,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
|
||||
@@ -950,6 +950,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
@@ -1442,6 +1443,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
@@ -1920,6 +1922,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
from flashinfer.fused_moe.core import (
|
||||
ActivationType,
|
||||
|
||||
@@ -20,10 +20,12 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
mxfp4_round_up_hidden_size_and_intermediate_size,
|
||||
select_gpt_oss_mxfp4_moe_backend,
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
@@ -217,6 +219,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
w13_weight_scale.quant_method = "block"
|
||||
|
||||
# down_proj (row parallel)
|
||||
w2_weight = torch.nn.Parameter(
|
||||
@@ -242,6 +245,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
w2_weight_scale.quant_method = "block"
|
||||
|
||||
if self.moe.has_bias:
|
||||
w13_bias = torch.nn.Parameter(
|
||||
@@ -397,6 +401,9 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w2_scale=w2_scale,
|
||||
w1_bias=w1_bias,
|
||||
w2_bias=w2_bias,
|
||||
gemm1_alpha=1.702,
|
||||
gemm1_beta=1.0,
|
||||
swiglu_limit=7.0,
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
@@ -437,6 +444,332 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
router_logits=router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
|
||||
class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_dtype = "mxfp4"
|
||||
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
|
||||
|
||||
self.max_capture_size = (
|
||||
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
|
||||
)
|
||||
|
||||
self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
||||
self.moe_kernel: mk.FusedMoEKernel | None = None
|
||||
|
||||
# Used for triton kernel precision configs
|
||||
self.w13_precision_config = None
|
||||
self.w2_precision_config = None
|
||||
|
||||
@property
|
||||
def skip_forward_padding(self) -> bool:
|
||||
# SM100_FI_MXFP4_MXFP8_TRTLLM supports padding with mxfp8 quant
|
||||
# so can skip the padding in the forward before applying the moe method
|
||||
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
act_dtype: torch.dtype,
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> tuple[int, int]:
|
||||
hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes(
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size_per_partition,
|
||||
act_dtype=act_dtype,
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
return mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
self.num_experts = num_experts
|
||||
weight_dtype = torch.uint8
|
||||
scale_dtype = torch.uint8
|
||||
mxfp4_block = 32
|
||||
|
||||
layer.params_dtype = params_dtype
|
||||
layer.num_experts = num_experts
|
||||
self.intermediate_size = intermediate_size_per_partition
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
# Fused gate_up_proj (column parallel)
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size // 2,
|
||||
dtype=weight_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size // mxfp4_block,
|
||||
dtype=scale_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
w13_weight_scale.quant_method = "block"
|
||||
|
||||
# down_proj (row parallel)
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // 2,
|
||||
dtype=weight_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // mxfp4_block,
|
||||
dtype=scale_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
w2_weight_scale.quant_method = "block"
|
||||
|
||||
if self.moe.has_bias:
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_bias", w13_bias)
|
||||
set_weight_attrs(w13_bias, extra_weight_attrs)
|
||||
|
||||
w2_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_bias", w2_bias)
|
||||
set_weight_attrs(w2_bias, extra_weight_attrs)
|
||||
|
||||
def _setup_kernel(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
num_experts = self.num_experts
|
||||
intermediate_size = self.intermediate_size
|
||||
hidden_size = self.hidden_size
|
||||
sf_block_size = 32
|
||||
|
||||
# Shape assertions
|
||||
assert (
|
||||
w13.dim() == 3
|
||||
and w13.shape[0] == num_experts
|
||||
and w13.shape[1] == intermediate_size * 2
|
||||
and w13.shape[2] == hidden_size // 2
|
||||
)
|
||||
assert (
|
||||
w13_scale.dim() == 3
|
||||
and w13_scale.shape[0] == num_experts
|
||||
and w13_scale.shape[1] == intermediate_size * 2
|
||||
and w13_scale.shape[2] == hidden_size // sf_block_size
|
||||
)
|
||||
assert (
|
||||
w2.dim() == 3
|
||||
and w2.shape[0] == num_experts
|
||||
and w2.shape[1] == hidden_size
|
||||
and w2.shape[2] == intermediate_size // 2
|
||||
)
|
||||
assert (
|
||||
w2_scale.dim() == 3
|
||||
and w2_scale.shape[1] == hidden_size
|
||||
and w2_scale.shape[2] == intermediate_size // sf_block_size
|
||||
)
|
||||
if w13_bias is not None:
|
||||
assert (
|
||||
w13_bias.dim() == 2
|
||||
and w13_bias.shape[0] == num_experts
|
||||
and w13_bias.shape[1] == intermediate_size * 2
|
||||
)
|
||||
if w2_bias is not None:
|
||||
assert (
|
||||
w2_bias.dim() == 2
|
||||
and w2_bias.shape[0] == num_experts
|
||||
and w2_bias.shape[1] == hidden_size
|
||||
)
|
||||
|
||||
# Convert weights to kernel format
|
||||
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
|
||||
convert_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
layer=layer,
|
||||
w13_weight=w13,
|
||||
w2_weight=w2,
|
||||
w13_weight_scale=w13_scale,
|
||||
w2_weight_scale=w2_scale,
|
||||
w13_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
_cache_permute_indices=self._cache_permute_indices,
|
||||
)
|
||||
)
|
||||
|
||||
# For TRITON backends, weights are wrapped tensors from triton_kernels
|
||||
# that don't support .detach(). Manually assign parameters.
|
||||
if self.mxfp4_backend not in TRITON_BACKENDS:
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
else:
|
||||
layer.w13_weight = w13
|
||||
layer.w2_weight = w2
|
||||
self.w13_precision_config = w13_scale
|
||||
self.w2_precision_config = w2_scale
|
||||
|
||||
if w13_bias is not None and w2_bias is not None:
|
||||
replace_parameter(layer, "w13_bias", w13_bias)
|
||||
replace_parameter(layer, "w2_bias", w2_bias)
|
||||
|
||||
# Build quant config
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
|
||||
# Build kernel (modular or monolithic)
|
||||
if self.moe_quant_config is not None and self.experts_cls is not None:
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._maybe_init_expert_routing_tables(),
|
||||
shared_experts=layer.shared_experts,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
w2_scale = layer.w2_weight_scale
|
||||
w13_bias = getattr(layer, "w13_bias", None)
|
||||
w2_bias = getattr(layer, "w2_bias", None)
|
||||
|
||||
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
|
||||
return
|
||||
|
||||
self._setup_kernel(layer, w13, w2, w13_scale, w2_scale, w13_bias, w2_bias)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
w1_scale = layer.w13_weight_scale
|
||||
w2_scale = layer.w2_weight_scale
|
||||
w1_bias = getattr(layer, "w13_bias", None)
|
||||
w2_bias = getattr(layer, "w2_bias", None)
|
||||
swiglu_limit = getattr(layer, "swiglu_limit", None)
|
||||
|
||||
if self.mxfp4_backend in TRITON_BACKENDS:
|
||||
assert self.w13_precision_config is not None
|
||||
assert self.w2_precision_config is not None
|
||||
w1_scale = self.w13_precision_config
|
||||
w2_scale = self.w2_precision_config
|
||||
|
||||
return make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
w1_bias=w1_bias,
|
||||
w2_bias=w2_bias,
|
||||
swiglu_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalize,
|
||||
layer: torch.nn.Module,
|
||||
) -> mk.FusedMoEExpertsModular:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel "
|
||||
"initialization logic. This function should not be called."
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
expert_map=layer.expert_map,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
|
||||
@@ -130,6 +130,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
layer: "FusedMoE", # type: ignore[name-defined] # noqa: F821
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
|
||||
@@ -1457,6 +1457,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
layer: FusedMoE,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
|
||||
@@ -149,6 +149,148 @@ def _per_token_group_quant_fp8(
|
||||
tl.store(y_s_ptr, y_s)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _silu_mul_quant_fp8_packed_kernel(
|
||||
input_ptr,
|
||||
output_q_ptr,
|
||||
output_scale_ptr,
|
||||
M,
|
||||
input_stride_m,
|
||||
output_q_stride_m,
|
||||
output_scale_stride_k,
|
||||
clamp_limit,
|
||||
N: tl.constexpr,
|
||||
NUM_GROUPS: tl.constexpr,
|
||||
fp8_min: tl.constexpr,
|
||||
fp8_max: tl.constexpr,
|
||||
GROUP_SIZE: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
HAS_CLAMP: tl.constexpr,
|
||||
):
|
||||
N_2: tl.constexpr = N // 2
|
||||
|
||||
pid_pack = tl.program_id(0)
|
||||
pid_m = tl.program_id(1)
|
||||
m_offset = pid_m * BLOCK_M
|
||||
|
||||
if m_offset >= M:
|
||||
return
|
||||
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_n = tl.arange(0, GROUP_SIZE)
|
||||
row_mask = (m_offset + offs_m) < M
|
||||
|
||||
base_row_offset = (m_offset + offs_m[:, None]) * input_stride_m
|
||||
base_out_offset = (m_offset + offs_m[:, None]) * output_q_stride_m
|
||||
|
||||
packed_scale = tl.zeros((BLOCK_M,), dtype=tl.int32)
|
||||
|
||||
for pack_idx in tl.static_range(4):
|
||||
group_id = pid_pack * 4 + pack_idx
|
||||
|
||||
if group_id < NUM_GROUPS:
|
||||
n_offset = group_id * GROUP_SIZE
|
||||
|
||||
act_ptrs = input_ptr + base_row_offset + n_offset + offs_n[None, :]
|
||||
act_in = tl.load(act_ptrs, mask=row_mask[:, None], other=0.0)
|
||||
|
||||
mul_ptrs = act_ptrs + N_2
|
||||
mul_in = tl.load(mul_ptrs, mask=row_mask[:, None], other=0.0)
|
||||
|
||||
act_f32 = act_in.to(tl.float32)
|
||||
mul_f32 = mul_in.to(tl.float32)
|
||||
|
||||
if HAS_CLAMP:
|
||||
act_f32 = tl.minimum(act_f32, clamp_limit)
|
||||
mul_f32 = tl.clamp(mul_f32, -clamp_limit, clamp_limit)
|
||||
|
||||
y = (act_f32 / (1.0 + tl.exp(-act_f32))) * mul_f32
|
||||
# Round through bf16 to match unfused precision path
|
||||
y = y.to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
absmax = tl.max(tl.abs(y), axis=1)
|
||||
|
||||
scale_raw = tl.maximum(absmax / fp8_max, 1e-10)
|
||||
exponent = tl.ceil(tl.log2(scale_raw))
|
||||
scale = tl.math.exp2(exponent)
|
||||
|
||||
y_q = tl.clamp(y / scale[:, None], fp8_min, fp8_max)
|
||||
|
||||
out_q_ptrs = output_q_ptr + base_out_offset + n_offset + offs_n[None, :]
|
||||
tl.store(
|
||||
out_q_ptrs,
|
||||
y_q.to(output_q_ptr.dtype.element_ty),
|
||||
mask=row_mask[:, None],
|
||||
)
|
||||
|
||||
exponent_biased = tl.clamp(exponent + 127.0, 0.0, 255.0).to(tl.int32)
|
||||
packed_scale = packed_scale | (exponent_biased << (pack_idx * 8))
|
||||
|
||||
scale_ptrs = output_scale_ptr + pid_pack * output_scale_stride_k + m_offset + offs_m
|
||||
tl.store(scale_ptrs, packed_scale, mask=row_mask)
|
||||
|
||||
|
||||
def silu_mul_quant_fp8_packed_triton(
|
||||
input: torch.Tensor,
|
||||
group_size: int = 128,
|
||||
output_q: torch.Tensor | None = None,
|
||||
clamp_limit: float | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert input.dim() == 2
|
||||
assert input.is_contiguous()
|
||||
|
||||
M, N = input.shape
|
||||
N_2 = N // 2
|
||||
|
||||
assert N_2 % group_size == 0
|
||||
|
||||
fp8_dtype = torch.float8_e4m3fn
|
||||
finfo = torch.finfo(fp8_dtype)
|
||||
fp8_min, fp8_max = finfo.min, finfo.max
|
||||
|
||||
num_groups_per_row = N_2 // group_size
|
||||
num_packed_groups = (num_groups_per_row + 3) // 4
|
||||
tma_aligned_M = ((M + 3) // 4) * 4
|
||||
|
||||
if output_q is None:
|
||||
output_q = torch.empty((M, N_2), dtype=fp8_dtype, device=input.device)
|
||||
|
||||
output_scale_packed = torch.zeros(
|
||||
(num_packed_groups, tma_aligned_M),
|
||||
dtype=torch.int32,
|
||||
device=input.device,
|
||||
).T[:M, :]
|
||||
|
||||
BLOCK_M = 8
|
||||
grid = (num_packed_groups, (M + BLOCK_M - 1) // BLOCK_M)
|
||||
|
||||
num_warps = max(4, group_size // 32)
|
||||
num_stages = 2
|
||||
|
||||
has_clamp = clamp_limit is not None
|
||||
_silu_mul_quant_fp8_packed_kernel[grid](
|
||||
input,
|
||||
output_q,
|
||||
output_scale_packed,
|
||||
M,
|
||||
input.stride(0),
|
||||
output_q.stride(0),
|
||||
output_scale_packed.stride(1),
|
||||
clamp_limit if has_clamp else 0.0,
|
||||
N=N,
|
||||
NUM_GROUPS=num_groups_per_row,
|
||||
fp8_min=fp8_min,
|
||||
fp8_max=fp8_max,
|
||||
GROUP_SIZE=group_size,
|
||||
BLOCK_M=BLOCK_M,
|
||||
HAS_CLAMP=has_clamp,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
|
||||
return output_q, output_scale_packed
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
y_ptr, # [M, N]
|
||||
@@ -823,19 +965,65 @@ def requant_weight_ue8m0_inplace(
|
||||
s_old.copy_(s_requant)
|
||||
|
||||
|
||||
def _upcast_e8m0_to_fp32(scale: torch.Tensor) -> torch.Tensor:
|
||||
"""Upcast E8M0 (exponent-only) scale to float32.
|
||||
|
||||
E8M0 stores only the 8-bit biased exponent (bias=127). To convert
|
||||
to float32 we place those 8 bits into the exponent field of an
|
||||
IEEE-754 float32 (bits 23-30) with sign=0 and mantissa=0.
|
||||
"""
|
||||
exp_bits = scale.view(torch.uint8).to(torch.int32)
|
||||
fp32_bits = exp_bits << 23
|
||||
return fp32_bits.view(torch.float32)
|
||||
|
||||
|
||||
def deepgemm_post_process_fp8_weight_block(
|
||||
wq: torch.Tensor, ws: torch.Tensor, quant_block_shape: tuple[int], use_e8m0: bool
|
||||
wq: torch.Tensor,
|
||||
ws: torch.Tensor,
|
||||
quant_block_shape: tuple[int, ...],
|
||||
use_e8m0: bool,
|
||||
is_bmm: bool = False,
|
||||
bmm_batch_size: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert wq.dtype == torch.float8_e4m3fn, (
|
||||
"Expected quantized tensor dtype "
|
||||
f"to be torch.float8_e4m3fn, got {wq.dtype} instead."
|
||||
)
|
||||
assert ws.dtype == torch.float32, (
|
||||
f"Expected tensor scales dtype to be torch.float32, got {ws.dtype} instead"
|
||||
)
|
||||
|
||||
if use_e8m0:
|
||||
requant_weight_ue8m0_inplace(wq, ws, block_size=quant_block_shape)
|
||||
if ws.dtype == torch.float8_e8m0fnu:
|
||||
# Scales already in E8M0 from checkpoint — upcast to fp32
|
||||
# and skip requantization (weights already have power-of-two scales).
|
||||
ws = _upcast_e8m0_to_fp32(ws)
|
||||
else:
|
||||
assert ws.dtype == torch.float32, (
|
||||
f"Expected tensor scales dtype to be torch.float32 or "
|
||||
f"torch.float8_e8m0fnu, got {ws.dtype} instead"
|
||||
)
|
||||
if use_e8m0:
|
||||
requant_weight_ue8m0_inplace(wq, ws, block_size=quant_block_shape)
|
||||
|
||||
if is_bmm:
|
||||
# Reshape 2D weight/scale to 3D for grouped BMM (einsum):
|
||||
# wq: (g*r, d) -> (g, r, d)
|
||||
# ws: (g*r/128, d/128) -> (g, r/128, d/128)
|
||||
g = bmm_batch_size
|
||||
assert wq.ndim == 2 and ws.ndim == 2
|
||||
d = wq.size(1)
|
||||
r = wq.size(0) // g
|
||||
wq = wq.view(g, r, d)
|
||||
ws = ws.view(g, r // quant_block_shape[0], d // quant_block_shape[1])
|
||||
# Pre-transform scale with recipe=(1, 128, 128) to broadcast + pack
|
||||
# into TMA-aligned UE8M0 (INT32) layout. At runtime fp8_einsum uses
|
||||
# recipe=(1, 1, 128) which sees INT dtype and skips re-transform.
|
||||
dg_ws = transform_sf_into_required_layout(
|
||||
sf=ws,
|
||||
mn=r,
|
||||
k=d,
|
||||
recipe=(1, quant_block_shape[0], quant_block_shape[1]),
|
||||
num_groups=g,
|
||||
is_sfa=False,
|
||||
)
|
||||
return wq, dg_ws
|
||||
|
||||
original_ndim = wq.ndim
|
||||
if wq.ndim == 2:
|
||||
@@ -984,11 +1172,13 @@ def create_fp8_scale_parameter(
|
||||
input_size_per_partition: int,
|
||||
block_size: list[int] | None,
|
||||
weight_loader: Callable | None,
|
||||
scale_dtype: torch.dtype | None = None,
|
||||
) -> torch.nn.Parameter:
|
||||
"""Create scale parameter based on quantization strategy."""
|
||||
dtype = scale_dtype if scale_dtype is not None else torch.float32
|
||||
if parameter_type == ChannelQuantScaleParameter:
|
||||
scale = parameter_type(
|
||||
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32),
|
||||
data=torch.empty((sum(output_partition_sizes), 1), dtype=dtype),
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
@@ -1000,7 +1190,7 @@ def create_fp8_scale_parameter(
|
||||
data=torch.empty(
|
||||
(output_size_per_partition + block_n - 1) // block_n,
|
||||
(input_size_per_partition + block_k - 1) // block_k,
|
||||
dtype=torch.float32,
|
||||
dtype=dtype,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
@@ -1008,13 +1198,14 @@ def create_fp8_scale_parameter(
|
||||
)
|
||||
elif parameter_type == PerTensorScaleParameter:
|
||||
scale = parameter_type(
|
||||
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
|
||||
data=torch.empty(len(output_partition_sizes), dtype=dtype),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown parameter type: {parameter_type}")
|
||||
|
||||
scale[:] = torch.finfo(torch.float32).min
|
||||
if dtype == torch.float32:
|
||||
scale[:] = torch.finfo(torch.float32).min
|
||||
set_weight_attrs(scale, {"scale_type": "weight_scale"})
|
||||
return scale
|
||||
|
||||
|
||||
@@ -7,7 +7,10 @@ from typing import Any
|
||||
import torch
|
||||
|
||||
from .base import RotaryEmbedding
|
||||
from .deepseek_scaling_rope import DeepseekScalingRotaryEmbedding
|
||||
from .deepseek_scaling_rope import (
|
||||
DeepseekScalingRotaryEmbedding,
|
||||
DeepseekV4ScalingRotaryEmbedding,
|
||||
)
|
||||
from .dual_chunk_rope import DualChunkRotaryEmbedding
|
||||
from .dynamic_ntk_alpha_rope import DynamicNTKAlphaRotaryEmbedding
|
||||
from .dynamic_ntk_scaling_rope import DynamicNTKScalingRotaryEmbedding
|
||||
@@ -60,11 +63,13 @@ def get_rope(
|
||||
rope_parameters = rope_parameters or {}
|
||||
base = rope_parameters.get("rope_theta", 10000)
|
||||
scaling_type = rope_parameters.get("rope_type", "default")
|
||||
partial_rotary_factor = rope_parameters.get("partial_rotary_factor", 1.0)
|
||||
|
||||
if partial_rotary_factor <= 0.0 or partial_rotary_factor > 1.0:
|
||||
raise ValueError(f"{partial_rotary_factor=} must be between 0.0 and 1.0")
|
||||
rotary_dim = int(head_size * partial_rotary_factor)
|
||||
if rotary_dim := rope_parameters.get("rope_dim", None):
|
||||
pass
|
||||
else:
|
||||
partial_rotary_factor = rope_parameters.get("partial_rotary_factor", 1.0)
|
||||
if partial_rotary_factor <= 0.0 or partial_rotary_factor > 1.0:
|
||||
raise ValueError(f"{partial_rotary_factor=} must be between 0.0 and 1.0")
|
||||
rotary_dim = int(head_size * partial_rotary_factor)
|
||||
|
||||
key = (
|
||||
head_size,
|
||||
@@ -289,7 +294,11 @@ def get_rope(
|
||||
"mscale_all_dim",
|
||||
)
|
||||
}
|
||||
rotary_emb = DeepseekScalingRotaryEmbedding(
|
||||
if rope_parameters.get("is_deepseek_v4", False):
|
||||
cls = DeepseekV4ScalingRotaryEmbedding
|
||||
else:
|
||||
cls = DeepseekScalingRotaryEmbedding
|
||||
rotary_emb = cls(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
original_max_position,
|
||||
|
||||
@@ -132,10 +132,8 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
|
||||
]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
if self.is_neox_style:
|
||||
# NOTE(woosuk): Here we assume that the positions tensor has the
|
||||
# shape [batch_size, seq_len].
|
||||
cos = cos.repeat(1, 1, 2).unsqueeze(-2)
|
||||
sin = sin.repeat(1, 1, 2).unsqueeze(-2)
|
||||
cos = torch.cat((cos, cos), dim=-1).unsqueeze(-2)
|
||||
sin = torch.cat((sin, sin), dim=-1).unsqueeze(-2)
|
||||
else:
|
||||
cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
|
||||
sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
|
||||
@@ -197,3 +195,118 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbeddingBase):
|
||||
return query, key
|
||||
else:
|
||||
return self.forward_native(positions, query, key, offsets)
|
||||
|
||||
|
||||
class DeepseekV4ScalingRotaryEmbedding(DeepseekScalingRotaryEmbedding):
|
||||
"""RotaryEmbedding extended with YaRN method.
|
||||
|
||||
Credits to Peng et al. github.com/jquesnelle/yarn
|
||||
|
||||
Compared to DeepseekScalingRotaryEmbedding:
|
||||
- Applies RoPE to the last rotary_dim
|
||||
- The forward method requires an inverse parameter to indicate
|
||||
whether to negate the sin
|
||||
- Supports applying RoPE to query only (without key)
|
||||
- cos_sin_cache stored as fp32 for higher precision RoPE
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
cache_fp32 = self._compute_cos_sin_cache()
|
||||
self.register_buffer("cos_sin_cache", cache_fp32, persistent=False)
|
||||
|
||||
def _compute_cos_sin_cache(self) -> torch.Tensor:
|
||||
inv_freq = self._compute_inv_freq(self.scaling_factor)
|
||||
t = torch.arange(
|
||||
self.max_position_embeddings * self.scaling_factor,
|
||||
device=current_platform.device_type,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
freqs = torch.einsum("i,j -> ij", t, inv_freq)
|
||||
cos = freqs.cos() * self.mscale
|
||||
sin = freqs.sin() * self.mscale
|
||||
cache = torch.cat((cos, sin), dim=-1)
|
||||
return cache
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
inverse: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""PyTorch-native implementation equivalent to forward()."""
|
||||
|
||||
head_size = query.size(-1)
|
||||
query_rot = query[..., -self.rotary_dim :]
|
||||
key_rot = key[..., -self.rotary_dim :] if key is not None else None
|
||||
|
||||
if self.rotary_dim < head_size:
|
||||
query_pass = query[..., : -self.rotary_dim]
|
||||
key_pass = key[..., : -self.rotary_dim] if key is not None else None
|
||||
|
||||
cos_sin = self.cos_sin_cache[
|
||||
torch.add(positions, offsets) if offsets is not None else positions
|
||||
]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
if self.is_neox_style:
|
||||
cos = torch.cat((cos, cos), dim=-1).unsqueeze(-2)
|
||||
sin = torch.cat((sin, sin), dim=-1).unsqueeze(-2)
|
||||
else:
|
||||
cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
|
||||
sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
|
||||
if inverse:
|
||||
sin = -sin
|
||||
rotate_fn = rotate_neox if self.is_neox_style else rotate_gptj
|
||||
orig_dtype = query.dtype
|
||||
query_rot = (query_rot * cos + rotate_fn(query_rot) * sin).to(orig_dtype)
|
||||
if key_rot is not None:
|
||||
key_rot = (key_rot * cos + rotate_fn(key_rot) * sin).to(orig_dtype)
|
||||
|
||||
if self.rotary_dim < head_size:
|
||||
query = torch.cat((query_pass, query_rot), dim=-1)
|
||||
key = torch.cat((key_pass, key_rot), dim=-1) if key is not None else None
|
||||
else:
|
||||
query = query_rot
|
||||
key = key_rot
|
||||
|
||||
return query, key
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
inverse: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
return self.forward_native(positions, query, key, offsets)
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
inverse: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
# The indexer and attention have different head_dim,
|
||||
# we obtain the corresponding head_dim via the query.
|
||||
head_size = query.size(-1)
|
||||
rope_dim_offset = head_size - self.rotary_dim
|
||||
# ops.rotary_embedding() is an in-place operation
|
||||
# that updates the query and key tensors.
|
||||
ops.rotary_embedding(
|
||||
torch.add(positions, offsets) if offsets is not None else positions,
|
||||
query,
|
||||
key,
|
||||
head_size,
|
||||
self.cos_sin_cache,
|
||||
self.is_neox_style,
|
||||
rope_dim_offset=rope_dim_offset,
|
||||
inverse=inverse,
|
||||
)
|
||||
return query, key
|
||||
|
||||
@@ -10,7 +10,11 @@ from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import fp8_mqa_logits, fp8_paged_mqa_logits, has_deep_gemm
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_mqa_logits,
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
has_deep_gemm,
|
||||
)
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
@@ -32,12 +36,57 @@ logger = init_logger(__name__)
|
||||
|
||||
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
|
||||
# MXFP4 layout: 2 values packed per byte, ue8m0 (1-byte) scale per block of 32.
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
def _gather_workspace_shapes(
|
||||
total_seq_lens: int,
|
||||
head_dim: int,
|
||||
fp8_dtype: torch.dtype,
|
||||
use_fp4_cache: bool,
|
||||
) -> tuple[tuple[tuple[int, int], torch.dtype], tuple[tuple[int, int], torch.dtype]]:
|
||||
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
|
||||
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
|
||||
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
|
||||
if use_fp4_cache:
|
||||
return (
|
||||
((total_seq_lens, head_dim // 2), torch.uint8),
|
||||
((total_seq_lens, head_dim // MXFP4_BLOCK_SIZE), torch.uint8),
|
||||
)
|
||||
return (
|
||||
((total_seq_lens, head_dim), fp8_dtype),
|
||||
((total_seq_lens, 4), torch.uint8),
|
||||
)
|
||||
|
||||
|
||||
def kv_cache_as_quant_view(
|
||||
kv_cache: torch.Tensor,
|
||||
head_dim: int,
|
||||
use_fp4_cache: bool,
|
||||
) -> torch.Tensor:
|
||||
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation."""
|
||||
if use_fp4_cache:
|
||||
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
|
||||
num_blocks, block_size, _ = kv_cache.shape
|
||||
page_bytes = int(kv_cache.stride(0))
|
||||
fp4_bytes = head_dim // 2 + head_dim // MXFP4_BLOCK_SIZE
|
||||
return torch.as_strided(
|
||||
kv_cache,
|
||||
size=(num_blocks, block_size, 1, fp4_bytes),
|
||||
stride=(page_bytes, fp4_bytes, fp4_bytes, 1),
|
||||
)
|
||||
return kv_cache.unsqueeze(-2)
|
||||
|
||||
|
||||
def sparse_attn_indexer(
|
||||
hidden_states: torch.Tensor,
|
||||
k_cache_prefix: LayerNameType,
|
||||
kv_cache: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
q_quant: torch.Tensor,
|
||||
q_scale: torch.Tensor | None,
|
||||
k: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
quant_block_size: int,
|
||||
@@ -47,6 +96,8 @@ def sparse_attn_indexer(
|
||||
max_model_len: int,
|
||||
total_seq_lens: int,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
@@ -56,9 +107,12 @@ def sparse_attn_indexer(
|
||||
# assert isinstance(attn_metadata, dict)
|
||||
if not isinstance(attn_metadata, dict):
|
||||
# Reserve workspace for indexer during profiling run
|
||||
values_spec, scales_spec = _gather_workspace_shapes(
|
||||
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
|
||||
)
|
||||
current_workspace_manager().get_simultaneous(
|
||||
((total_seq_lens, head_dim), torch.float8_e4m3fn),
|
||||
((total_seq_lens, 4), torch.uint8),
|
||||
values_spec,
|
||||
scales_spec,
|
||||
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
|
||||
)
|
||||
|
||||
@@ -73,7 +127,8 @@ def sparse_attn_indexer(
|
||||
hidden_states,
|
||||
k_cache_prefix,
|
||||
kv_cache,
|
||||
q_fp8,
|
||||
q_quant,
|
||||
q_scale,
|
||||
k,
|
||||
weights,
|
||||
quant_block_size,
|
||||
@@ -83,6 +138,8 @@ def sparse_attn_indexer(
|
||||
max_model_len,
|
||||
total_seq_lens,
|
||||
topk_indices_buffer,
|
||||
skip_k_cache_insert,
|
||||
use_fp4_cache,
|
||||
)
|
||||
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
|
||||
assert isinstance(attn_metadata_narrowed, DeepseekV32IndexerMetadata)
|
||||
@@ -91,49 +148,81 @@ def sparse_attn_indexer(
|
||||
has_prefill = attn_metadata_narrowed.num_prefills > 0
|
||||
num_decode_tokens = attn_metadata_narrowed.num_decode_tokens
|
||||
|
||||
# q_scale is required iff the FP4 cache path is enabled; the FP8 path
|
||||
# folds the Q scale into `weights` inside fused_indexer_q_rope_quant.
|
||||
if use_fp4_cache:
|
||||
assert q_scale is not None, "use_fp4_cache=True requires q_scale"
|
||||
else:
|
||||
assert q_scale is None, "q_scale must be None when use_fp4_cache=False"
|
||||
|
||||
# During speculative decoding, k may be padded to the CUDA graph batch
|
||||
# size while slot_mapping only covers actual tokens. Truncate k to avoid
|
||||
# out-of-bounds reads in the kernel.
|
||||
num_tokens = slot_mapping.shape[0]
|
||||
k = k[:num_tokens]
|
||||
if k is not None:
|
||||
k = k[:num_tokens]
|
||||
|
||||
# scale_fmt can be None, but the function expects str
|
||||
assert scale_fmt is not None
|
||||
ops.indexer_k_quant_and_cache(
|
||||
k,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
)
|
||||
if not skip_k_cache_insert:
|
||||
# scale_fmt can be None, but the function expects str
|
||||
assert scale_fmt is not None
|
||||
assert not use_fp4_cache, "Unfused FP4 Insert is not supported yet"
|
||||
ops.indexer_k_quant_and_cache(
|
||||
k,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
)
|
||||
|
||||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||||
if has_prefill:
|
||||
prefill_metadata = attn_metadata_narrowed.prefill
|
||||
assert prefill_metadata is not None
|
||||
|
||||
# Get the full shared workspace buffers once (will allocate on first use)
|
||||
# Get the full shared workspace buffers once (will allocate on first use).
|
||||
# Layout switches between FP8 (head_dim bytes + 4-byte fp32 scale) and
|
||||
# MXFP4 (head_dim/2 bytes packed + head_dim/MXFP4_BLOCK_SIZE ue8m0
|
||||
# scales) based on use_fp4_cache.
|
||||
workspace_manager = current_workspace_manager()
|
||||
k_fp8_full, k_scale_full = workspace_manager.get_simultaneous(
|
||||
((total_seq_lens, head_dim), fp8_dtype),
|
||||
((total_seq_lens, 4), torch.uint8),
|
||||
values_spec, scales_spec = _gather_workspace_shapes(
|
||||
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
|
||||
)
|
||||
k_quant_full, k_scale_full = workspace_manager.get_simultaneous(
|
||||
values_spec,
|
||||
scales_spec,
|
||||
)
|
||||
for chunk in prefill_metadata.chunks:
|
||||
k_fp8 = k_fp8_full[: chunk.total_seq_lens]
|
||||
k_quant = k_quant_full[: chunk.total_seq_lens]
|
||||
k_scale = k_scale_full[: chunk.total_seq_lens]
|
||||
|
||||
if not chunk.skip_kv_gather:
|
||||
ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache,
|
||||
k_fp8,
|
||||
k_quant,
|
||||
k_scale,
|
||||
chunk.block_table,
|
||||
chunk.cu_seq_lens,
|
||||
)
|
||||
|
||||
logits = fp8_mqa_logits(
|
||||
q_fp8[chunk.token_start : chunk.token_end],
|
||||
(k_fp8, k_scale.view(torch.float32).flatten()),
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
q_scale_slice = (
|
||||
q_scale[chunk.token_start : chunk.token_end]
|
||||
if q_scale is not None
|
||||
else None
|
||||
)
|
||||
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
|
||||
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
|
||||
if use_fp4_cache:
|
||||
q_slice_cast = q_slice.view(torch.int8)
|
||||
k_quant_cast = k_quant.view(torch.int8)
|
||||
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
|
||||
else:
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
@@ -171,32 +260,55 @@ def sparse_attn_indexer(
|
||||
if has_decode:
|
||||
decode_metadata = attn_metadata_narrowed.decode
|
||||
assert decode_metadata is not None
|
||||
# kv_cache shape [
|
||||
# kv_cache size requirement [num_block, block_size, n_head, head_dim],
|
||||
# we only have [num_block, block_size, head_dim],
|
||||
kv_cache = kv_cache.unsqueeze(-2)
|
||||
kv_cache = kv_cache_as_quant_view(kv_cache, head_dim, use_fp4_cache)
|
||||
decode_lens = decode_metadata.decode_lens
|
||||
if decode_metadata.requires_padding:
|
||||
# pad in edge case where we have short chunked prefill length <
|
||||
# decode_threshold since we unstrictly split
|
||||
# prefill and decode by decode_threshold
|
||||
# (currently set to 1 + speculative tokens)
|
||||
padded_q_fp8_decode_tokens = pack_seq_triton(
|
||||
q_fp8[:num_decode_tokens], decode_lens
|
||||
)
|
||||
# (currently set to 1 + speculative tokens).
|
||||
# FP8 Q is float8_e4m3fn (pack_seq_triton's fp32 pad path is OK —
|
||||
# downstream context_lens masks stale slots). MXFP4 Q is two
|
||||
# uint8 tensors (values + ue8m0 scales) — use the dedicated uint8
|
||||
# packer with pad_byte=0 so padded slots dequantize to 0 and
|
||||
# can't produce NaN/Inf in the logits kernel.
|
||||
if q_scale is not None:
|
||||
padded_q_quant_decode_tokens = pack_seq_triton(
|
||||
q_quant[:num_decode_tokens], decode_lens, pad_value=0
|
||||
)
|
||||
padded_q_scale = pack_seq_triton(
|
||||
q_scale[:num_decode_tokens], decode_lens, pad_value=0
|
||||
)
|
||||
else:
|
||||
padded_q_quant_decode_tokens = pack_seq_triton(
|
||||
q_quant[:num_decode_tokens], decode_lens
|
||||
)
|
||||
padded_q_scale = None
|
||||
else:
|
||||
padded_q_fp8_decode_tokens = q_fp8[:num_decode_tokens].reshape(
|
||||
decode_lens.shape[0], -1, *q_fp8.shape[1:]
|
||||
padded_q_quant_decode_tokens = q_quant[:num_decode_tokens].reshape(
|
||||
decode_lens.shape[0], -1, *q_quant.shape[1:]
|
||||
)
|
||||
if q_scale is not None:
|
||||
padded_q_scale = q_scale[:num_decode_tokens].reshape(
|
||||
decode_lens.shape[0], -1, *q_scale.shape[1:]
|
||||
)
|
||||
else:
|
||||
padded_q_scale = None
|
||||
# TODO: move and optimize below logic with triton kernels
|
||||
batch_size = padded_q_fp8_decode_tokens.shape[0]
|
||||
next_n = padded_q_fp8_decode_tokens.shape[1]
|
||||
batch_size = padded_q_quant_decode_tokens.shape[0]
|
||||
next_n = padded_q_quant_decode_tokens.shape[1]
|
||||
num_padded_tokens = batch_size * next_n
|
||||
seq_lens = decode_metadata.seq_lens[:batch_size]
|
||||
# seq_lens is (B, next_n) for native spec decode, (B,) otherwise.
|
||||
# fp8_paged_mqa_logits and all topk kernels accept both shapes.
|
||||
logits = fp8_paged_mqa_logits(
|
||||
padded_q_fp8_decode_tokens,
|
||||
# seq_lens is always 2D: (B, next_n) for native spec decode, (B, 1)
|
||||
# otherwise. deep_gemm fp8_fp4_paged_mqa_logits requires 2D context_lens;
|
||||
# the downstream topk kernels accept both 1D and 2D.
|
||||
padded_q_quant_cast = (
|
||||
padded_q_quant_decode_tokens.view(torch.int8)
|
||||
if use_fp4_cache
|
||||
else padded_q_quant_decode_tokens
|
||||
)
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
weights[:num_padded_tokens],
|
||||
seq_lens,
|
||||
@@ -208,7 +320,7 @@ def sparse_attn_indexer(
|
||||
num_rows = logits.shape[0]
|
||||
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
|
||||
|
||||
if current_platform.is_cuda():
|
||||
if current_platform.is_cuda() and topk_tokens in (512, 1024, 2048):
|
||||
workspace_manager = current_workspace_manager()
|
||||
(topk_workspace,) = workspace_manager.get_simultaneous(
|
||||
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
|
||||
@@ -263,7 +375,8 @@ def sparse_attn_indexer_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
k_cache_prefix: LayerNameType,
|
||||
kv_cache: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
q_quant: torch.Tensor,
|
||||
q_scale: torch.Tensor | None,
|
||||
k: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
quant_block_size: int,
|
||||
@@ -273,6 +386,8 @@ def sparse_attn_indexer_fake(
|
||||
max_model_len: int,
|
||||
total_seq_lens: int,
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return topk_indices_buffer
|
||||
|
||||
@@ -309,6 +424,8 @@ class SparseAttnIndexer(CustomOp):
|
||||
max_model_len: int,
|
||||
max_total_seq_len: int,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool = False,
|
||||
use_fp4_cache: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.k_cache = k_cache
|
||||
@@ -319,6 +436,8 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.max_model_len = max_model_len
|
||||
self.max_total_seq_len = max_total_seq_len
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self.skip_k_cache_insert = skip_k_cache_insert
|
||||
self.use_fp4_cache = use_fp4_cache
|
||||
if current_platform.is_cuda() and not has_deep_gemm():
|
||||
raise RuntimeError(
|
||||
"Sparse Attention Indexer CUDA op requires DeepGEMM to be installed."
|
||||
@@ -327,14 +446,14 @@ class SparseAttnIndexer(CustomOp):
|
||||
def forward_native(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
if current_platform.is_cuda() or current_platform.is_xpu():
|
||||
return self.forward_cuda(hidden_states, q_fp8, k, weights)
|
||||
return self.forward_cuda(hidden_states, q_quant, k, weights)
|
||||
elif current_platform.is_rocm():
|
||||
return self.forward_hip(hidden_states, q_fp8, k, weights)
|
||||
return self.forward_hip(hidden_states, q_quant, k, weights)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"SparseAttnIndexer native forward is only implemented for "
|
||||
@@ -344,15 +463,22 @@ class SparseAttnIndexer(CustomOp):
|
||||
def forward_cuda(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
# FP8 path: single tensor (per-token scale is folded into `weights`).
|
||||
# FP4 path: (values, scales) tuple with scales required by the kernel.
|
||||
if isinstance(q_quant, tuple):
|
||||
q_values, q_scale = q_quant
|
||||
else:
|
||||
q_values, q_scale = q_quant, None
|
||||
return torch.ops.vllm.sparse_attn_indexer(
|
||||
hidden_states,
|
||||
_encode_layer_name(self.k_cache.prefix),
|
||||
self.k_cache.kv_cache,
|
||||
q_fp8,
|
||||
q_values,
|
||||
q_scale,
|
||||
k,
|
||||
weights,
|
||||
self.quant_block_size,
|
||||
@@ -362,21 +488,30 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
self.skip_k_cache_insert,
|
||||
self.use_fp4_cache,
|
||||
)
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
assert not self.skip_k_cache_insert, (
|
||||
"AMD platform doesn't support skip cache insert yet"
|
||||
)
|
||||
assert not self.use_fp4_cache, "AMD platform doesn't support fp4 cache yet"
|
||||
assert isinstance(q_quant, torch.Tensor), (
|
||||
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
|
||||
)
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
|
||||
hidden_states,
|
||||
_encode_layer_name(self.k_cache.prefix),
|
||||
self.k_cache.kv_cache,
|
||||
q_fp8,
|
||||
q_quant,
|
||||
k,
|
||||
weights,
|
||||
self.quant_block_size,
|
||||
|
||||
@@ -299,6 +299,13 @@ def cpu_unquantized_gemm(
|
||||
return layer.cpu_linear(x, weight, bias)
|
||||
|
||||
|
||||
def cublas_gemm_bf16_bf16_fp32(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
):
|
||||
return ops.router_gemm_bf16_fp32(x, weight)
|
||||
|
||||
|
||||
def dispatch_unquantized_gemm() -> Callable[..., torch.Tensor]:
|
||||
if current_platform.is_rocm():
|
||||
return rocm_unquantized_gemm
|
||||
|
||||
@@ -107,6 +107,31 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
quant_config = getattr(model_config.hf_config, "quantization_config", None)
|
||||
if quant_config is not None and quant_config.get("quant_method") == "fp8":
|
||||
model_type = getattr(model_config.hf_config, "model_type", None)
|
||||
if model_type == "deepseek_v4":
|
||||
model_config.hf_config.quantization_config["quant_method"] = (
|
||||
"deepseek_v4_fp8"
|
||||
)
|
||||
|
||||
hf_text_quant_config = getattr(
|
||||
model_config.hf_text_config, "quantization_config", None
|
||||
)
|
||||
if (
|
||||
hf_text_quant_config is not None
|
||||
and hf_text_quant_config.get("quant_method") == "fp8"
|
||||
):
|
||||
model_type = getattr(model_config.hf_text_config, "model_type", None)
|
||||
if model_type == "deepseek_v4":
|
||||
model_config.hf_text_config.quantization_config["quant_method"] = (
|
||||
"deepseek_v4_fp8"
|
||||
)
|
||||
|
||||
|
||||
class GptOssForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
@@ -635,6 +660,7 @@ class VoyageQwen3BidirectionalEmbedModelConfig(VerifyAndUpdateConfig):
|
||||
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
|
||||
"ColQwen3_5": Qwen3_5ForConditionalGenerationConfig,
|
||||
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
|
||||
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
|
||||
"FalconMambaForCausalLM": MambaModelConfig,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,483 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MTP draft model for DeepSeek V4 (internal codename: DeepseekV4).
|
||||
|
||||
Split from ``deepseek_mtp.py`` because the V4 architecture introduces several
|
||||
pieces that have no analogue in V3/V32:
|
||||
* separate ``e_proj`` / ``h_proj`` with fp8 linear quantization (instead of
|
||||
the fused ``eh_proj``);
|
||||
* ``hc_head`` hypercompressed vocab projection applied in ``compute_logits``;
|
||||
* ``DeepseekV4DecoderLayer`` with its own aux-stream management;
|
||||
* V4-specific checkpoint weight-name remapping in ``load_weights``.
|
||||
"""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.multi_stream_utils import AuxStreamType
|
||||
|
||||
from .deepseek_mtp import SharedHead
|
||||
from .deepseek_v2 import get_spec_layer_idx_from_weight_name
|
||||
from .deepseek_v4 import (
|
||||
DeepseekV4DecoderLayer,
|
||||
hc_head,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
)
|
||||
from .utils import maybe_prefix
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MoE expert scales are fused into per-layer w13/w2 tensors; other FP8 linear
|
||||
# scales use `.weight_scale_inv`. Mirrors the regex in
|
||||
# DeepseekV4ForCausalLM.hf_to_vllm_mapper.
|
||||
_EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$")
|
||||
|
||||
|
||||
class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
# V4 keeps e_ and h_ proj separate (with fp8 linear quant) rather than
|
||||
# fusing them the way V3 does with eh_proj.
|
||||
self.e_proj = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.h_proj = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
self.hc_eps = config.hc_eps
|
||||
self.hc_mult = config.hc_mult
|
||||
self.hc_dim = self.hc_mult * config.hidden_size
|
||||
self.hc_head_fn = nn.Parameter(
|
||||
torch.empty(self.hc_mult, self.hc_dim, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_base = nn.Parameter(
|
||||
torch.empty(self.hc_mult, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_scale = nn.Parameter(
|
||||
torch.empty(1, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
self.aux_stream_dict = {
|
||||
AuxStreamType.Attention: torch.cuda.Stream(),
|
||||
}
|
||||
self.mtp_block = DeepseekV4DecoderLayer(
|
||||
vllm_config,
|
||||
prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
aux_stream_dict=self.aux_stream_dict,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None
|
||||
# masking inputs at position 0, as not needed by MTP
|
||||
inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
|
||||
inputs_embeds = self.enorm(inputs_embeds)
|
||||
|
||||
# Target stashes pre-hc_head residual as flat (T, hc_mult * D);
|
||||
# reshape to (T, hc_mult, D) — the training-time layout.
|
||||
previous_hidden_states = previous_hidden_states.view(
|
||||
-1, self.hc_mult, self.config.hidden_size
|
||||
)
|
||||
previous_hidden_states = self.hnorm(previous_hidden_states)
|
||||
hidden_states = self.h_proj(previous_hidden_states) + self.e_proj(
|
||||
inputs_embeds
|
||||
).unsqueeze(-2)
|
||||
hidden_states = self.mtp_block(
|
||||
positions=positions, x=hidden_states, input_ids=None
|
||||
)
|
||||
# Return the flat pre-hc_head residual so it can be re-fed as the
|
||||
# next spec step's `previous_hidden_states` when
|
||||
# num_speculative_tokens > 1. hc_head is deferred to compute_logits.
|
||||
return hidden_states.flatten(1)
|
||||
|
||||
|
||||
class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = config.num_nextn_predict_layers
|
||||
self.device = current_platform.device_type
|
||||
|
||||
topk_tokens = config.index_topk
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
# to map the exact layer index from weights
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
str(idx): DeepSeekV4MultiTokenPredictorLayer(
|
||||
vllm_config,
|
||||
self.topk_indices_buffer,
|
||||
f"{prefix}.layers.{idx}",
|
||||
)
|
||||
for idx in range(
|
||||
self.mtp_start_layer_idx,
|
||||
self.mtp_start_layer_idx + self.num_mtp_layers,
|
||||
)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
# MTP forward returns the pre-hc_head residual (T, hc_mult * D); apply
|
||||
# hc_head here so logits are computed from the dense hidden state.
|
||||
hidden_states = hidden_states.view(
|
||||
-1, mtp_layer.hc_mult, mtp_layer.config.hidden_size
|
||||
)
|
||||
hidden_states = hc_head(
|
||||
hidden_states,
|
||||
mtp_layer.hc_head_fn,
|
||||
mtp_layer.hc_head_scale,
|
||||
mtp_layer.hc_head_base,
|
||||
mtp_layer.rms_norm_eps,
|
||||
mtp_layer.hc_eps,
|
||||
)
|
||||
logits = self.logits_processor(
|
||||
mtp_layer.shared_head.head, mtp_layer.shared_head(hidden_states)
|
||||
)
|
||||
return logits
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class DeepSeekV4MTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = DeepSeekV4MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Weight name remapping for checkpoint compatibility.
|
||||
# Maps checkpoint weight paths to model parameter paths.
|
||||
WEIGHT_NAME_REMAPPING: dict[str, str] = {
|
||||
".emb.tok_emb.weight": ".embed_tokens.weight",
|
||||
".head.weight": ".shared_head.head.weight",
|
||||
".norm.weight": ".shared_head.norm.weight",
|
||||
}
|
||||
|
||||
def _remap_weight_name(name: str) -> str:
|
||||
"""Remap checkpoint weight names to model parameter names."""
|
||||
for old_pattern, new_pattern in WEIGHT_NAME_REMAPPING.items():
|
||||
if old_pattern in name:
|
||||
name = name.replace(old_pattern, new_pattern)
|
||||
return name
|
||||
|
||||
def _find_mtp_layer_idx(name: str) -> int:
|
||||
subnames = name.split(".")
|
||||
for subname in subnames:
|
||||
try:
|
||||
# we return the first encountered integer
|
||||
return int(subname)
|
||||
except ValueError:
|
||||
continue
|
||||
return 0
|
||||
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "w1", 0),
|
||||
("gate_up_proj", "w3", 1),
|
||||
("attn.fused_wqa_wkv", "attn.wq_a", 0),
|
||||
("attn.fused_wqa_wkv", "attn.wkv", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
n_head = self.config.num_attention_heads
|
||||
n_local_head = n_head // tp_size
|
||||
head_rank_start = n_local_head * tp_rank
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
if first_layer.mtp_block.ffn.use_mega_moe:
|
||||
expert_mapping = make_deepseek_v4_expert_params_mapping(
|
||||
self.config.n_routed_experts
|
||||
)
|
||||
else:
|
||||
expert_mapping = FusedMoE.make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mtp_layer_idx = _find_mtp_layer_idx(name)
|
||||
# V4 checkpoints store MTP weights as `mtp.{i}.*`; remap to
|
||||
# `model.layers.{num_hidden_layers + i}.*` so that
|
||||
# get_spec_layer_idx_from_weight_name can identify them.
|
||||
name = name.replace(
|
||||
f"mtp.{mtp_layer_idx}.",
|
||||
f"model.layers.{self.config.num_hidden_layers + mtp_layer_idx}.",
|
||||
)
|
||||
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
if spec_layer is None:
|
||||
continue
|
||||
|
||||
name = _remap_weight_name(name)
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
|
||||
if spec_layer != self.model.mtp_start_layer_idx and ".layers" not in name:
|
||||
continue
|
||||
if name.endswith(".scale"):
|
||||
suffix = (
|
||||
".weight_scale"
|
||||
if _EXPERT_SCALE_RE.search(name)
|
||||
else ".weight_scale_inv"
|
||||
)
|
||||
name = name.removesuffix(".scale") + suffix
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if ".experts." in name:
|
||||
continue
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(name)
|
||||
break
|
||||
else:
|
||||
if ".experts." in name:
|
||||
# Reinterpret E8M0 scales as uint8 to preserve raw
|
||||
# exponent bytes; numeric copy_() would zero them.
|
||||
# Mirrors the main DeepseekV4 loader.
|
||||
if (
|
||||
"weight_scale" in name
|
||||
and loaded_weight.dtype == torch.float8_e8m0fnu
|
||||
):
|
||||
loaded_weight = loaded_weight.view(torch.uint8)
|
||||
for mapping in expert_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
param = params_dict[name_mapped]
|
||||
# We should ask the weight loader to return success or not
|
||||
# here since otherwise we may skip experts with other
|
||||
# available replicas.
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
continue
|
||||
elif "attn_sink" in name:
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
else:
|
||||
if ".shared_experts.w2" in name:
|
||||
name = name.replace(
|
||||
".shared_experts.w2", ".shared_experts.down_proj"
|
||||
)
|
||||
if name.endswith(".ffn.gate.bias"):
|
||||
name = name.replace(".bias", ".e_score_correction_bias")
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
loaded_layers: set[int] = set()
|
||||
for param_name in loaded_params:
|
||||
spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name)
|
||||
if spec_layer is not None:
|
||||
loaded_layers.add(spec_layer)
|
||||
for layer_idx in range(
|
||||
self.model.mtp_start_layer_idx,
|
||||
self.model.mtp_start_layer_idx + self.model.num_mtp_layers,
|
||||
):
|
||||
if layer_idx not in loaded_layers:
|
||||
raise ValueError(
|
||||
f"MTP speculative decoding layer {layer_idx} weights "
|
||||
f"missing from checkpoint. The checkpoint may have "
|
||||
f"been quantized without including the MTP layers. "
|
||||
f"Use a checkpoint that includes MTP layer weights, "
|
||||
f"or disable speculative decoding."
|
||||
)
|
||||
self.finalize_mega_moe_weights()
|
||||
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
for layer in self.model.layers.values():
|
||||
layer.mtp_block.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
"enorm",
|
||||
"hnorm",
|
||||
"h_proj",
|
||||
"e_proj",
|
||||
"shared_head",
|
||||
"hc_head_fn",
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
]
|
||||
shared_weight_names = ["embed_tokens"]
|
||||
spec_layer_weight = False
|
||||
shared_weight = False
|
||||
for weight_name in spec_layer_weight_names:
|
||||
if weight_name in name:
|
||||
spec_layer_weight = True
|
||||
if weight_name in shared_weight_names:
|
||||
shared_weight = True
|
||||
break
|
||||
if not spec_layer_weight:
|
||||
# treat rest weights as weights for transformer layer block
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
|
||||
)
|
||||
elif shared_weight:
|
||||
# treat shared weights as top level weights
|
||||
name = name.replace(f"model.layers.{spec_layer}.", "model.")
|
||||
return name
|
||||
@@ -96,6 +96,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
|
||||
"DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
|
||||
"DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
|
||||
"DeepseekV4ForCausalLM": ("deepseek_v4", "DeepseekV4ForCausalLM"),
|
||||
"Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
|
||||
"Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
|
||||
"Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
|
||||
@@ -586,6 +587,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
|
||||
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
|
||||
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
|
||||
"DeepSeekV4MTPModel": ("deepseek_v4_mtp", "DeepSeekV4MTP"),
|
||||
"ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
|
||||
"ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
|
||||
"Exaone4_5_MTP": ("exaone4_5_mtp", "Exaone4_5_MTP"),
|
||||
|
||||
Reference in New Issue
Block a user