Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -802,18 +802,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
)
|
||||
if cache is not None:
|
||||
mirror_target = cache.mirror_write_target(kv_c_normed.shape[0])
|
||||
if mirror_target is not None:
|
||||
mirror_cache, mirror_slots = mirror_target
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
mirror_cache,
|
||||
mirror_slots,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
)
|
||||
|
||||
def prepare_kv_cache_update(
|
||||
self, attn_metadata: "MLACommonMetadata | None"
|
||||
@@ -1272,23 +1260,34 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
quark_quantize_weight_to_mxfp4,
|
||||
)
|
||||
|
||||
self.W_K, self.W_K_scale = quark_quantize_weight_to_mxfp4(W_UK)
|
||||
# Convert from (L, N, P) to (N, L, P)
|
||||
self.W_K = self.W_K.transpose(0, 1)
|
||||
self.W_K_scale = self.W_K_scale.transpose(0, 1)
|
||||
|
||||
self.W_V, self.W_V_scale = quark_quantize_weight_to_mxfp4(
|
||||
W_UV.permute(1, 2, 0)
|
||||
)
|
||||
W_K, W_K_scale = quark_quantize_weight_to_mxfp4(W_UK)
|
||||
W_V, W_V_scale = quark_quantize_weight_to_mxfp4(W_UV.permute(1, 2, 0))
|
||||
# Copy in place on reload so captured CUDA graphs stay valid.
|
||||
# Convert W_K from (L, N, P) to (N, L, P).
|
||||
for name, tensor in (
|
||||
("W_K", W_K.transpose(0, 1)),
|
||||
("W_K_scale", W_K_scale.transpose(0, 1)),
|
||||
("W_V", W_V),
|
||||
("W_V_scale", W_V_scale),
|
||||
):
|
||||
replace_parameter(self, name, tensor, prefer_copy=True)
|
||||
elif self.is_aiter_triton_fp8_bmm_enabled:
|
||||
W_K = W_UK.transpose(0, 1) # 16 512 128
|
||||
W_V = W_UV.permute(1, 2, 0) # 16 128 512
|
||||
self.W_K, self.W_K_scale = dynamic_per_batched_tensor_quant(
|
||||
W_K, W_K_scale = dynamic_per_batched_tensor_quant(
|
||||
W_K, dtype=current_platform.fp8_dtype()
|
||||
)
|
||||
self.W_V, self.W_V_scale = dynamic_per_batched_tensor_quant(
|
||||
W_V, W_V_scale = dynamic_per_batched_tensor_quant(
|
||||
W_V, dtype=current_platform.fp8_dtype()
|
||||
)
|
||||
# Copy in place on reload so captured CUDA graphs stay valid.
|
||||
for name, tensor in (
|
||||
("W_K", W_K),
|
||||
("W_K_scale", W_K_scale),
|
||||
("W_V", W_V),
|
||||
("W_V_scale", W_V_scale),
|
||||
):
|
||||
replace_parameter(self, name, tensor, prefer_copy=True)
|
||||
|
||||
# The kernel operates on non-padded inputs. Hence, pre-compiling
|
||||
# triton kernel to avoid runtime compilation for unseen batch sizes
|
||||
@@ -1327,8 +1326,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# Convert from (L, N, P) to (N, P, L)
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
if self.dcp_q_replicate:
|
||||
self.W_UK_T_dcp_qrep = get_dcp_group().all_gather(
|
||||
self.W_UK_T.contiguous(), dim=0
|
||||
replace_parameter(
|
||||
self,
|
||||
"W_UK_T_dcp_qrep",
|
||||
get_dcp_group().all_gather(self.W_UK_T.contiguous(), dim=0),
|
||||
prefer_copy=True,
|
||||
)
|
||||
|
||||
# If we should not load quant weights, we initialize the scales to 1.0
|
||||
@@ -2384,8 +2386,12 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self.non_causal_multi_token_decode = getattr(
|
||||
kv_cache_spec, "non_causal_multi_token_decode", False
|
||||
# merge() unions this flag across the KV group, so a shared target/draft
|
||||
# group would mark the target builder as non-causal.
|
||||
ctx = self.compilation_config.static_forward_context
|
||||
self.non_causal_multi_token_decode = any(
|
||||
getattr(ctx[name], "non_causal_multi_token_decode", False)
|
||||
for name in layer_names
|
||||
)
|
||||
self._validate_dspark_dcp_support(supports_dcp_with_varlen)
|
||||
|
||||
@@ -3033,7 +3039,13 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
attn_metadata: MLACommonMetadata,
|
||||
k_scale: torch.Tensor,
|
||||
dcp_world_size: int,
|
||||
fused_mla_kv_concat_fn: Callable[
|
||||
[torch.Tensor, torch.Tensor, bool], tuple[torch.Tensor, torch.Tensor]
|
||||
]
|
||||
| None = None,
|
||||
):
|
||||
# fused_mla_kv_concat_fn(kv_nope, k_pe, use_fp8_prefill) -> (k, v) replaces
|
||||
# the per-chunk cast + split + concat below with one fused kernel.
|
||||
assert attn_metadata.prefill is not None
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
assert prefill_metadata.prefill_backend is not None
|
||||
@@ -3127,11 +3139,16 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
|
||||
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
|
||||
)
|
||||
if use_fp8_prefill:
|
||||
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
|
||||
k_pe = k_pe.to(prefill_metadata.q_data_type)
|
||||
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
|
||||
k = self._concat_k_nope_k_pe(k_nope, k_pe)
|
||||
if fused_mla_kv_concat_fn is not None:
|
||||
k, v = fused_mla_kv_concat_fn(kv_nope, k_pe, use_fp8_prefill)
|
||||
else:
|
||||
if use_fp8_prefill:
|
||||
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
|
||||
k_pe = k_pe.to(prefill_metadata.q_data_type)
|
||||
k_nope, v = kv_nope.split(
|
||||
[self.qk_nope_head_dim, self.v_head_dim], dim=-1
|
||||
)
|
||||
k = self._concat_k_nope_k_pe(k_nope, k_pe)
|
||||
|
||||
attn_output, attn_softmax_lse = (
|
||||
prefill_metadata.prefill_backend.run_prefill_context_chunk(
|
||||
|
||||
@@ -1293,6 +1293,7 @@ class FusedMoEConfig:
|
||||
has_bias: bool = False
|
||||
is_lora_enabled: bool = False
|
||||
has_hash_routing: bool = False
|
||||
shared_expert_prefix: str | None = None
|
||||
|
||||
# When True, the MoE skips its final cross-rank all-reduce (and the separate
|
||||
# shared-expert reduce), returning the partial per-rank sum. The caller is
|
||||
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 4,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 4,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 16,
|
||||
"num_stages": 4,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 16,
|
||||
"num_stages": 4,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 4,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
}
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 16,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 8,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3,
|
||||
"grf_mode": "256"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,466 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepGEMM MegaMoE backends: fused EP dispatch, expert GEMMs and combine."""
|
||||
|
||||
from inspect import signature
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def ue8m0_uint8_to_float(sf: torch.Tensor) -> torch.Tensor:
|
||||
"""Reinterpret a uint8 UE8M0 exponent tensor as float32 scale values."""
|
||||
return (sf.to(torch.int32) << 23).view(torch.float32)
|
||||
|
||||
|
||||
class DeepGemmMegaMoEBackend:
|
||||
"""Abstract MegaMoE backend that hides deep_gemm-specific details.
|
||||
|
||||
A backend advertises its expected hidden-state quantization layout via
|
||||
``hidden_quant`` and its expected MMA type via ``mma_type``. It is
|
||||
responsible for the routed-expert weight transform run at load time, the
|
||||
optional shared-expert weight transform, and the actual MegaMoE kernel
|
||||
invocation at inference time. Runtime capability checks live in
|
||||
:func:`get_deep_gemm_mega_moe_backend`: obtaining a backend already implies
|
||||
that the current device and shapes are supported.
|
||||
"""
|
||||
|
||||
# Human-readable MMA type tag used by deep_gemm helpers. SM100 default.
|
||||
mma_type: str = "fp8xfp4"
|
||||
hidden_quant: QuantKey = kMxfp8Dynamic
|
||||
|
||||
def transform_weights(
|
||||
self,
|
||||
*,
|
||||
w13_weight: torch.Tensor,
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
num_local_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
activation: str | None = None,
|
||||
) -> tuple[
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
]:
|
||||
raise NotImplementedError
|
||||
|
||||
def supports_shared_experts(self, deep_gemm) -> bool:
|
||||
return False
|
||||
|
||||
def transform_shared_expert_weights(
|
||||
self,
|
||||
*,
|
||||
shared_experts: nn.Module,
|
||||
num_shared_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
prefix: str,
|
||||
) -> (
|
||||
tuple[
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
]
|
||||
| None
|
||||
):
|
||||
"""Return ``(l1, l2)`` transformed shared-expert weights, or ``None``.
|
||||
|
||||
``shared_experts`` must expose ``gate_up_proj`` and ``down_proj``
|
||||
linear layers. A ``None`` return means the backend cannot fuse shared
|
||||
experts for this module; the caller should fall back to a separate
|
||||
shared MLP.
|
||||
"""
|
||||
return None
|
||||
|
||||
def run_mega_moe(
|
||||
self,
|
||||
*,
|
||||
y: torch.Tensor,
|
||||
l1_weights: tuple[torch.Tensor, torch.Tensor],
|
||||
l2_weights: tuple[torch.Tensor, torch.Tensor],
|
||||
symm_buffer,
|
||||
activation_clamp: float | None,
|
||||
fast_math: bool,
|
||||
activation: str | None = None,
|
||||
activation_alpha: float | None = None,
|
||||
activation_beta: float | None = None,
|
||||
shared_l1_weights: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
shared_l2_weights: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def supports_bf16_mega_gate(self) -> bool:
|
||||
"""Whether this backend can run the DeepGEMM bf16 MegaMoE gate kernel.
|
||||
|
||||
Defaults to ``False`` (the symbol is not guaranteed to exist in every
|
||||
deep_gemm build). Backends that target a deep_gemm build exposing the
|
||||
``bf16_mega_gate`` symbol override this and :meth:`bf16_mega_gate`.
|
||||
"""
|
||||
return False
|
||||
|
||||
def bf16_mega_gate(
|
||||
self,
|
||||
*,
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
num_topk: int,
|
||||
scoring_func: str,
|
||||
routed_scaling_factor: float,
|
||||
ep_rank: int,
|
||||
bias: torch.Tensor | None = None,
|
||||
image_bias: torch.Tensor | None = None,
|
||||
image_token_mask: torch.Tensor | None = None,
|
||||
fix_routing_mask: torch.Tensor | None = None,
|
||||
unmapped_topk_idx: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class DeepGemmSm100MegaMoEBackend(DeepGemmMegaMoEBackend):
|
||||
"""Default MegaMoE backend targeting SM100 via deep_gemm fp8_fp4_mega_moe."""
|
||||
|
||||
mma_type = "fp8xfp4"
|
||||
hidden_quant = kMxfp8Dynamic
|
||||
|
||||
def transform_weights(
|
||||
self,
|
||||
*,
|
||||
w13_weight: torch.Tensor,
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
num_local_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
activation: str | None = None,
|
||||
) -> tuple[
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
]:
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
|
||||
w13_scale = deep_gemm.transform_sf_into_required_layout(
|
||||
ue8m0_uint8_to_float(w13_weight_scale).contiguous(),
|
||||
2 * intermediate_size,
|
||||
hidden_size,
|
||||
(1, 32),
|
||||
num_local_experts,
|
||||
)
|
||||
w2_scale = deep_gemm.transform_sf_into_required_layout(
|
||||
ue8m0_uint8_to_float(w2_weight_scale).contiguous(),
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
(1, 32),
|
||||
num_local_experts,
|
||||
)
|
||||
kwargs = {} if activation is None else {"activation": activation}
|
||||
return deep_gemm.transform_weights_for_mega_moe(
|
||||
(w13_weight.view(torch.int8).contiguous(), w13_scale),
|
||||
(w2_weight.view(torch.int8).contiguous(), w2_scale),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def supports_shared_experts(self, deep_gemm) -> bool:
|
||||
"""Check the Python API before touching a symmetric-memory group.
|
||||
|
||||
This also gives users of an older precompiled vLLM wheel a safe serial
|
||||
fallback instead of failing halfway through multi-rank buffer setup.
|
||||
"""
|
||||
try:
|
||||
buffer_params = signature(deep_gemm.get_symm_buffer_for_mega_moe).parameters
|
||||
kernel_params = signature(deep_gemm.fp8_fp4_mega_moe).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return (
|
||||
hasattr(deep_gemm, "get_block_m_for_mega_moe")
|
||||
and hasattr(deep_gemm, "transform_weights_for_mega_moe")
|
||||
and "num_shared_experts" in buffer_params
|
||||
and "shared_l1_weights" in kernel_params
|
||||
and "shared_l2_weights" in kernel_params
|
||||
)
|
||||
|
||||
def transform_shared_expert_weights(
|
||||
self,
|
||||
*,
|
||||
shared_experts: nn.Module,
|
||||
num_shared_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
prefix: str,
|
||||
) -> (
|
||||
tuple[
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
]
|
||||
| None
|
||||
):
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
|
||||
gate_up = shared_experts.gate_up_proj
|
||||
down = shared_experts.down_proj
|
||||
gate_up_weight = gate_up.weight.data
|
||||
gate_up_scale = (
|
||||
gate_up.weight_scale
|
||||
if hasattr(gate_up, "weight_scale")
|
||||
else gate_up.weight_scale_inv
|
||||
).data
|
||||
down_weight = down.weight.data
|
||||
down_scale = (
|
||||
down.weight_scale
|
||||
if hasattr(down, "weight_scale")
|
||||
else down.weight_scale_inv
|
||||
).data
|
||||
|
||||
# MegaMoE's shared FP8 MMA consumes a 1x32 scale for every weight row,
|
||||
# while the checkpoint uses coarser block-FP8 scales (usually
|
||||
# 128x128). Build a dedicated, numerically equivalent scale view.
|
||||
ue8m0_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8, torch.int32)
|
||||
if (
|
||||
gate_up_scale.dtype in ue8m0_scale_dtypes
|
||||
and down_scale.dtype in ue8m0_scale_dtypes
|
||||
):
|
||||
gate_up_scale = self._prepare_shared_expert_scale(
|
||||
deep_gemm,
|
||||
gate_up,
|
||||
gate_up_scale,
|
||||
gate_up_weight.shape[0],
|
||||
gate_up_weight.shape[1],
|
||||
prefix,
|
||||
)
|
||||
down_scale = self._prepare_shared_expert_scale(
|
||||
deep_gemm,
|
||||
down,
|
||||
down_scale,
|
||||
down_weight.shape[0],
|
||||
down_weight.shape[1],
|
||||
prefix,
|
||||
)
|
||||
|
||||
if gate_up_scale is None or down_scale is None:
|
||||
return None
|
||||
|
||||
shared_intermediate_size = intermediate_size * num_shared_experts
|
||||
expected_gate_up_shape = (
|
||||
2 * shared_intermediate_size,
|
||||
hidden_size,
|
||||
)
|
||||
expected_down_shape = (hidden_size, shared_intermediate_size)
|
||||
if (
|
||||
gate_up_weight.dtype != torch.float8_e4m3fn
|
||||
or down_weight.dtype != torch.float8_e4m3fn
|
||||
or gate_up_scale.dtype != torch.int32
|
||||
or down_scale.dtype != torch.int32
|
||||
or tuple(gate_up_weight.shape) != expected_gate_up_shape
|
||||
or tuple(down_weight.shape) != expected_down_shape
|
||||
):
|
||||
logger.warning(
|
||||
"Disabling native MegaMoE shared-expert fusion for %s: expected "
|
||||
"replicated block-FP8 weights with gate_up=%s, down=%s, and "
|
||||
"DeepGEMM int32 scales; got gate_up=%s/%s/%s and down=%s/%s/%s.",
|
||||
prefix,
|
||||
expected_gate_up_shape,
|
||||
expected_down_shape,
|
||||
tuple(gate_up_weight.shape),
|
||||
gate_up_weight.dtype,
|
||||
gate_up_scale.dtype,
|
||||
tuple(down_weight.shape),
|
||||
down_weight.dtype,
|
||||
down_scale.dtype,
|
||||
)
|
||||
return None
|
||||
|
||||
transformed_l1, transformed_l2 = deep_gemm.transform_weights_for_mega_moe(
|
||||
(gate_up_weight, gate_up_scale),
|
||||
(down_weight, down_scale),
|
||||
)
|
||||
# L1 interleaving allocates a full copy. Re-home the loader Parameter on
|
||||
# that storage so the original 2*intermediate*hidden FP8 tensor can be
|
||||
# released instead of adding roughly 0.7 GiB per rank on DSV4-Flash.
|
||||
# The generic linear post-load hook may still repack the serial scales,
|
||||
# but this shared MLP is never called after native fusion is enabled.
|
||||
gate_up.weight.data = transformed_l1[0]
|
||||
transformed_l1 = (gate_up.weight.data, transformed_l1[1])
|
||||
return transformed_l1, transformed_l2
|
||||
|
||||
@staticmethod
|
||||
def _prepare_shared_expert_scale(
|
||||
deep_gemm,
|
||||
linear: nn.Module,
|
||||
scale: torch.Tensor,
|
||||
mn: int,
|
||||
k: int,
|
||||
prefix: str,
|
||||
) -> torch.Tensor | None:
|
||||
block_size = getattr(linear, "weight_block_size", None)
|
||||
if block_size is None or len(block_size) != 2:
|
||||
logger.warning(
|
||||
"Disabling native MegaMoE shared-expert fusion for %s: "
|
||||
"shared FP8 weight block size is unavailable.",
|
||||
prefix,
|
||||
)
|
||||
return None
|
||||
|
||||
block_m, block_k = block_size
|
||||
num_k_blocks = (k + block_k - 1) // block_k
|
||||
# The linear hook's DeepGEMM layout packs four k-block bytes per int32.
|
||||
packed = scale.dtype == torch.int32
|
||||
expected_shape = (
|
||||
(mn, (num_k_blocks + 3) // 4)
|
||||
if packed
|
||||
else ((mn + block_m - 1) // block_m, num_k_blocks)
|
||||
)
|
||||
if block_k % 32 != 0 or tuple(scale.shape) != expected_shape:
|
||||
logger.warning(
|
||||
"Disabling native MegaMoE shared-expert fusion for %s: "
|
||||
"cannot convert shared scale shape %s with block size %s "
|
||||
"to MegaMoE's 1x32 layout for weight (%d, %d).",
|
||||
prefix,
|
||||
tuple(scale.shape),
|
||||
tuple(block_size),
|
||||
mn,
|
||||
k,
|
||||
)
|
||||
return None
|
||||
|
||||
if packed:
|
||||
row_scale = scale.flatten().view(torch.uint8).view(mn, -1)[:, :num_k_blocks]
|
||||
else:
|
||||
row_scale = scale.view(torch.uint8).repeat_interleave(block_m, dim=0)[:mn]
|
||||
scale_1x32 = (
|
||||
ue8m0_uint8_to_float(row_scale)
|
||||
.repeat_interleave(block_k // 32, dim=1)[:, : k // 32]
|
||||
.contiguous()
|
||||
)
|
||||
# The grouped API is used with a singleton dimension to request the
|
||||
# MN-major, TMA-aligned packed-UE8M0 strides, then squeezed back to the
|
||||
# 2D layout required for a shared expert.
|
||||
return deep_gemm.transform_sf_into_required_layout(
|
||||
scale_1x32.unsqueeze(0),
|
||||
mn,
|
||||
k,
|
||||
(1, 32),
|
||||
1,
|
||||
).squeeze(0)
|
||||
|
||||
def run_mega_moe(
|
||||
self,
|
||||
*,
|
||||
y: torch.Tensor,
|
||||
l1_weights: tuple[torch.Tensor, torch.Tensor],
|
||||
l2_weights: tuple[torch.Tensor, torch.Tensor],
|
||||
symm_buffer,
|
||||
activation_clamp: float | None,
|
||||
fast_math: bool,
|
||||
activation: str | None = None,
|
||||
activation_alpha: float | None = None,
|
||||
activation_beta: float | None = None,
|
||||
shared_l1_weights: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
shared_l2_weights: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
kwargs: dict[str, Any] = {
|
||||
name: value
|
||||
for name, value in (
|
||||
("activation", activation),
|
||||
("activation_alpha", activation_alpha),
|
||||
("activation_beta", activation_beta),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
if shared_l1_weights is not None and shared_l2_weights is not None:
|
||||
kwargs["shared_l1_weights"] = shared_l1_weights
|
||||
kwargs["shared_l2_weights"] = shared_l2_weights
|
||||
deep_gemm.fp8_fp4_mega_moe(
|
||||
y,
|
||||
l1_weights,
|
||||
l2_weights,
|
||||
symm_buffer,
|
||||
activation_clamp=activation_clamp,
|
||||
fast_math=fast_math,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def supports_bf16_mega_gate(self) -> bool:
|
||||
"""SM100 deep_gemm build is expected to provide the bf16 gate kernel."""
|
||||
return True
|
||||
|
||||
def bf16_mega_gate(
|
||||
self,
|
||||
*,
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
num_topk: int,
|
||||
scoring_func: str,
|
||||
routed_scaling_factor: float,
|
||||
ep_rank: int,
|
||||
bias: torch.Tensor | None = None,
|
||||
image_bias: torch.Tensor | None = None,
|
||||
image_token_mask: torch.Tensor | None = None,
|
||||
fix_routing_mask: torch.Tensor | None = None,
|
||||
unmapped_topk_idx: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from vllm.utils.deep_gemm import bf16_mega_gate
|
||||
|
||||
return bf16_mega_gate(
|
||||
x,
|
||||
weight,
|
||||
num_topk,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
ep_rank=ep_rank,
|
||||
bias=bias,
|
||||
image_bias=image_bias,
|
||||
image_token_mask=image_token_mask,
|
||||
fix_routing_mask=fix_routing_mask,
|
||||
unmapped_topk_idx=unmapped_topk_idx,
|
||||
)
|
||||
|
||||
|
||||
def get_deep_gemm_mega_moe_backend(
|
||||
device: torch.device,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
) -> DeepGemmMegaMoEBackend:
|
||||
"""Return the DeepGEMM MegaMoE backend for ``device``.
|
||||
|
||||
A successful return implies the backend can run on this device with the
|
||||
given ``hidden_size`` / ``intermediate_size``. All runtime capability
|
||||
checks (arch + shape divisibility) live here; there is no separate check
|
||||
on the returned backend.
|
||||
|
||||
Currently only the SM100 deep_gemm backend is registered here. SM90 and
|
||||
other backends are expected to be provided by new backends or plugins
|
||||
that override this factory.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If the device architecture is unsupported.
|
||||
ValueError: If the shapes are unsupported.
|
||||
|
||||
"""
|
||||
if not current_platform.is_device_capability_family(
|
||||
100, device_id=device.index or 0
|
||||
):
|
||||
raise NotImplementedError("DeepGEMM MegaMoE requires SM100 GPUs.")
|
||||
if hidden_size % 128 != 0 or intermediate_size % 128 != 0:
|
||||
raise ValueError(
|
||||
"DeepGEMM MegaMoE requires hidden and intermediate sizes "
|
||||
"to be multiples of 128."
|
||||
)
|
||||
return DeepGemmSm100MegaMoEBackend()
|
||||
@@ -119,6 +119,7 @@ def _fwd_kernel_ep_scatter_1(
|
||||
BLOCK_E: tl.constexpr,
|
||||
BLOCK_EXPERT_NUM: tl.constexpr,
|
||||
ALIGN_M: tl.constexpr,
|
||||
USE_PSUM_LAYOUT: tl.constexpr,
|
||||
):
|
||||
cur_expert = tl.program_id(0)
|
||||
|
||||
@@ -140,20 +141,22 @@ def _fwd_kernel_ep_scatter_1(
|
||||
tl.store(expert_start_loc + cur_expert, cur_expert_start)
|
||||
cur_expert_token_num = tl.load(num_recv_tokens_per_expert + cur_expert)
|
||||
|
||||
m_indices_start_ptr = m_indices + cur_expert_start
|
||||
off_expert = tl.arange(0, BLOCK_E)
|
||||
if USE_PSUM_LAYOUT:
|
||||
# Logical ends include preceding experts' alignment gaps.
|
||||
tl.store(m_indices + cur_expert, cur_expert_start + cur_expert_token_num)
|
||||
else:
|
||||
m_indices_start_ptr = m_indices + cur_expert_start
|
||||
off_expert = tl.arange(0, BLOCK_E)
|
||||
|
||||
# any rows in the per-expert aligned region that do not correspond to
|
||||
# real tokens are left untouched here and should remain initialized to
|
||||
# -1 so DeepGEMM can skip them
|
||||
for start_m in tl.range(0, cur_expert_token_num, BLOCK_E):
|
||||
offs = start_m + off_expert
|
||||
mask = offs < cur_expert_token_num
|
||||
tl.store(
|
||||
m_indices_start_ptr + offs,
|
||||
cur_expert,
|
||||
mask=mask,
|
||||
)
|
||||
# Unwritten padding retains the -1 initialization.
|
||||
for start_m in tl.range(0, cur_expert_token_num, BLOCK_E):
|
||||
offs = start_m + off_expert
|
||||
mask = offs < cur_expert_token_num
|
||||
tl.store(
|
||||
m_indices_start_ptr + offs,
|
||||
cur_expert,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -285,6 +288,7 @@ def ep_scatter(
|
||||
align_m: int = 128,
|
||||
block_size: int = 128,
|
||||
pack_ue8m0: bool = False,
|
||||
use_psum_layout: bool = False,
|
||||
):
|
||||
# BLOCK_E is the m_indices fill-loop tile (masked), independent of align_m.
|
||||
BLOCK_E = 128
|
||||
@@ -295,7 +299,10 @@ def ep_scatter(
|
||||
# grid = (triton.cdiv(hidden_size, BLOCK_D), num_experts)
|
||||
grid = num_experts
|
||||
|
||||
assert m_indices.shape[0] % align_m == 0
|
||||
if use_psum_layout:
|
||||
assert m_indices.shape == (num_experts,)
|
||||
else:
|
||||
assert m_indices.shape[0] % align_m == 0
|
||||
assert expert_start_loc.shape[0] == num_experts
|
||||
|
||||
# pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is.
|
||||
@@ -311,6 +318,7 @@ def ep_scatter(
|
||||
BLOCK_E=BLOCK_E,
|
||||
BLOCK_EXPERT_NUM=triton.next_power_of_2(num_experts),
|
||||
ALIGN_M=align_m,
|
||||
USE_PSUM_LAYOUT=use_psum_layout,
|
||||
)
|
||||
|
||||
grid = min(recv_topk.shape[0], 1024 * 8)
|
||||
@@ -407,7 +415,7 @@ def _fwd_kernel_ep_gather(
|
||||
|
||||
tl.store(
|
||||
output_tensor
|
||||
+ cur_token * output_tensor_stride0
|
||||
+ tl.cast(cur_token, tl.int64) * output_tensor_stride0
|
||||
+ cur_block * BLOCK_D
|
||||
+ off_d,
|
||||
accumulator.to(output_tensor.dtype.element_ty),
|
||||
@@ -465,6 +473,7 @@ def deepgemm_moe_permute(
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
aq_out: torch.Tensor | None = None,
|
||||
block_size: int | None = None,
|
||||
use_psum_layout: bool = False,
|
||||
):
|
||||
assert aq.ndim == 2
|
||||
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
|
||||
@@ -509,17 +518,15 @@ def deepgemm_moe_permute(
|
||||
else:
|
||||
aq_scale_out = torch.zeros((M_sum, sf_k), device=device, dtype=torch.float32)
|
||||
|
||||
# DeepGEMM uses negative values in m_indices (here expert_ids) to mark
|
||||
# completely invalid / padded blocks that should be skipped. We always
|
||||
# initialize expert_ids to -1 so any row that is not explicitly written
|
||||
# by the scatter kernel will be treated as invalid and skipped by
|
||||
# DeepGEMM's scheduler.
|
||||
expert_ids = torch.full(
|
||||
(M_sum,),
|
||||
fill_value=-1,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
if use_psum_layout:
|
||||
grouped_layout = torch.empty(
|
||||
(local_num_experts,), device=device, dtype=torch.int32
|
||||
)
|
||||
else:
|
||||
# Negative expert IDs mark padding for the contiguous scheduler.
|
||||
grouped_layout = torch.full(
|
||||
(M_sum,), fill_value=-1, device=device, dtype=torch.int32
|
||||
)
|
||||
inv_perm = torch.empty(topk_ids.shape, device=device, dtype=torch.int32)
|
||||
|
||||
expert_num_tokens = None
|
||||
@@ -541,14 +548,15 @@ def deepgemm_moe_permute(
|
||||
expert_map=expert_map,
|
||||
output_tensor=aq_out,
|
||||
output_tensor_scale=aq_scale_out,
|
||||
m_indices=expert_ids,
|
||||
m_indices=grouped_layout,
|
||||
output_index=inv_perm,
|
||||
align_m=align_used,
|
||||
block_size=block_k,
|
||||
pack_ue8m0=pack_ue8m0,
|
||||
use_psum_layout=use_psum_layout,
|
||||
)
|
||||
|
||||
return aq_out, aq_scale_out, expert_ids, inv_perm, align_used
|
||||
return aq_out, aq_scale_out, grouped_layout, inv_perm, align_used
|
||||
|
||||
|
||||
def deepgemm_unpermute_and_reduce(
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
MoEPermuteScratch,
|
||||
get_moe_permute_scratch,
|
||||
moe_permute,
|
||||
moe_permute_unpermute_supported,
|
||||
moe_unpermute,
|
||||
@@ -304,7 +305,6 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
|
||||
self.ab_strides2 = ab_strides2
|
||||
self.c_strides1 = c_strides1
|
||||
self.c_strides2 = ab_strides1_c_strides2
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
@@ -335,15 +335,13 @@ class CutlassExpertsFp8Base(mk.FusedMoEExpertsModular):
|
||||
return TopKWeightAndReduceDelegate()
|
||||
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if self._permute_scratch is None and moe_permute_unpermute_supported():
|
||||
self._permute_scratch = MoEPermuteScratch(
|
||||
max_num_tokens=self.moe_config.max_num_tokens,
|
||||
topk=self.moe_config.experts_per_token,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
)
|
||||
return self._permute_scratch
|
||||
if not moe_permute_unpermute_supported():
|
||||
return None
|
||||
return get_moe_permute_scratch(
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -1296,7 +1294,6 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
|
||||
self.s_strides2[:, 0] = k
|
||||
|
||||
self.group_size = group_size
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -1357,15 +1354,13 @@ class CutlassExpertsW4A8Fp8(mk.FusedMoEExpertsModular):
|
||||
return self.out_dtype if self.out_dtype is not None else act_dtype
|
||||
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if self._permute_scratch is None and moe_permute_unpermute_supported():
|
||||
self._permute_scratch = MoEPermuteScratch(
|
||||
max_num_tokens=self.moe_config.max_num_tokens,
|
||||
topk=self.moe_config.experts_per_token,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
)
|
||||
return self._permute_scratch
|
||||
if not moe_permute_unpermute_supported():
|
||||
return None
|
||||
return get_moe_permute_scratch(
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
|
||||
@@ -244,7 +244,12 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def _act_mul_quant(
|
||||
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
expert_ends: torch.Tensor | None = None,
|
||||
expert_alignment: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert self.block_shape is not None
|
||||
block_k = self.block_shape[1]
|
||||
@@ -271,6 +276,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
clamp_limit=self.gemm1_clamp_limit,
|
||||
alpha=self.gemm1_alpha,
|
||||
beta=self.gemm1_beta,
|
||||
expert_ends=expert_ends,
|
||||
expert_alignment=expert_alignment,
|
||||
)
|
||||
act_out = torch.empty(
|
||||
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
|
||||
@@ -294,6 +301,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
group_size=block_k,
|
||||
alpha=self.gemm1_alpha,
|
||||
beta=self.gemm1_beta,
|
||||
expert_ends=expert_ends,
|
||||
expert_alignment=expert_alignment,
|
||||
)
|
||||
|
||||
# 3. fallback path for non-SiLU activations in non‑UE8M0 cases.
|
||||
@@ -349,7 +358,20 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
a1q_perm = _resize_cache(
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
|
||||
)
|
||||
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
|
||||
# Both GEMMs and quantization must agree on the live expert ranges.
|
||||
use_psum_layout = (
|
||||
not self.mxfp8
|
||||
and (
|
||||
current_platform.is_device_capability_family(90)
|
||||
or (
|
||||
current_platform.is_device_capability_family(100)
|
||||
and DeepGemmQuantScaleFMT.from_oracle()
|
||||
== DeepGemmQuantScaleFMT.UE8M0
|
||||
)
|
||||
)
|
||||
and activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI_UNINTERLEAVE)
|
||||
)
|
||||
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
|
||||
aq=a1q,
|
||||
aq_scale=a1q_scale,
|
||||
topk_ids=topk_ids,
|
||||
@@ -360,17 +382,21 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
# MXFP8 uses a 32-element activation-scale group (block_shape[1]);
|
||||
# FP8-block keeps the default (128) alignment.
|
||||
block_size=self.block_shape[1] if self.mxfp8 else None,
|
||||
use_psum_layout=use_psum_layout,
|
||||
)
|
||||
assert a1q.size(0) == M_sum
|
||||
|
||||
# MXFP8 (1x32) drives the fp8_fp4-aliased grouped GEMM with recipe
|
||||
# (1, 32); the FP8 block path keeps the default (128) recipe.
|
||||
gemm_kwargs = (
|
||||
gemm_kwargs: dict = (
|
||||
{"recipe_a": (1, self.block_shape[1]), "recipe_b": (1, self.block_shape[1])}
|
||||
if self.mxfp8
|
||||
else {}
|
||||
)
|
||||
|
||||
if use_psum_layout:
|
||||
gemm_kwargs["use_psum_layout"] = True
|
||||
|
||||
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
|
||||
# otherwise the scheduler can pick the wrong expert id from m_indices
|
||||
# under cudagraph replay.
|
||||
@@ -380,7 +406,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
(a1q, a1q_scale),
|
||||
(w1, self.w1_scale),
|
||||
mm1_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
**gemm_kwargs,
|
||||
)
|
||||
|
||||
@@ -389,7 +415,11 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
|
||||
)
|
||||
a2q, a2q_scale = self._act_mul_quant(
|
||||
input=mm1_out.view(-1, N), output=quant_out, activation=activation
|
||||
input=mm1_out.view(-1, N),
|
||||
output=quant_out,
|
||||
activation=activation,
|
||||
expert_ends=grouped_layout if use_psum_layout else None,
|
||||
expert_alignment=align_used if use_psum_layout else 0,
|
||||
)
|
||||
|
||||
mm2_out = _resize_cache(workspace2, (M_sum, K))
|
||||
@@ -397,7 +427,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
(a2q, a2q_scale),
|
||||
(w2, self.w2_scale),
|
||||
mm2_out,
|
||||
expert_ids,
|
||||
grouped_layout,
|
||||
**gemm_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -258,25 +258,14 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _get_permute_scratch(
|
||||
self, topk: int, indices_only: bool = False
|
||||
) -> MoEPermuteScratch | None:
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if not moe_permute_unpermute_supported():
|
||||
return None
|
||||
|
||||
max_expanded_rows = (
|
||||
self.moe_config.max_num_tokens
|
||||
* self.moe_config.dp_size
|
||||
* self.moe_config.experts_per_token
|
||||
)
|
||||
return get_moe_permute_scratch(
|
||||
max_num_tokens=math.ceil(max_expanded_rows / topk),
|
||||
topk=topk,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
hidden_size=None if indices_only else self.moe_config.hidden_dim,
|
||||
hidden_dtype=None if indices_only else self.moe_config.in_dtype,
|
||||
)
|
||||
|
||||
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
|
||||
@@ -911,14 +900,18 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
)
|
||||
buffers["output"] = output
|
||||
|
||||
scratch = self._get_permute_scratch()
|
||||
if a1q_scale is None:
|
||||
scratch = self._get_permute_scratch(topk_ids.size(1), indices_only=True)
|
||||
assert scratch is not None
|
||||
scatter_metadata = moe_prepare_scatter(topk_ids, expert_map, scratch)
|
||||
expert_offsets, scatter_idx = scatter_metadata
|
||||
inv_perm = scatter_idx.flatten()
|
||||
num_valid_tokens = expert_offsets[-1:] if expert_map is not None else None
|
||||
else:
|
||||
# Prequantized inputs skip w13 quantization, so its buffer is free
|
||||
# to hold the permuted hidden states.
|
||||
permuted_hidden_states = buffers["quanted_gate_up_input"]
|
||||
assert permuted_hidden_states.dtype == hidden_states.dtype
|
||||
hidden_states, a1q_scale, expert_offsets, inv_perm, _ = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=a1q_scale,
|
||||
@@ -926,7 +919,8 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
n_expert=global_num_experts,
|
||||
n_local_expert=self.num_experts,
|
||||
expert_map=expert_map,
|
||||
scratch=self._get_permute_scratch(topk_ids.size(1)),
|
||||
permuted_hidden_states=permuted_hidden_states,
|
||||
scratch=scratch,
|
||||
)
|
||||
scatter_idx = None
|
||||
num_valid_tokens = None
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
@@ -41,11 +43,11 @@ def _triton_kernel_moe_supports_current_device() -> bool:
|
||||
p = current_platform
|
||||
if p.is_cuda():
|
||||
cap = p.get_device_capability()
|
||||
# Keep the original `(9, 0) <= cap < (11, 0)` window on
|
||||
# CUDA (covers Hopper SM90 and Blackwell SM100, excludes
|
||||
# SM120) — this PR is ROCm-scoped and the broader CUDA
|
||||
# range was not validated.
|
||||
return cap is not None and (9, 0) <= (cap.major, cap.minor) < (11, 0)
|
||||
# Hopper SM90, datacenter Blackwell SM100/103 and consumer Blackwell
|
||||
# SM120/121. SM12x needs the opt-flag constraints applied by
|
||||
# `_swizzle_mxfp4` and `_constrain_sm12x_block_m` to fit its 99KB of
|
||||
# shared memory; it is still ordered after MARLIN by the selector.
|
||||
return cap is not None and (9, 0) <= (cap.major, cap.minor) < (13, 0)
|
||||
if p.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx9
|
||||
|
||||
@@ -57,6 +59,36 @@ def _triton_kernel_moe_supports_current_device() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _is_sm12x() -> bool:
|
||||
return current_platform.is_cuda() and current_platform.is_device_capability_family(
|
||||
120
|
||||
)
|
||||
|
||||
|
||||
# Consumer Blackwell has 99KB of opt-in shared memory: with block_m >= 64 the
|
||||
# persistent mxfp4 kernel fits a single pipeline stage and runs 2-3x slower
|
||||
# than with 32, but triton_kernels' block_m heuristic still tops out at 128.
|
||||
_SM12X_MAX_BLOCK_M = 32
|
||||
|
||||
|
||||
def _sm12x_block_m(num_rows: int, num_experts: int) -> int:
|
||||
"""triton_kernels' tokens-per-expert block_m heuristic, capped at 32."""
|
||||
tokens_per_expt = max(1, num_rows // num_experts)
|
||||
return max(16, min(triton.next_power_of_2(tokens_per_expt), _SM12X_MAX_BLOCK_M))
|
||||
|
||||
|
||||
def _constrain_sm12x_block_m(num_rows: int, num_experts: int) -> None:
|
||||
"""Pin block_m for the following matmul_ogs calls on SM12x."""
|
||||
if not _is_sm12x():
|
||||
return
|
||||
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
|
||||
|
||||
opt_flags.update_opt_flags_constraints(
|
||||
{"block_m": _sm12x_block_m(num_rows, num_experts)}
|
||||
)
|
||||
|
||||
|
||||
def _patch_make_bitmatrix_metadata() -> None:
|
||||
"""Monkey-patch make_bitmatrix_metadata to support non-power-of-2 top_k.
|
||||
|
||||
@@ -784,6 +816,7 @@ def triton_kernel_fused_experts(
|
||||
)
|
||||
)
|
||||
gammas = routing_data.gate_scal if routing_data else None
|
||||
_constrain_sm12x_block_m(M * topk, E)
|
||||
|
||||
matmul_ogs(
|
||||
hidden_states,
|
||||
@@ -1403,6 +1436,7 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
|
||||
intermediate_cache2 = _resize_cache(workspace13, (M * topk, activation_out_dim))
|
||||
|
||||
gammas = routing_data.gate_scal if routing_data else None
|
||||
_constrain_sm12x_block_m(M * topk, E)
|
||||
|
||||
matmul_ogs(
|
||||
hidden_states,
|
||||
|
||||
@@ -254,6 +254,7 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
RoutingMethodType.SigmoidRenorm,
|
||||
RoutingMethodType.MiniMax2,
|
||||
RoutingMethodType.Sigmoid,
|
||||
]
|
||||
|
||||
|
||||
@@ -175,6 +175,11 @@ class XPUExperts(mk.FusedMoEExpertsModular):
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
assert self.fused_moe_impl is not None
|
||||
if self.moe_config.ep_size > 1:
|
||||
# Avoid topk_ids = -1, remap them to an
|
||||
# expert not owned by this rank so they are skipped.
|
||||
sentinel = global_num_experts - 1 if self.moe_config.ep_rank == 0 else 0
|
||||
topk_ids = torch.where(topk_ids < 0, sentinel, topk_ids)
|
||||
self.fused_moe_impl.apply(
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -118,10 +118,11 @@ def FusedMoEFactory(
|
||||
has_bias: bool = False,
|
||||
is_sequence_parallel: bool = False,
|
||||
reduce_results: bool = True,
|
||||
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
|
||||
ckpt_names: tuple[str, str, str | None] = ("gate_proj", "down_proj", "up_proj"),
|
||||
is_fused_checkpoint_transposed: bool = False,
|
||||
n_shared_experts: int | None = None,
|
||||
fuse_shared_experts: bool = False,
|
||||
shared_expert_prefix: str | None = None,
|
||||
router_logits_dtype: torch.dtype | None = None,
|
||||
gate: torch.nn.Module | None = None,
|
||||
shared_experts: torch.nn.Module | None = None,
|
||||
@@ -197,6 +198,7 @@ def FusedMoEFactory(
|
||||
n_shared_experts: Number of shared experts to fuse into the routed
|
||||
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
|
||||
fuse_shared_experts: Whether to enable shared-expert fusion.
|
||||
shared_expert_prefix: Checkpoint prefix for the fused shared expert.
|
||||
router_logits_dtype: Data type for router logits buffers
|
||||
gate: Pre-configured gate module
|
||||
shared_experts: Pre-configured shared experts module
|
||||
@@ -360,6 +362,7 @@ def FusedMoEFactory(
|
||||
elastic_ep_max_dp_size=vllm_config.parallel_config.elastic_ep_max_dp_size,
|
||||
has_bias=has_bias,
|
||||
is_lora_enabled=vllm_config.lora_config is not None,
|
||||
shared_expert_prefix=shared_expert_prefix,
|
||||
activation=moe_activation,
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=router.routing_method_type, # Not ideal
|
||||
@@ -442,7 +445,7 @@ def fused_moe_make_expert_params_mapping(
|
||||
model: torch.nn.Module,
|
||||
ckpt_gate_proj_name: str,
|
||||
ckpt_down_proj_name: str,
|
||||
ckpt_up_proj_name: str,
|
||||
ckpt_up_proj_name: str | None,
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
|
||||
@@ -197,6 +197,20 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def allocate_fused_expert_output(
|
||||
self,
|
||||
shape: tuple[int, ...],
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
weight_and_reduce_impl: TopKWeightAndReduce,
|
||||
) -> torch.Tensor | None:
|
||||
"""Optionally provide the exact output buffer for the expert kernel.
|
||||
|
||||
Implementations returning a tensor must preserve its identity through
|
||||
finalize; replacing it may silently reintroduce a write-back copy.
|
||||
"""
|
||||
return None
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def activation_format(self) -> FusedMoEActivationFormat:
|
||||
@@ -1054,13 +1068,14 @@ class FusedMoEKernelModularImpl:
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool]:
|
||||
"""Allocate temporary and output buffers for the fused experts op.
|
||||
Inputs:
|
||||
- out_dtype: output type of workspace and output tensors.
|
||||
- device: the device of the workspace and output tensors.
|
||||
See `workspace_shapes` for a description of the remainder of arguments.
|
||||
Returns a tuple of (workspace13, workspace2, output) tensors.
|
||||
Returns the workspace tensors, output tensor, and whether the output
|
||||
uses a prepare/finalize-provided allocation.
|
||||
"""
|
||||
assert M_full > 0 and M_chunk > 0
|
||||
|
||||
@@ -1090,9 +1105,20 @@ class FusedMoEKernelModularImpl:
|
||||
activation,
|
||||
)
|
||||
|
||||
fused_out_alias = self.prepare_finalize.allocate_fused_expert_output(
|
||||
fused_out_shape,
|
||||
workspace_dtype,
|
||||
device,
|
||||
self.fused_experts.finalize_weight_and_reduce_impl(),
|
||||
)
|
||||
|
||||
# We can reuse the memory between cache1 and cache3 because by the
|
||||
# time we need cache3, we're done with cache1.
|
||||
# Reuse workspace13 for the output since there is only one chunk.
|
||||
# Keep the original fused-output reservation even when the output is
|
||||
# externally aliased. The workspace manager profiles and locks a
|
||||
# process-wide high-water mark; later graph shapes may need this
|
||||
# storage for workspace13.
|
||||
max_shape_size = max(prod(workspace13_shape), prod(fused_out_shape))
|
||||
|
||||
if current_platform.is_cpu():
|
||||
@@ -1110,16 +1136,23 @@ class FusedMoEKernelModularImpl:
|
||||
)
|
||||
workspace13 = _resize_cache(common_workspace, workspace13_shape)
|
||||
fused_out = _resize_cache(common_workspace, fused_out_shape)
|
||||
return workspace13, workspace2, fused_out
|
||||
return workspace13, workspace2, fused_out, False
|
||||
|
||||
common_workspace, workspace2 = current_workspace_manager().get_simultaneous(
|
||||
((max_shape_size,), workspace_dtype),
|
||||
(workspace2_shape, workspace_dtype),
|
||||
)
|
||||
workspace13 = _resize_cache(common_workspace, workspace13_shape)
|
||||
fused_out = _resize_cache(common_workspace, fused_out_shape)
|
||||
if fused_out_alias is None:
|
||||
fused_out = _resize_cache(common_workspace, fused_out_shape)
|
||||
else:
|
||||
assert fused_out_alias.shape == fused_out_shape
|
||||
assert fused_out_alias.dtype == workspace_dtype
|
||||
assert fused_out_alias.device == device
|
||||
assert fused_out_alias.is_contiguous()
|
||||
fused_out = fused_out_alias
|
||||
|
||||
return workspace13, workspace2, fused_out
|
||||
return workspace13, workspace2, fused_out, fused_out_alias is not None
|
||||
|
||||
def _maybe_apply_shared_experts(
|
||||
self,
|
||||
@@ -1251,22 +1284,25 @@ class FusedMoEKernelModularImpl:
|
||||
if M_full == 0:
|
||||
return torch.empty_like(a1q, dtype=in_dtype)
|
||||
|
||||
workspace13, workspace2, fused_out = self._allocate_buffers(
|
||||
in_dtype,
|
||||
a1q.device,
|
||||
M_full,
|
||||
M_full,
|
||||
N,
|
||||
K,
|
||||
top_k,
|
||||
global_num_experts,
|
||||
local_num_experts,
|
||||
expert_tokens_meta,
|
||||
activation,
|
||||
workspace13, workspace2, fused_out, fused_out_is_specialized = (
|
||||
self._allocate_buffers(
|
||||
in_dtype,
|
||||
a1q.device,
|
||||
M_full,
|
||||
M_full,
|
||||
N,
|
||||
K,
|
||||
top_k,
|
||||
global_num_experts,
|
||||
local_num_experts,
|
||||
expert_tokens_meta,
|
||||
activation,
|
||||
)
|
||||
)
|
||||
|
||||
use_output_alias = (
|
||||
output_alias is not None
|
||||
not fused_out_is_specialized
|
||||
and output_alias is not None
|
||||
and output_alias.shape == fused_out.shape
|
||||
and output_alias.dtype == fused_out.dtype
|
||||
and output_alias.device == fused_out.device
|
||||
|
||||
@@ -13,90 +13,70 @@ from vllm.v1.worker.workspace import (
|
||||
|
||||
@dataclass
|
||||
class MoEPermuteScratch:
|
||||
# Reused metadata buffers for repeated grouped-MoE permutes.
|
||||
max_num_tokens: int
|
||||
topk: int
|
||||
"""Reused metadata buffers for repeated grouped-MoE permutes.
|
||||
|
||||
Capacity follows the largest input seen, like the shared workspace: it grows
|
||||
during warmup and is fixed once the workspace manager is locked.
|
||||
"""
|
||||
|
||||
num_experts: int
|
||||
num_local_experts: int
|
||||
device: torch.device
|
||||
hidden_size: int | None = None
|
||||
hidden_dtype: torch.dtype | None = None
|
||||
max_expanded_rows: int = field(init=False, default=0)
|
||||
token_expert_indices: torch.Tensor = field(init=False)
|
||||
expert_first_token_offset: torch.Tensor = field(init=False)
|
||||
permuted_idx: torch.Tensor = field(init=False)
|
||||
inv_permuted_idx: torch.Tensor = field(init=False)
|
||||
permuted_hidden_states: torch.Tensor | None = field(init=False, default=None)
|
||||
sort_workspace: torch.Tensor = field(init=False)
|
||||
permuted_experts_id: torch.Tensor = field(init=False)
|
||||
sorted_row_idx: torch.Tensor = field(init=False)
|
||||
topk_ids_int32: torch.Tensor = field(init=False)
|
||||
topk_ids_for_sort: torch.Tensor = field(init=False)
|
||||
max_expanded_rows: int = field(init=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
assert self.max_num_tokens > 0
|
||||
assert self.topk > 0
|
||||
assert self.num_experts > 0
|
||||
assert self.num_local_experts > 0
|
||||
if self.hidden_size is None:
|
||||
assert self.hidden_dtype is None
|
||||
else:
|
||||
assert self.hidden_dtype is not None
|
||||
|
||||
self.max_expanded_rows = self.max_num_tokens * self.topk
|
||||
self.token_expert_indices = torch.arange(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.expert_first_token_offset = torch.empty(
|
||||
self.num_local_experts + 1, dtype=torch.int64, device=self.device
|
||||
)
|
||||
self.permuted_idx = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.inv_permuted_idx = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
if self.hidden_size is not None:
|
||||
hidden_numel = self.max_expanded_rows * self.hidden_size
|
||||
self.permuted_hidden_states = torch.empty(
|
||||
hidden_numel, dtype=self.hidden_dtype, device=self.device
|
||||
# torch.device("cuda") in config, after initialized,
|
||||
# will be changed to cuda:{index}, so we need to refresh here.
|
||||
self.device = self.expert_first_token_offset.device
|
||||
|
||||
def reserve(self, expanded_rows: int) -> None:
|
||||
"""Ensure capacity for ``expanded_rows`` (num_tokens * topk) rows."""
|
||||
if expanded_rows <= self.max_expanded_rows:
|
||||
return
|
||||
if (
|
||||
is_workspace_manager_initialized()
|
||||
and current_workspace_manager().is_locked()
|
||||
):
|
||||
raise AssertionError(
|
||||
f"Workspace is locked but MoE permute scratch requires "
|
||||
f"{expanded_rows} rows, current capacity is "
|
||||
f"{self.max_expanded_rows} rows. Scratch growth is not allowed "
|
||||
"after locking."
|
||||
)
|
||||
self.permuted_experts_id = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.sorted_row_idx = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.topk_ids_int32 = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.topk_ids_for_sort = torch.empty(
|
||||
self.max_expanded_rows, dtype=torch.int32, device=self.device
|
||||
|
||||
def empty_rows() -> torch.Tensor:
|
||||
return torch.empty(expanded_rows, dtype=torch.int32, device=self.device)
|
||||
|
||||
self.token_expert_indices = torch.arange(
|
||||
expanded_rows, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.permuted_idx = empty_rows()
|
||||
self.inv_permuted_idx = empty_rows()
|
||||
self.permuted_experts_id = empty_rows()
|
||||
self.sorted_row_idx = empty_rows()
|
||||
self.topk_ids_int32 = empty_rows()
|
||||
self.topk_ids_for_sort = empty_rows()
|
||||
sorter_size = torch.ops._moe_C.moe_permute_sort_workspace_size(
|
||||
self.max_expanded_rows, self.num_experts
|
||||
expanded_rows, self.num_experts
|
||||
)
|
||||
self.sort_workspace = torch.empty(
|
||||
sorter_size, dtype=torch.int8, device=self.device
|
||||
)
|
||||
# torch.device("cuda") in config, after initialized,
|
||||
# will be changed to cuda:{index}, so we need to refresh here.
|
||||
self.device = self.token_expert_indices.device
|
||||
|
||||
def validate(self, hidden_states: torch.Tensor, topk_ids: torch.Tensor) -> None:
|
||||
n_token, n_hidden = hidden_states.shape
|
||||
assert hidden_states.device == self.device
|
||||
assert topk_ids.device == self.device
|
||||
assert n_token <= self.max_num_tokens
|
||||
assert topk_ids.size(1) == self.topk
|
||||
assert topk_ids.size(0) == n_token
|
||||
if self.hidden_size is not None:
|
||||
assert n_hidden == self.hidden_size
|
||||
assert hidden_states.dtype == self.hidden_dtype
|
||||
assert self.permuted_hidden_states is not None
|
||||
|
||||
def token_expert_indices_view(self, n_token: int) -> torch.Tensor:
|
||||
return self.token_expert_indices[: n_token * self.topk].view(n_token, self.topk)
|
||||
self.max_expanded_rows = expanded_rows
|
||||
|
||||
def prepare_topk_ids(self, topk_ids: torch.Tensor) -> torch.Tensor:
|
||||
if topk_ids.dtype == torch.int32:
|
||||
@@ -114,17 +94,16 @@ def moe_prepare_scatter(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Generate expert offsets and shared scatter/unpermute destination indices."""
|
||||
assert topk_ids.device == scratch.device
|
||||
assert topk_ids.size(0) <= scratch.max_num_tokens
|
||||
assert topk_ids.size(1) == scratch.topk
|
||||
n_token, topk = topk_ids.shape
|
||||
expanded_rows = topk_ids.numel()
|
||||
scratch.reserve(expanded_rows)
|
||||
inverse = scratch.inv_permuted_idx[:expanded_rows].view(n_token, topk)
|
||||
if expanded_rows == 0:
|
||||
scratch.expert_first_token_offset.zero_()
|
||||
return scratch.expert_first_token_offset, inverse
|
||||
torch.ops._moe_C.moe_prepare_scatter(
|
||||
scratch.prepare_topk_ids(topk_ids),
|
||||
scratch.token_expert_indices_view(n_token),
|
||||
scratch.token_expert_indices[:expanded_rows].view(n_token, topk),
|
||||
expert_map,
|
||||
scratch.num_experts,
|
||||
scratch.num_local_experts,
|
||||
@@ -140,13 +119,9 @@ def moe_prepare_scatter(
|
||||
|
||||
def get_moe_permute_scratch(
|
||||
*,
|
||||
max_num_tokens: int,
|
||||
topk: int,
|
||||
num_experts: int,
|
||||
num_local_experts: int,
|
||||
device: torch.device,
|
||||
hidden_size: int | None = None,
|
||||
hidden_dtype: torch.dtype | None = None,
|
||||
) -> MoEPermuteScratch:
|
||||
"""Share scratch across sequential layers in the current ubatch and lane.
|
||||
|
||||
@@ -155,28 +130,15 @@ def get_moe_permute_scratch(
|
||||
|
||||
def create_scratch() -> MoEPermuteScratch:
|
||||
return MoEPermuteScratch(
|
||||
max_num_tokens=max_num_tokens,
|
||||
topk=topk,
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
device=device,
|
||||
hidden_size=hidden_size,
|
||||
hidden_dtype=hidden_dtype,
|
||||
)
|
||||
|
||||
if not is_workspace_manager_initialized():
|
||||
return create_scratch()
|
||||
|
||||
key = (
|
||||
MoEPermuteScratch,
|
||||
max_num_tokens,
|
||||
topk,
|
||||
num_experts,
|
||||
num_local_experts,
|
||||
device,
|
||||
hidden_size,
|
||||
hidden_dtype,
|
||||
)
|
||||
key = (MoEPermuteScratch, num_experts, num_local_experts, device)
|
||||
return current_workspace_manager().get_persistent_resource(key, create_scratch)
|
||||
|
||||
|
||||
@@ -204,9 +166,8 @@ def moe_permute(
|
||||
parallel shard.
|
||||
permuted_hidden_states (Optional[torch.Tensor]): Optional output tensor.
|
||||
If None, the output tensor will be created in this function.
|
||||
scratch (Optional[MoEPermuteScratch]): Optional preallocated scratch
|
||||
buffers. Validated against hidden_states and topk_ids when given,
|
||||
otherwise the buffers are allocated in this function.
|
||||
scratch (Optional[MoEPermuteScratch]): Optional reusable metadata
|
||||
buffers. Otherwise they are allocated in this function.
|
||||
|
||||
Returns:
|
||||
- permuted_hidden_states (torch.Tensor): permuted activation.
|
||||
@@ -227,20 +188,11 @@ def moe_permute(
|
||||
if n_local_expert == -1:
|
||||
n_local_expert = n_expert
|
||||
if permuted_hidden_states is None:
|
||||
if scratch is None:
|
||||
permuted_hidden_states = torch.empty(
|
||||
(permuted_row_size, n_hidden),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
else:
|
||||
scratch.validate(hidden_states, topk_ids)
|
||||
hidden_numel = permuted_row_size * n_hidden
|
||||
scratch_hidden_states = scratch.permuted_hidden_states
|
||||
assert scratch_hidden_states is not None
|
||||
permuted_hidden_states = scratch_hidden_states[:hidden_numel].view(
|
||||
permuted_row_size, n_hidden
|
||||
)
|
||||
permuted_hidden_states = torch.empty(
|
||||
(permuted_row_size, n_hidden),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
assert permuted_hidden_states.size() == (permuted_row_size, n_hidden), (
|
||||
f"Expected permuted hidden states to be {(permuted_row_size, n_hidden)}"
|
||||
f" but got {permuted_hidden_states.size()}"
|
||||
@@ -278,10 +230,15 @@ def moe_permute(
|
||||
permuted_idx,
|
||||
)
|
||||
else:
|
||||
scratch.validate(hidden_states, topk_ids)
|
||||
assert hidden_states.device == scratch.device
|
||||
assert topk_ids.device == scratch.device
|
||||
assert topk_ids.size(0) == n_token
|
||||
assert n_expert == scratch.num_experts
|
||||
assert n_local_expert == scratch.num_local_experts
|
||||
token_expert_indices = scratch.token_expert_indices_view(n_token)
|
||||
scratch.reserve(permuted_row_size)
|
||||
token_expert_indices = scratch.token_expert_indices[:permuted_row_size].view(
|
||||
n_token, topk
|
||||
)
|
||||
expert_first_token_offset = scratch.expert_first_token_offset
|
||||
permuted_idx = scratch.permuted_idx[:permuted_row_size]
|
||||
permuted_idx.fill_(permuted_row_size)
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
prepare_fp8_moe_layer_for_deepgemm,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_fp8_moe_layer_for_marlin,
|
||||
)
|
||||
@@ -37,6 +38,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Static128BlockSym,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
@@ -98,6 +100,7 @@ def _get_priority_backends(
|
||||
Fp8MoeBackend.CPU,
|
||||
Fp8MoeBackend.HPC,
|
||||
]
|
||||
_AVAILABLE_BACKENDS = prioritize_humming(_AVAILABLE_BACKENDS)
|
||||
|
||||
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
|
||||
backends.insert(0, backends.pop(backends.index(backend)))
|
||||
@@ -607,6 +610,21 @@ def convert_to_fp8_moe_kernel_format(
|
||||
convert_to_humming_moe_kernel_format,
|
||||
)
|
||||
|
||||
# Online quantization has not installed the quantized tensors yet.
|
||||
scale_name = (
|
||||
"weight_scale_inv"
|
||||
if hasattr(layer, "w13_weight_scale_inv")
|
||||
else "weight_scale"
|
||||
)
|
||||
for prefix, weight, scale, input_scale in (
|
||||
("w13", w13, w13_scale, w13_input_scale),
|
||||
("w2", w2, w2_scale, w2_input_scale),
|
||||
):
|
||||
replace_parameter(layer, f"{prefix}_weight", weight)
|
||||
replace_parameter(layer, f"{prefix}_{scale_name}", scale)
|
||||
if input_scale is not None:
|
||||
replace_parameter(layer, f"{prefix}_input_scale", input_scale)
|
||||
|
||||
convert_to_humming_moe_kernel_format(
|
||||
layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale)
|
||||
)
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
|
||||
TrtLlmMxint4ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_config,
|
||||
marlin_act_int8_process_scales,
|
||||
@@ -129,7 +130,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
|
||||
if current_platform.is_xpu():
|
||||
return [WNA16MoEBackend.XPU]
|
||||
|
||||
return [
|
||||
backends = [
|
||||
# Native HIP kernel, gated on gfx1100 by _supports_current_device().
|
||||
WNA16MoEBackend.RDNA3,
|
||||
WNA16MoEBackend.FLASHINFER_TRTLLM,
|
||||
@@ -139,6 +140,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
|
||||
WNA16MoEBackend.HUMMING,
|
||||
WNA16MoEBackend.EMULATION,
|
||||
]
|
||||
return prioritize_humming(backends)
|
||||
|
||||
|
||||
def _backend_incompatibility_reason(
|
||||
@@ -924,15 +926,18 @@ def _process_weights_rdna3(
|
||||
]:
|
||||
"""RDNA3 (gfx1100) W4A16 weight post-processing.
|
||||
|
||||
Interleaves the packed nibbles per expert (the exllama shuffle the dense
|
||||
RDNA3 kernel also uses) and synthesizes the symmetric zero points that
|
||||
``moe_gptq_gemm_rdna3`` dequantizes with. The packed layout
|
||||
``[E, K // 8, N]`` and the ``[E, groups, N]`` scales are already what the
|
||||
kernel wants, so neither is repacked.
|
||||
Transposes the canonical N-first inputs to the K-first layout of
|
||||
``moe_gptq_gemm_rdna3`` (weights ``[E, K // 8, N]``, scales
|
||||
``[E, groups, N]``), interleaves the packed nibbles per expert (the exllama
|
||||
shuffle the dense RDNA3 kernel also uses) and synthesizes the symmetric
|
||||
zero points the kernel dequantizes with.
|
||||
"""
|
||||
device = w13.device
|
||||
num_experts = w13.size(0)
|
||||
|
||||
w13 = w13.transpose(1, 2).contiguous()
|
||||
w2 = w2.transpose(1, 2).contiguous()
|
||||
|
||||
for e in range(num_experts):
|
||||
w13_e = w13[e].contiguous()
|
||||
ops.gptq_shuffle(w13_e, 4)
|
||||
@@ -948,8 +953,8 @@ def _process_weights_rdna3(
|
||||
return (
|
||||
w13,
|
||||
w2,
|
||||
w13_scale.contiguous(),
|
||||
w2_scale.contiguous(),
|
||||
w13_scale.transpose(1, 2).contiguous(),
|
||||
w2_scale.transpose(1, 2).contiguous(),
|
||||
_qzeros(w13),
|
||||
_qzeros(w2),
|
||||
None, # w13_input_global_scale
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
swap_w13_to_w31,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
_swizzle_mxfp4,
|
||||
)
|
||||
@@ -113,6 +114,18 @@ def _pack_deepgemm_mxfp4_scales(
|
||||
)
|
||||
|
||||
|
||||
def _swizzle_cutlass_mxfp4_scales(scale: torch.Tensor) -> torch.Tensor:
|
||||
"""Swizzle [E, N, K // 32] block scales into the CUTLASS tiled layout."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
swizzle_mxfp4_scales,
|
||||
)
|
||||
|
||||
n, scale_k = scale.shape[1:]
|
||||
return torch.stack(
|
||||
[swizzle_mxfp4_scales(s, n, scale_k * 32).reshape(n, scale_k) for s in scale]
|
||||
)
|
||||
|
||||
|
||||
class Mxfp4MoeBackend(Enum):
|
||||
NONE = "None"
|
||||
# b12x backends
|
||||
@@ -126,6 +139,8 @@ class Mxfp4MoeBackend(Enum):
|
||||
# FlashInfer CUTLASS backends
|
||||
FLASHINFER_CUTLASS_MXFP4_MXFP8 = "FLASHINFER_CUTLASS_MXFP4_MXFP8"
|
||||
FLASHINFER_CUTLASS_MXFP4_BF16 = "FLASHINFER_CUTLASS_MXFP4_BF16"
|
||||
# vLLM CUTLASS backend
|
||||
CUTLASS_MXFP4_MXFP4 = "CUTLASS_MXFP4_MXFP4"
|
||||
# Marlin
|
||||
BATCHED_MARLIN = "BATCHED_MARLIN"
|
||||
MARLIN = "MARLIN"
|
||||
@@ -217,6 +232,13 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [FlashInferExperts]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsMxfp4,
|
||||
)
|
||||
|
||||
return [CutlassExpertsMxfp4]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.TRITON:
|
||||
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
|
||||
OAITritonExperts,
|
||||
@@ -377,21 +399,32 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
Mxfp4MoeBackend.TRITON,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
|
||||
# TRITON_UNFUSED has bug with MTP support
|
||||
# TODO re-enable after kernel is fixed
|
||||
# TRITON_UNFUSED
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN,
|
||||
Mxfp4MoeBackend.HUMMING,
|
||||
Mxfp4MoeBackend.XPU,
|
||||
Mxfp4MoeBackend.CPU,
|
||||
Mxfp4MoeBackend.EMULATION,
|
||||
]
|
||||
return _AVAILABLE_BACKENDS
|
||||
if current_platform.is_cuda() and current_platform.is_device_capability_family(120):
|
||||
# Consumer Blackwell: the OAI Triton kernels run but their 99KB shared
|
||||
# memory budget caps them at small tiles, so MARLIN prefills ~10%
|
||||
# faster at equal decode throughput. Keep TRITON as an opt-in.
|
||||
_AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.TRITON)
|
||||
_AVAILABLE_BACKENDS.insert(
|
||||
_AVAILABLE_BACKENDS.index(Mxfp4MoeBackend.BATCHED_MARLIN) + 1,
|
||||
Mxfp4MoeBackend.TRITON,
|
||||
)
|
||||
return prioritize_humming(_AVAILABLE_BACKENDS)
|
||||
|
||||
|
||||
def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
"""Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
|
||||
TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the
|
||||
TRTLLM MXFP8; SM90 falls through to Humming or Marlin (the
|
||||
backend-level ``is_supported_config`` check filters by device capability).
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
@@ -411,8 +444,9 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
# TRITON_UNFUSED
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN,
|
||||
Mxfp4MoeBackend.HUMMING,
|
||||
]
|
||||
return _AVAILABLE_BACKENDS
|
||||
return prioritize_humming(_AVAILABLE_BACKENDS)
|
||||
|
||||
|
||||
def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
|
||||
@@ -428,7 +462,10 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
|
||||
return kMxfp8Dynamic
|
||||
if backend == Mxfp4MoeBackend.AITER_MXFP4_FP8:
|
||||
return kFp8StaticTensorSym
|
||||
if backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
|
||||
if backend in (
|
||||
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
|
||||
):
|
||||
return kMxfp4Dynamic
|
||||
return None # BF16 activation
|
||||
|
||||
@@ -462,6 +499,22 @@ def _make_log_backend(backend: Mxfp4MoeBackend) -> str:
|
||||
return f"Using '{backend.value}' Mxfp4 MoE backend."
|
||||
|
||||
|
||||
def _warn_on_bf16_fallback(
|
||||
backend: Mxfp4MoeBackend, requested_activation_key: QuantKey | None
|
||||
) -> None:
|
||||
if (
|
||||
requested_activation_key is not None
|
||||
and backend != Mxfp4MoeBackend.EMULATION
|
||||
and _backend_activation_key(backend) is None
|
||||
):
|
||||
logger.warning_once(
|
||||
"No Mxfp4 MoE backend supports activation=%s for the deployment "
|
||||
"configuration. Falling back to '%s' with BF16 activations.",
|
||||
requested_activation_key,
|
||||
backend.value,
|
||||
)
|
||||
|
||||
|
||||
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
base = (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
@@ -493,17 +546,19 @@ def _filter_by_activation(
|
||||
backends: list[Mxfp4MoeBackend],
|
||||
requested_activation_key: QuantKey | None,
|
||||
) -> list[Mxfp4MoeBackend]:
|
||||
"""Pick variants matching ``requested_activation_key``; without one,
|
||||
prefer BF16 if the list has any, else keep the list as-is so explicit
|
||||
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
|
||||
"""Pick variants matching ``requested_activation_key``, followed by BF16
|
||||
variants as a higher-precision fallback; without one, prefer BF16 if the
|
||||
list has any, else keep the list as-is so explicit non-BF16 picks (e.g.
|
||||
the ``_afp8`` aliases) still land."""
|
||||
bf16 = [b for b in backends if _backend_activation_key(b) is None]
|
||||
if requested_activation_key is not None:
|
||||
return [
|
||||
matching = [
|
||||
b
|
||||
for b in backends
|
||||
if _backend_activation_key(b) == requested_activation_key
|
||||
or b == Mxfp4MoeBackend.EMULATION
|
||||
]
|
||||
bf16 = [b for b in backends if _backend_activation_key(b) is None]
|
||||
return matching + [b for b in bf16 if b not in matching]
|
||||
return bf16 if bf16 else backends
|
||||
|
||||
|
||||
@@ -564,8 +619,9 @@ def select_mxfp4_moe_backend(
|
||||
|
||||
Args:
|
||||
config: MoE configuration
|
||||
activation_key: Optional activation quantization key. If provided,
|
||||
overrides the default activation key for backend selection.
|
||||
activation_key: Optional activation quantization key declared by
|
||||
the checkpoint. Backends that match it are preferred, with BF16
|
||||
activation backends as a fallback.
|
||||
Use kFp8StaticTensorSym for W4A8 scheme.
|
||||
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
@@ -602,7 +658,7 @@ def select_mxfp4_moe_backend(
|
||||
else _backend_activation_key(requested_backend)
|
||||
)
|
||||
try:
|
||||
return _return_or_raise(
|
||||
selected = _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
@@ -611,6 +667,9 @@ def select_mxfp4_moe_backend(
|
||||
)
|
||||
except ValueError as e:
|
||||
last_error = e
|
||||
else:
|
||||
_warn_on_bf16_fallback(requested_backend, requested_activation_key)
|
||||
return selected
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
@@ -636,10 +695,10 @@ def select_mxfp4_moe_backend(
|
||||
|
||||
unsupported_reasons = []
|
||||
for backend in AVAILABLE_BACKENDS:
|
||||
# Use requested_activation_key if provided, otherwise use backend default
|
||||
# EMULATION takes the requested key, other backends use their own
|
||||
act_key = (
|
||||
requested_activation_key
|
||||
if requested_activation_key is not None
|
||||
if backend == Mxfp4MoeBackend.EMULATION
|
||||
else _backend_activation_key(backend)
|
||||
)
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
@@ -648,6 +707,7 @@ def select_mxfp4_moe_backend(
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
_warn_on_bf16_fallback(backend, requested_activation_key)
|
||||
return backend, k_cls
|
||||
else:
|
||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
||||
@@ -777,6 +837,10 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
# DeepGEMM requires M/N/K alignment
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
hidden_size = round_up(hidden_size, 128)
|
||||
elif backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
|
||||
# CUTLASS scale tiles cover 128 rows by 128 elements
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
hidden_size = round_up(hidden_size, 128)
|
||||
elif backend in (Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN):
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
if current_platform.is_xpu():
|
||||
@@ -1421,7 +1485,8 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
]:
|
||||
"""Convert loaded weights into backend-specific kernel format.
|
||||
|
||||
Supports DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin backends.
|
||||
Supports CUTLASS, DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin
|
||||
backends.
|
||||
"""
|
||||
is_gfx1250 = False
|
||||
if current_platform.is_rocm():
|
||||
@@ -1439,6 +1504,16 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4:
|
||||
return (
|
||||
w13_weight.data,
|
||||
w2_weight.data,
|
||||
_swizzle_cutlass_mxfp4_scales(w13_weight_scale.data),
|
||||
_swizzle_cutlass_mxfp4_scales(w2_weight_scale.data),
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
|
||||
w13_weight,
|
||||
@@ -1993,7 +2068,10 @@ def make_mxfp4_moe_quant_config(
|
||||
block_shape=None,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
|
||||
elif mxfp4_backend in (
|
||||
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
|
||||
):
|
||||
return ocp_mx_moe_quant_config(
|
||||
quant_dtype="mxfp4",
|
||||
w1_bias=w1_bias,
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
@@ -129,7 +130,8 @@ def select_mxfp8_moe_backend(
|
||||
return backend, _select_kernel_cls(backend, config)
|
||||
|
||||
# Auto-select: pick the first supported backend.
|
||||
for backend in _SUPPORTED_BACKENDS:
|
||||
backends = prioritize_humming(list(_SUPPORTED_BACKENDS))
|
||||
for backend in backends:
|
||||
try:
|
||||
experts_cls = _select_kernel_cls(backend, config)
|
||||
except ValueError:
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_nvfp4_moe_layer_for_marlin,
|
||||
)
|
||||
@@ -209,6 +210,7 @@ def select_nvfp4_moe_backend(
|
||||
NvFp4MoeBackend.HUMMING,
|
||||
NvFp4MoeBackend.EMULATION,
|
||||
]
|
||||
AVAILABLE_BACKENDS = prioritize_humming(AVAILABLE_BACKENDS)
|
||||
|
||||
NVFP4_BACKENDS_WITH_CLAMP = {
|
||||
NvFp4MoeBackend.B12X,
|
||||
@@ -332,6 +334,7 @@ def convert_to_nvfp4_moe_kernel_format(
|
||||
a2_scale: torch.Tensor | None,
|
||||
is_act_and_mul: bool,
|
||||
use_a16: bool = False,
|
||||
trtllm_hidden_alignment: int = 256,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
@@ -417,6 +420,7 @@ def convert_to_nvfp4_moe_kernel_format(
|
||||
w2_scale_2=w2_scale_2,
|
||||
a2_scale=a2_scale,
|
||||
is_act_and_mul=is_act_and_mul,
|
||||
trtllm_hidden_alignment=trtllm_hidden_alignment,
|
||||
)
|
||||
elif nvfp4_backend == NvFp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceContiguous,
|
||||
TopKWeightAndReduceDelegate,
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
|
||||
@@ -108,6 +109,30 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
def output_is_reduced(self) -> bool:
|
||||
return False
|
||||
|
||||
def _allocate_combine_input(
|
||||
self,
|
||||
shape: tuple[int, ...],
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
return get_ep_group().allocate_combine_input(
|
||||
shape,
|
||||
dtype,
|
||||
device,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
)
|
||||
|
||||
def allocate_fused_expert_output(
|
||||
self,
|
||||
shape: tuple[int, ...],
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce,
|
||||
) -> torch.Tensor | None:
|
||||
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP):
|
||||
return self._allocate_combine_input(shape, dtype, device)
|
||||
return None
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
@@ -194,16 +219,29 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceDelegate):
|
||||
weight_and_reduce_impl = TopKWeightAndReduceContiguous()
|
||||
|
||||
if isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP):
|
||||
# allocate_fused_expert_output already selected the combine input.
|
||||
# Reuse it directly; the symmetric allocator may return a new view
|
||||
# object for the same high-water backing allocation.
|
||||
combine_input = fused_expert_output
|
||||
else:
|
||||
combine_input = self._allocate_combine_input(
|
||||
(topk_ids.shape[0], fused_expert_output.shape[-1]),
|
||||
fused_expert_output.dtype,
|
||||
fused_expert_output.device,
|
||||
)
|
||||
out = weight_and_reduce_impl.apply(
|
||||
output=None,
|
||||
output=combine_input,
|
||||
fused_expert_output=fused_expert_output,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
output.copy_(
|
||||
get_ep_group().combine(out, is_sequence_parallel=self.is_sequence_parallel)
|
||||
get_ep_group().combine_into_output(
|
||||
out,
|
||||
output,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
@@ -87,7 +88,7 @@ class RoutedExperts(PluggableLayer):
|
||||
expert_map_manager: ExpertMapManager,
|
||||
ckpt_gate_proj_name: str = "gate_proj",
|
||||
ckpt_down_proj_name: str = "down_proj",
|
||||
ckpt_up_proj_name: str = "up_proj",
|
||||
ckpt_up_proj_name: str | None = "up_proj",
|
||||
is_fused_checkpoint_transposed: bool = False,
|
||||
#
|
||||
# Extra params that are needed by quant_methods, pass along for now
|
||||
@@ -114,6 +115,13 @@ class RoutedExperts(PluggableLayer):
|
||||
self.ckpt_gate_proj_name = ckpt_gate_proj_name
|
||||
self.ckpt_down_proj_name = ckpt_down_proj_name
|
||||
self.ckpt_up_proj_name = ckpt_up_proj_name
|
||||
if moe_config.is_act_and_mul == (ckpt_up_proj_name is None):
|
||||
raise ValueError(
|
||||
f"{layer_name}: ckpt_up_proj_name={ckpt_up_proj_name!r} is "
|
||||
f"inconsistent with activation {moe_config.activation.value!r}. "
|
||||
"Gated MoE needs an up projection name; non-gated MoE must "
|
||||
"pass None."
|
||||
)
|
||||
self.is_fused_checkpoint_transposed = is_fused_checkpoint_transposed
|
||||
self.expert_map_manager = expert_map_manager
|
||||
self.hidden_size = moe_config.hidden_dim
|
||||
@@ -146,6 +154,7 @@ class RoutedExperts(PluggableLayer):
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
# End random parameters
|
||||
self._loaded_expert_biases: set[str] = set()
|
||||
self._fused_shared_expert_quantizer = self._get_fused_shared_expert_quantizer()
|
||||
|
||||
self.quant_method = self._get_quant_method(
|
||||
self.layer_name,
|
||||
@@ -623,6 +632,31 @@ class RoutedExperts(PluggableLayer):
|
||||
# _to_scalar's reshape(()) would reject the size-2 weight_shape.
|
||||
param_data[expert_id] = loaded_weight
|
||||
|
||||
def _get_fused_shared_expert_quantizer(
|
||||
self,
|
||||
) -> Callable[[torch.Tensor], tuple[torch.Tensor, torch.Tensor]] | None:
|
||||
"""Return the quantizer for an online-quantized fused shared expert."""
|
||||
online_config = getattr(self.quant_config, "online_quantization_config", None)
|
||||
prefix = self.moe_config.shared_expert_prefix
|
||||
if (
|
||||
online_config is None
|
||||
or prefix is None
|
||||
or self.expert_map_manager.num_fused_shared_experts == 0
|
||||
):
|
||||
return None
|
||||
# FSE compatibility checks require every shared-expert projection to
|
||||
# use the routed experts' weight key.
|
||||
resolved = online_config.resolve_quant_method_cls(
|
||||
LinearBase, f"{prefix}.down_proj"
|
||||
)
|
||||
if resolved is None:
|
||||
return None
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
ONLINE_SHARED_EXPERT_QUANTIZERS,
|
||||
)
|
||||
|
||||
return ONLINE_SHARED_EXPERT_QUANTIZERS[resolved[3].weight]
|
||||
|
||||
@overload
|
||||
def weight_loader(
|
||||
self,
|
||||
@@ -680,6 +714,35 @@ class RoutedExperts(PluggableLayer):
|
||||
return False if return_success else None
|
||||
# Hereafter, `expert_id` is local physical id
|
||||
|
||||
# A full-precision fused shared expert is quantized into the checkpoint
|
||||
# layout, then loaded like a pre-quantized expert weight and scale.
|
||||
if (
|
||||
self._fused_shared_expert_quantizer is not None
|
||||
and self.moe_config.num_logical_experts
|
||||
<= global_expert_id
|
||||
< self.moe_config.num_logical_experts
|
||||
+ self.expert_map_manager.num_fused_shared_experts
|
||||
and weight_name.endswith("_weight")
|
||||
and loaded_weight.is_floating_point()
|
||||
and loaded_weight.dtype != param.dtype
|
||||
):
|
||||
weight, weight_scale = self._fused_shared_expert_quantizer(
|
||||
loaded_weight.to(self.moe_config.device)
|
||||
)
|
||||
stem = "w2" if shard_id == "w2" else "w13"
|
||||
self.weight_loader(
|
||||
getattr(self, f"{stem}_weight_scale"),
|
||||
weight_scale,
|
||||
f"{weight_name}_scale",
|
||||
shard_id,
|
||||
global_expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
loaded = self.weight_loader(
|
||||
param, weight, weight_name, shard_id, global_expert_id, True
|
||||
)
|
||||
return loaded if return_success else None
|
||||
|
||||
# is_transposed: if the dim to shard the weight
|
||||
# should be flipped. Required by GPTQ/AWQ (K-first format).
|
||||
# should be whatever dimension intermediate_size_per_partition is
|
||||
@@ -971,7 +1034,7 @@ class RoutedExperts(PluggableLayer):
|
||||
loaded_weight,
|
||||
self.is_fused_checkpoint_transposed and uses_weight_layout,
|
||||
)
|
||||
if shard_id in {"w1", "w3"}:
|
||||
if shard_id in {"w1", "w3"} and self.moe_config.is_act_and_mul:
|
||||
# Repurpose expert_id for deconcatenating w1 and w3
|
||||
experts_shard = fused_weight.chunk(2, dim=1)[expert_id]
|
||||
else:
|
||||
@@ -1029,6 +1092,7 @@ class RoutedExperts(PluggableLayer):
|
||||
routed_experts_prefix="",
|
||||
lora_base_layer_prefix=self.lora_base_layer_prefix,
|
||||
include_fused=include_fused,
|
||||
is_gated=moe_config.is_act_and_mul,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1036,7 +1100,7 @@ class RoutedExperts(PluggableLayer):
|
||||
model: torch.nn.Module,
|
||||
ckpt_gate_proj_name: str,
|
||||
ckpt_down_proj_name: str,
|
||||
ckpt_up_proj_name: str,
|
||||
ckpt_up_proj_name: str | None,
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
@@ -1067,13 +1131,14 @@ class RoutedExperts(PluggableLayer):
|
||||
def build_expert_params_mapping(
|
||||
ckpt_gate_proj_name: str,
|
||||
ckpt_down_proj_name: str,
|
||||
ckpt_up_proj_name: str,
|
||||
ckpt_up_proj_name: str | None,
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
lora_base_layer_prefix: str = "",
|
||||
lora_base_layer_prefix_on_param_name: str = "",
|
||||
include_fused: bool = False,
|
||||
is_gated: bool = True,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
"""Create expert parameter mapping for weight loading with redundant experts.
|
||||
|
||||
@@ -1095,6 +1160,8 @@ class RoutedExperts(PluggableLayer):
|
||||
``make_expert_params_mapping`` indexes the model-wide
|
||||
``params_dict`` (prefix included).
|
||||
include_fused: Prepend the fused pre-fused-checkpoint entries
|
||||
is_gated: Whether w13 holds gate and up (gated activation) or a
|
||||
single up projection. Selects the fused checkpoint layout.
|
||||
routed_experts_prefix: Prefix of the routed experts submodule
|
||||
|
||||
Returns:
|
||||
@@ -1129,7 +1196,13 @@ class RoutedExperts(PluggableLayer):
|
||||
fused_mapping = []
|
||||
if include_fused:
|
||||
gate_up = None
|
||||
if ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
|
||||
w13_shards: tuple[str, ...] = ("w1", "w3")
|
||||
if not is_gated:
|
||||
# Non-gated: the stacked checkpoint tensor is the up projection
|
||||
# itself, nothing to split into gate and up.
|
||||
gate_up = ckpt_gate_proj_name
|
||||
w13_shards = ("w1",)
|
||||
elif ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
|
||||
gate_up = "gate_up_proj"
|
||||
elif ckpt_gate_proj_name == "w1" and ckpt_up_proj_name == "w3":
|
||||
gate_up = "w13"
|
||||
@@ -1143,10 +1216,16 @@ class RoutedExperts(PluggableLayer):
|
||||
if gate_up is not None:
|
||||
fused_mapping = [
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
(f"{w13}weight", f"experts.{gate_up}", 0, "w1"),
|
||||
(f"{w13}weight", f"experts.{gate_up}", 1, "w3"),
|
||||
# expert_id doubles as the chunk index when splitting w13.
|
||||
*(
|
||||
(f"{w13}weight", f"experts.{gate_up}", chunk, shard_id)
|
||||
for chunk, shard_id in enumerate(w13_shards)
|
||||
),
|
||||
(f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"),
|
||||
]
|
||||
if gate_up is not None and is_gated:
|
||||
# Per-expert gate_up_proj tensors; the non-gated equivalent is
|
||||
# already covered by the per-expert mapping below.
|
||||
fused_mapping.extend(
|
||||
(
|
||||
w13,
|
||||
@@ -1175,6 +1254,7 @@ class RoutedExperts(PluggableLayer):
|
||||
("w2", ckpt_down_proj_name),
|
||||
("w3", ckpt_up_proj_name),
|
||||
]
|
||||
if weight_name is not None
|
||||
]
|
||||
|
||||
return fused_mapping + per_expert_mapping
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm BF16x3 MoE router GEMM for fp32 gate weights.
|
||||
|
||||
Gates whose weights ship in fp32 (MiniMax-M2/M3, HunYuan-V3) otherwise land in
|
||||
the ``F.linear`` fallback, which casts the bf16 activations up to fp32 and runs
|
||||
the GEMM on the fp32 MFMA pipe. Splitting the weight into three bf16 terms
|
||||
whose sum reproduces it exactly moves the GEMM onto the bf16 pipe and is more
|
||||
accurate than the path it replaces, which rounds the activations on the way to
|
||||
fp32 while this one leaves them untouched.
|
||||
|
||||
The terms are stacked along the expert dimension into one ``(TERMS * E, K)``
|
||||
weight, so the router costs a single hipBLASLt GEMM plus a reduction over the
|
||||
``TERMS`` column blocks of its output.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
__all__ = [
|
||||
"BF16X3_TERMS",
|
||||
"MIN_TOKENS",
|
||||
"bf16x3_router_gemm",
|
||||
"is_supported",
|
||||
"platform_supported",
|
||||
"split_bf16x3",
|
||||
]
|
||||
|
||||
BF16X3_TERMS = 3
|
||||
|
||||
# Below this the fp32 fallback is faster: the GEMM plus reduction has a ~32us
|
||||
# floor from launch and allocation that small batches cannot amortize.
|
||||
# Measured on gfx950; the crossover is 1536 at (K=6144, E=128) and (K=3072,
|
||||
# E=256), and this leaves margin for untested shapes.
|
||||
MIN_TOKENS = 2048
|
||||
|
||||
_MAX_INT32 = (1 << 31) - 1
|
||||
|
||||
|
||||
def split_bf16x3(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Split an fp32 weight into bf16 terms whose sum reproduces it exactly.
|
||||
|
||||
Returns a contiguous ``(BF16X3_TERMS, E, K)`` tensor. Three
|
||||
round-to-nearest bf16 terms carry 24 mantissa bits, matching fp32's 24, so
|
||||
the reconstruction is bit-exact for finite normals down to ``2**-108``.
|
||||
|
||||
Raises ValueError on non-finite input: ``inf`` rounds to bf16 ``inf`` and
|
||||
``inf - inf`` is NaN, which would poison every logit rather than just the
|
||||
affected experts.
|
||||
"""
|
||||
if weight.dtype != torch.float32:
|
||||
raise ValueError(f"expected fp32 weight, got {weight.dtype}")
|
||||
if not torch.isfinite(weight).all():
|
||||
raise ValueError("router weight contains inf/NaN")
|
||||
terms = []
|
||||
resid = weight.detach().clone()
|
||||
for _ in range(BF16X3_TERMS):
|
||||
hi = resid.to(torch.bfloat16)
|
||||
terms.append(hi)
|
||||
resid -= hi.to(torch.float32)
|
||||
return torch.stack(terms).contiguous()
|
||||
|
||||
|
||||
def platform_supported() -> bool:
|
||||
"""Whether this platform has a benchmarked bf16x3 router path.
|
||||
|
||||
gfx942 has a different fp32:bf16 MFMA ratio and is not yet evaluated.
|
||||
"""
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
def is_supported(x: torch.Tensor, weight: torch.Tensor) -> bool:
|
||||
"""Whether this path can serve ``x @ weight.T`` -> fp32 profitably."""
|
||||
if x.dim() != 2 or weight.dim() != 2:
|
||||
return False
|
||||
n, k = weight.shape
|
||||
m = x.shape[0]
|
||||
return (
|
||||
m >= MIN_TOKENS
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.dtype == torch.float32
|
||||
and x.shape[1] == k
|
||||
# The weight may be offloaded while the activations are not; the
|
||||
# fallback this guards would then fail on a device mismatch.
|
||||
and x.device == weight.device
|
||||
# torch.mm accepts any stride, but a non-K-contiguous activation costs
|
||||
# a hidden copy that gives back the win over the fallback.
|
||||
and x.stride(1) == 1
|
||||
and 0 < m * BF16X3_TERMS * n <= _MAX_INT32
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _reduce_terms_kernel(
|
||||
y_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
stride_ym,
|
||||
stride_om,
|
||||
N: tl.constexpr,
|
||||
TERMS: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = tl.arange(0, BLOCK_N)
|
||||
mask = (offs_m < M)[:, None] & (offs_n < N)[None, :]
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for t in tl.static_range(TERMS):
|
||||
acc += tl.load(
|
||||
y_ptr + offs_m[:, None] * stride_ym + t * N + offs_n[None, :],
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(out_ptr + offs_m[:, None] * stride_om + offs_n[None, :], acc, mask=mask)
|
||||
|
||||
|
||||
def bf16x3_router_gemm(
|
||||
x: torch.Tensor,
|
||||
weight_split: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Router logits in fp32 from bf16 activations and a bf16x3 split weight."""
|
||||
m, k = x.shape
|
||||
terms, n, k_w = weight_split.shape
|
||||
assert terms == BF16X3_TERMS, f"expected {BF16X3_TERMS} terms, got {terms}"
|
||||
assert k == k_w, f"K mismatch: x has {k}, weight_split has {k_w}"
|
||||
assert weight_split.dtype == torch.bfloat16, weight_split.dtype
|
||||
assert weight_split.is_contiguous(), "weight_split must be contiguous"
|
||||
assert x.dtype == torch.bfloat16, x.dtype
|
||||
if out is None:
|
||||
out = torch.empty((m, n), dtype=torch.float32, device=x.device)
|
||||
else:
|
||||
assert out.shape == (m, n) and out.is_contiguous(), "bad out layout"
|
||||
|
||||
# Rows [t*E, (t+1)*E) of the (TERMS * E, K) view hold term t, so the GEMM
|
||||
# emits one column block per term and the reduction sums them.
|
||||
y = torch.empty((m, terms * n), dtype=torch.float32, device=x.device)
|
||||
torch.mm(x, weight_split.view(terms * n, k).T, out_dtype=torch.float32, out=y)
|
||||
|
||||
# 64 rows/program keeps the accumulator small enough to stay in registers
|
||||
# at the largest supported expert count.
|
||||
block_m = 64
|
||||
_reduce_terms_kernel[(triton.cdiv(m, block_m),)](
|
||||
y,
|
||||
out,
|
||||
m,
|
||||
y.stride(0),
|
||||
out.stride(0),
|
||||
N=n,
|
||||
TERMS=terms,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=triton.next_power_of_2(n),
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
@@ -4,6 +4,7 @@ import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
@@ -13,6 +14,8 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@PluggableLayer.register("gate_linear")
|
||||
class GateLinear(ReplicatedLinear):
|
||||
@@ -23,8 +26,10 @@ class GateLinear(ReplicatedLinear):
|
||||
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
|
||||
M<=32, model-specific shapes)
|
||||
3. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
4. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
5. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
4. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
|
||||
M>=2048)
|
||||
5. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
6. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
|
||||
The ``out_dtype`` attribute is mutable and can be set after init
|
||||
(e.g. when the required dtype depends on the expert quantization
|
||||
@@ -140,6 +145,33 @@ class GateLinear(ReplicatedLinear):
|
||||
and self.out_dtype == torch.float32
|
||||
)
|
||||
|
||||
# ROCm bf16x3 router GEMM eligibility. out_dtype may still be None here
|
||||
# and is folded in by set_out_dtype; the split itself is only built
|
||||
# once the tier is known to be on (see _GateLinearMethod).
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm,
|
||||
)
|
||||
|
||||
self._rocm_bf16x3_weight_eligible = (
|
||||
self._router_gemm_no_bias
|
||||
and self.weight.dtype == torch.float32
|
||||
and bf16x3_router_gemm_rocm.platform_supported()
|
||||
)
|
||||
self.allow_rocm_bf16x3_router_gemm = (
|
||||
self._rocm_bf16x3_weight_eligible and self.out_dtype == torch.float32
|
||||
)
|
||||
# Filled in by process_weights_after_loading. Not persistent: it is
|
||||
# derived from `weight` rather than loaded from a checkpoint.
|
||||
self.register_buffer("_bf16x3_weight", None, persistent=False)
|
||||
if self._rocm_bf16x3_weight_eligible:
|
||||
# Safe to swap after create_weights() because
|
||||
# UnquantizedLinearMethod holds no per-layer state. Asserted
|
||||
# because the swap would silently drop quantization if a
|
||||
# quant_config were ever plumbed into this layer.
|
||||
assert isinstance(self.quant_method, UnquantizedLinearMethod)
|
||||
self.quant_method = _GateLinearMethod()
|
||||
logger.info_once("Enabled ROCm BF16x3 router GEMM.")
|
||||
|
||||
# cuteDSL ll_bf16_gemm eligibility. Any dims supported, but SM90+ required bc:
|
||||
# 1. PDL support. Both dot-product and split-K kernels.
|
||||
# 2. Thread Block Clusters. Split-K kernel for cross-CTA reduction.
|
||||
@@ -173,6 +205,10 @@ class GateLinear(ReplicatedLinear):
|
||||
self.allow_cublas_router_gemm = self.weight.dtype == torch.bfloat16
|
||||
|
||||
# out_dtype may start as None -> recompute eligibility here
|
||||
self.allow_rocm_bf16x3_router_gemm = (
|
||||
self._rocm_bf16x3_weight_eligible and out_dtype == torch.float32
|
||||
)
|
||||
|
||||
if self.allow_specialized_router_gemm:
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
|
||||
is_available,
|
||||
@@ -205,6 +241,23 @@ class GateLinear(ReplicatedLinear):
|
||||
output = ll_bf16_gemm(x, self.weight)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 4: ROCm bf16x3 router GEMM. Checked before tier 2 because both
|
||||
# tiers claim the same gfx950 shapes and tier 2 returns
|
||||
# unconditionally; the custom op below picks between them on the
|
||||
# runtime num_tokens.
|
||||
if self.allow_rocm_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
if self._bf16x3_weight is None:
|
||||
logger.warning_once(
|
||||
"ROCm BF16x3 router GEMM is enabled for %s but its weight "
|
||||
"split is missing; falling back to fp32.",
|
||||
self.prefix,
|
||||
)
|
||||
else:
|
||||
output = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(
|
||||
x, self.weight, self._bf16x3_weight
|
||||
)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 2: fp32 specialized kernel (model-specific shapes, M<=32)
|
||||
# Dispatch is wrapped in a custom op so that torch.compile/CUDA-graph
|
||||
# capture does not freeze the runtime num_tokens branch.
|
||||
@@ -226,12 +279,12 @@ class GateLinear(ReplicatedLinear):
|
||||
output = bf16x3_router_gemm(x, self.weight)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 4: cuBLAS bf16→fp32
|
||||
# Tier 5: cuBLAS bf16→fp32
|
||||
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
|
||||
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 5: F.linear (ReplicatedLinear)
|
||||
# Tier 6: F.linear (ReplicatedLinear)
|
||||
if (
|
||||
self.out_dtype is not None
|
||||
and self.is_unquantized
|
||||
@@ -249,9 +302,90 @@ class GateLinear(ReplicatedLinear):
|
||||
return output if not self.return_bias else (output, output_bias)
|
||||
|
||||
|
||||
class _GateLinearMethod(UnquantizedLinearMethod):
|
||||
"""UnquantizedLinearMethod plus the ROCm bf16x3 weight split.
|
||||
|
||||
Building the split here rather than lazily in forward() charges it to the
|
||||
weights memory pool ahead of KV cache profiling, and rebuilds it on weight
|
||||
reload. The latter matters because an in-place ``param.data.copy_()``
|
||||
leaves both ``data_ptr()`` and ``_version`` untouched, so a stale split
|
||||
cannot be detected from inside forward().
|
||||
|
||||
``allow_rocm_bf16x3_router_gemm`` is final by the time this runs: the only
|
||||
caller of ``set_out_dtype`` does so during model construction.
|
||||
"""
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# The base CPU branch may delete layer.weight, so it must run first;
|
||||
# nothing breaks today only because platform_supported() is False on
|
||||
# CPU.
|
||||
super().process_weights_after_loading(layer)
|
||||
if not layer.allow_rocm_bf16x3_router_gemm:
|
||||
return
|
||||
try:
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm,
|
||||
)
|
||||
|
||||
layer._bf16x3_weight = bf16x3_router_gemm_rocm.split_bf16x3(layer.weight)
|
||||
except ValueError as exc:
|
||||
layer.allow_rocm_bf16x3_router_gemm = False
|
||||
logger.warning_once(
|
||||
"Disabling ROCm BF16x3 router GEMM for %s: %s", layer.prefix, exc
|
||||
)
|
||||
|
||||
|
||||
_FP32_ROUTER_GEMM_MAX_TOKENS = GateLinear.FP32_MAX_TOKENS
|
||||
|
||||
|
||||
def rocm_bf16x3_router_gemm_dispatch_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_split: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Run the ROCm bf16x3 router GEMM, or fall back for small batches.
|
||||
|
||||
This must be a custom op because our torch.compile integration does not
|
||||
support runtime dispatching on num_tokens: models carrying fp32 router
|
||||
weights (MiniMax-M2, HunYuan-V3) call the gate from
|
||||
``@support_torch_compile`` model code and vLLM drops all Dynamo guards, so
|
||||
a plain Python branch on ``x.shape[0]`` would be frozen at first trace.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm,
|
||||
)
|
||||
|
||||
if bf16x3_router_gemm_rocm.is_supported(x, weight):
|
||||
return bf16x3_router_gemm_rocm.bf16x3_router_gemm(x, weight_split)
|
||||
# Below MIN_TOKENS, hand back to the low-M gfx950 kernel this tier is
|
||||
# checked ahead of.
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
|
||||
can_use_rocm_fp32_router_gemm,
|
||||
rocm_fp32_router_gemm,
|
||||
)
|
||||
|
||||
x = x.contiguous()
|
||||
if can_use_rocm_fp32_router_gemm(x, weight):
|
||||
return rocm_fp32_router_gemm(x, weight)
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
|
||||
|
||||
def rocm_bf16x3_router_gemm_dispatch_fake(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_split: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return x.new_empty((x.shape[0], weight.shape[0]), dtype=torch.float32)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_bf16x3_router_gemm_dispatch",
|
||||
op_func=rocm_bf16x3_router_gemm_dispatch_impl,
|
||||
fake_impl=rocm_bf16x3_router_gemm_dispatch_fake,
|
||||
)
|
||||
|
||||
|
||||
def fp32_router_gemm_dispatch_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
|
||||
@@ -84,6 +84,40 @@ def resolve_layer_fused_shared_expert(
|
||||
else (True, None)
|
||||
)
|
||||
is_fused_shared_expert_enabled = fse_requested and fse_compatible
|
||||
|
||||
# In online shared_expert quantization is used, register it to
|
||||
# `online_quantization_config.quantized_layers` ahead of weight loading.
|
||||
if is_fused_shared_expert_enabled and quant_config is not None:
|
||||
online_quantization_config = quant_config.online_quantization_config
|
||||
if online_quantization_config is not None:
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
for projection_name in ("gate_up_proj", "down_proj"):
|
||||
projection_prefix = f"{prefix}.{shared_expert_name}.{projection_name}"
|
||||
quant_method_metadata = (
|
||||
online_quantization_config.resolve_quant_method_cls(
|
||||
LinearBase, projection_prefix
|
||||
)
|
||||
)
|
||||
|
||||
if quant_method_metadata is None:
|
||||
continue
|
||||
|
||||
source, quant_key_str, target_pattern, _, quant_method_cls = (
|
||||
quant_method_metadata
|
||||
)
|
||||
|
||||
if quant_method_cls in (None, UnquantizedLinearMethod):
|
||||
continue
|
||||
|
||||
online_quantization_config.quantized_layers[projection_prefix] = (
|
||||
source.value,
|
||||
quant_key_str,
|
||||
target_pattern,
|
||||
)
|
||||
if fse_requested and not is_fused_shared_expert_enabled:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
|
||||
@@ -63,10 +63,10 @@ def _silu_and_mul_fp8_dynamic_block(
|
||||
x: torch.Tensor, linear: LinearBase, group_size: int, quant_key: QuantKey
|
||||
) -> QuantizedActivation:
|
||||
"""SiluAndMul + FP8 dynamic per-block quantization."""
|
||||
assert x.ndim == 2, f"Input must be 2D [batch, hidden*2], got {x.shape}"
|
||||
|
||||
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
|
||||
d = x.shape[-1] // 2
|
||||
out_shape = x.shape[:-1] + (d,)
|
||||
x = x.reshape(-1, x.shape[-1])
|
||||
num_tokens = x.shape[0]
|
||||
num_groups = d // group_size
|
||||
|
||||
@@ -102,10 +102,10 @@ def _silu_and_mul_nvfp4_dynamic(
|
||||
x: torch.Tensor, linear: LinearBase
|
||||
) -> QuantizedActivation:
|
||||
"""SiluAndMul + NVFP4 dynamic quantization."""
|
||||
assert x.ndim == 2, f"Input must be 2D [batch, hidden*2], got {x.shape}"
|
||||
|
||||
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
|
||||
d = x.shape[-1] // 2
|
||||
out_shape = x.shape[:-1] + (d,)
|
||||
x = x.reshape(-1, x.shape[-1])
|
||||
num_tokens = x.shape[0]
|
||||
|
||||
# NVFP4 packs 2 values into 1 byte
|
||||
|
||||
@@ -152,6 +152,7 @@ def _load_norm_params(model_config: ModelConfig) -> NormParams:
|
||||
for key in ("do_normalize", "do_rescale", "image_mean", "image_std")
|
||||
):
|
||||
config = model_config.hf_image_processor_config
|
||||
config = config.get("media_proc_cfg", config)
|
||||
|
||||
has_norm_params = "image_mean" in config and "image_std" in config
|
||||
do_normalize = bool(config.get("do_normalize", has_norm_params))
|
||||
@@ -312,19 +313,15 @@ class FusedMMInputNorm(CustomOp):
|
||||
def forward_xpu(
|
||||
self, pixel_values: torch.Tensor, visual_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""XPU fused custom kernel path.
|
||||
|
||||
On XPU, fuse the whole rescale + normalise into a single custom
|
||||
kernel. The eager path materializes an fp32 intermediate and then
|
||||
casts back, which adds device-side compute that cancels the
|
||||
bandwidth saving of transferring uint8 pixel_values. The fused
|
||||
kernel reads uint8 directly and writes ``visual_dtype`` in one pass.
|
||||
"""XPU fused custom kernel path for uint8 and Triton kernel path
|
||||
for others.
|
||||
"""
|
||||
# The out-of-tree XPU kernel only supports the uint8 input that
|
||||
# device-side normalisation guarantees; fail loudly otherwise.
|
||||
assert pixel_values.dtype == torch.uint8, (
|
||||
f"xpu_fused_input_norm requires uint8 input, got {pixel_values.dtype}"
|
||||
)
|
||||
if pixel_values.dtype != torch.uint8:
|
||||
return self.forward_cuda(pixel_values, visual_dtype)
|
||||
|
||||
# For uint8 input, falls to xpu_fused_input_norm from xpu kernels
|
||||
import vllm._xpu_ops # noqa: F401
|
||||
|
||||
return torch.ops.vllm.xpu_fused_input_norm(
|
||||
pixel_values, self.weight, self.bias, visual_dtype
|
||||
)
|
||||
|
||||
@@ -29,12 +29,10 @@ IDX_OOB_FILL_VALUE = -1
|
||||
|
||||
try:
|
||||
import vllm._deepselect_C # noqa: F401 (registers torch.ops.deep_select)
|
||||
except ImportError as e:
|
||||
from vllm.logger import init_logger
|
||||
|
||||
init_logger(__name__).warning(
|
||||
"Failed to import the DeepSelect extension (vllm._deepselect_C): %s", e
|
||||
)
|
||||
_HAS_DEEP_SELECT_C = True
|
||||
except ImportError:
|
||||
_HAS_DEEP_SELECT_C = False
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
@@ -196,6 +194,8 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
failures.append("requires a CUDA platform")
|
||||
elif not self._has_deep_select:
|
||||
failures.append("requires SM100a/SM103a (10.x device family)")
|
||||
elif not _HAS_DEEP_SELECT_C:
|
||||
failures.append("vllm._deepselect_C could not be imported")
|
||||
elif not is_deep_select_supported(logits, topk_tokens):
|
||||
failures.append(
|
||||
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
|
||||
|
||||
@@ -25,6 +25,7 @@ def compute_mamba_prefill_checkpoints(
|
||||
mamba_block_size: int,
|
||||
checkpoint_alignment: int | None,
|
||||
drop_eagle_block: bool,
|
||||
cache_hit_alignment_tokens: int,
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Per-row internal prefill checkpoint offsets and cache block columns.
|
||||
|
||||
@@ -43,7 +44,7 @@ def compute_mamba_prefill_checkpoints(
|
||||
for seq_len, query_len in zip(seq_lens, query_lens):
|
||||
query_start = seq_len - query_len
|
||||
position = get_mamba_prefill_checkpoint_position(
|
||||
seq_len, hash_block_size, drop_eagle_block=drop_eagle_block
|
||||
seq_len, cache_hit_alignment_tokens, drop_eagle_block=drop_eagle_block
|
||||
)
|
||||
valid = is_mamba_prefill_checkpoint_valid(
|
||||
query_start=query_start,
|
||||
@@ -70,6 +71,9 @@ def _gather_checkpoint_state_indices(
|
||||
class MambaPrefillCheckpointMetadata:
|
||||
checkpoint_offsets: torch.Tensor
|
||||
state_indices: torch.Tensor
|
||||
# Host copy of ``checkpoint_offsets``, for backends that lay out the
|
||||
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
|
||||
offsets: list[int] | None = None
|
||||
# Gather indices into the block table, kept so that another KV cache group
|
||||
# with the same spec can re-derive its own ``state_indices``.
|
||||
request_rows: torch.Tensor | None = None
|
||||
@@ -103,12 +107,17 @@ class MambaPrefillCheckpointBuilder:
|
||||
return None
|
||||
if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0:
|
||||
return None
|
||||
cache_config = self.vllm_config.cache_config
|
||||
hash_block_size = cache_config.hash_block_size
|
||||
cache_hit_alignment_tokens = cache_config.cache_hit_alignment_tokens
|
||||
if hash_block_size is None or cache_hit_alignment_tokens is None:
|
||||
# Unresolved only for the minimal KV cache built for profiling.
|
||||
return None
|
||||
assert m.seq_lens_cpu_upper_bound is not None
|
||||
all_query_lens = m.query_start_loc_cpu.diff().tolist()
|
||||
query_lens = [all_query_lens[row] for row in request_rows]
|
||||
seq_lens = m.seq_lens_cpu_upper_bound.tolist()
|
||||
block_size = self.kv_cache_spec.block_size
|
||||
hash_block_size = self.vllm_config.cache_config.prefix_match_unit or block_size
|
||||
speculative_config = self.vllm_config.speculative_config
|
||||
drop_eagle_block = (
|
||||
speculative_config is not None and speculative_config.use_eagle_block_drop()
|
||||
@@ -120,6 +129,7 @@ class MambaPrefillCheckpointBuilder:
|
||||
mamba_block_size=block_size,
|
||||
checkpoint_alignment=self.kv_cache_spec.prefill_checkpoint_alignment,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
cache_hit_alignment_tokens=cache_hit_alignment_tokens,
|
||||
)
|
||||
if not any(checkpoint_offsets):
|
||||
return None
|
||||
@@ -139,6 +149,7 @@ class MambaPrefillCheckpointBuilder:
|
||||
_gather_checkpoint_state_indices(
|
||||
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
|
||||
),
|
||||
offsets=checkpoint_offsets,
|
||||
request_rows=request_rows_tensor,
|
||||
block_cols=checkpoint_cols_tensor,
|
||||
)
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Literal
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
from transformers import Qwen3NextConfig
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
@@ -58,7 +59,6 @@ from vllm.third_party.flash_linear_attention.ops import (
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
@@ -92,7 +92,7 @@ FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
|
||||
|
||||
def _resolve_gdn_prefill_backend(
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl"]]:
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
|
||||
"""Resolve GDN prefill backend.
|
||||
|
||||
FlashInfer's GDN prefill kernel is chosen when:
|
||||
@@ -106,6 +106,11 @@ def _resolve_gdn_prefill_backend(
|
||||
In-tree CuteDSL GDN prefill kernel is chosen when:
|
||||
* "cutedsl" is requested; (opt-in only)
|
||||
* Blackwell (SM10.x) with ``head_k_dim == 128``;
|
||||
|
||||
AITER FlyDSL GDN prefill kernels are chosen when:
|
||||
* "aiter_flydsl" is requested; (opt-in only)
|
||||
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
|
||||
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
|
||||
"""
|
||||
additional_config = vllm_config.additional_config
|
||||
backend_cfg = (
|
||||
@@ -115,12 +120,50 @@ def _resolve_gdn_prefill_backend(
|
||||
)
|
||||
backend = str(backend_cfg).strip().lower()
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
return backend, "triton"
|
||||
|
||||
head_k_dim = getattr(
|
||||
vllm_config.model_config.hf_text_config, "linear_key_head_dim", None
|
||||
)
|
||||
head_v_dim = getattr(
|
||||
vllm_config.model_config.hf_text_config, "linear_value_head_dim", None
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
if backend == "aiter_flydsl":
|
||||
if not rocm_aiter_ops.is_gdn_flydsl_prefill_available():
|
||||
raise RuntimeError(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but the "
|
||||
"AITER FlyDSL prefill kernels are not importable. Check "
|
||||
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
|
||||
"newer GPU, and that the installed AITER exports them."
|
||||
)
|
||||
if head_k_dim != 128 or head_v_dim != 128:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
|
||||
"Falling back to Triton/FLA.",
|
||||
head_k_dim,
|
||||
head_v_dim,
|
||||
)
|
||||
return backend, "triton"
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"model dtype is %s; FlyDSL requires bfloat16. "
|
||||
"Falling back to Triton/FLA.",
|
||||
vllm_config.model_config.dtype,
|
||||
)
|
||||
return backend, "triton"
|
||||
return backend, "aiter_flydsl"
|
||||
return backend, "triton"
|
||||
|
||||
if backend == "aiter_flydsl":
|
||||
raise RuntimeError(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but it is a ROCm "
|
||||
f"AITER backend and the current platform is {current_platform.device_name}."
|
||||
)
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
return backend, "triton"
|
||||
|
||||
supports_flashinfer = False
|
||||
supports_cutedsl = False
|
||||
@@ -172,6 +215,7 @@ def _log_gdn_backend_decision(
|
||||
chosen = {
|
||||
"flashinfer": "FlashInfer",
|
||||
"cutedsl": "CuteDSL",
|
||||
"aiter_flydsl": "AITER FlyDSL",
|
||||
"triton": "Triton/FLA",
|
||||
}[active_backend]
|
||||
logger.info_once(
|
||||
@@ -259,6 +303,8 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
self._forward_method = self.forward_cuda
|
||||
elif active_backend == "cutedsl":
|
||||
self._forward_method = self.forward_cutedsl
|
||||
elif active_backend == "aiter_flydsl":
|
||||
self._forward_method = self.forward_aiter_flydsl
|
||||
else:
|
||||
self._forward_method = self.forward_native
|
||||
|
||||
@@ -276,6 +322,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
o, final_state = fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
@@ -308,6 +355,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
return fla_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
@@ -338,6 +386,7 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
|
||||
chunk_gated_delta_rule_cutedsl,
|
||||
@@ -367,6 +416,39 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
final_state = None
|
||||
return o, final_state
|
||||
|
||||
def forward_aiter_flydsl(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
initial_state: torch.Tensor,
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
aiter_prefill_metadata: object | None = None,
|
||||
):
|
||||
o, final_state = rocm_aiter_ops.gdn_flydsl_prefill(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
)
|
||||
if core_attn_out is not None:
|
||||
o_flat = o.reshape(-1)
|
||||
core_attn_out.reshape(-1)[: o_flat.numel()].copy_(o_flat)
|
||||
return o, final_state
|
||||
|
||||
|
||||
@PluggableLayer.register("qwen_gated_delta_net_attention")
|
||||
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
@@ -1193,12 +1275,18 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
# CuteDSL kernels require metadata
|
||||
chunk_indices = None
|
||||
chunk_offsets = None
|
||||
aiter_prefill_metadata = None
|
||||
if self.gdn_prefill_backend == "cutedsl":
|
||||
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
|
||||
prepare_metadata_cutedsl,
|
||||
)
|
||||
|
||||
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, T)
|
||||
elif self.gdn_prefill_backend == "aiter_flydsl":
|
||||
aiter_prefill_metadata = rocm_aiter_ops.build_gdn_flydsl_prefill_metadata(
|
||||
[T],
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
|
||||
try:
|
||||
self.chunk_gated_delta_rule(
|
||||
@@ -1213,6 +1301,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
@@ -1243,6 +1332,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
chunk_offsets,
|
||||
aiter_prefill_metadata,
|
||||
)
|
||||
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1634,6 +1724,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
chunk_indices=attn_metadata.chunk_indices,
|
||||
chunk_offsets=attn_metadata.chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
|
||||
)
|
||||
# Init cache
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
|
||||
|
||||
@@ -3,11 +3,12 @@
|
||||
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
|
||||
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
|
||||
selective_state_update_replayssm_output_only,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import store_prefill_checkpoint
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
@@ -66,7 +68,7 @@ from vllm.utils.torch_utils import (
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec, MambaSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -766,6 +768,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
query_start_loc_d = attn_metadata.query_start_loc_d
|
||||
num_decodes = attn_metadata.num_decodes
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
checkpoint_chunk_idx = attn_metadata.checkpoint_chunk_idx
|
||||
checkpoint_meta = attn_metadata.checkpoint_meta
|
||||
|
||||
if attn_metadata is None:
|
||||
# V1 profile run -- warm up SSD kernels so that autotuning
|
||||
@@ -842,6 +846,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
|
||||
# NOTE: final output is an in-place update of out tensor
|
||||
assert preallocated_ssm_out_p is not None
|
||||
has_checkpoints = checkpoint_chunk_idx is not None
|
||||
varlen_states = mamba_chunk_scan_combined_varlen(
|
||||
hidden_states_p.view(
|
||||
num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim
|
||||
@@ -859,6 +864,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
cu_chunk_seqlens=cu_chunk_seqlen_p,
|
||||
last_chunk_indices=last_chunk_indices_p,
|
||||
initial_states=initial_states,
|
||||
return_intermediate_states=has_checkpoints,
|
||||
dt_softplus=True,
|
||||
dt_limit=(0.0, float("inf")),
|
||||
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
|
||||
@@ -869,7 +875,24 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
# - varlen state is a (num_prefills, nheads, headdim, dstate)
|
||||
# tensor
|
||||
assert state_indices_tensor_p is not None
|
||||
ssm_state[state_indices_tensor_p] = varlen_states
|
||||
final_states = (
|
||||
varlen_states[last_chunk_indices_p]
|
||||
if has_checkpoints
|
||||
else varlen_states
|
||||
)
|
||||
if has_checkpoints:
|
||||
assert checkpoint_meta is not None
|
||||
assert query_start_loc_p is not None
|
||||
store_prefill_checkpoint(
|
||||
checkpoint_meta,
|
||||
checkpoint_chunk_idx,
|
||||
x,
|
||||
conv_state,
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
query_start_loc_p,
|
||||
)
|
||||
ssm_state[state_indices_tensor_p] = final_states
|
||||
if ring_start is not None and self._updates_replayssm_trackers:
|
||||
assert prev_num_accepted is not None
|
||||
reset_replayssm_ring_trackers(
|
||||
@@ -1070,6 +1093,18 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
is_blackwell=self.is_blackwell,
|
||||
)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
if not isinstance(spec, MambaSpec):
|
||||
return spec
|
||||
# The block-boundary state can be exported from inside one forward
|
||||
# pass, so the scheduler need not split the prefill; every reader
|
||||
# acts on this only in align mode. The metadata builder puts a chunk
|
||||
# end exactly on the checkpoint, so any token position works.
|
||||
return replace(
|
||||
spec, num_prefill_checkpoint_blocks=1, prefill_checkpoint_alignment=1
|
||||
)
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"96": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"384": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"768": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"6144": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"12288": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"24576": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"49152": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"384": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"768": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"1536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"3072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"6144": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
},
|
||||
"12288": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"24576": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 2
|
||||
},
|
||||
"49152": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 2
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"16384": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"32768": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"65536": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
},
|
||||
"131072": {
|
||||
"BLOCK_SIZE_M": 4,
|
||||
"num_warps": 1
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"80": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 4
|
||||
},
|
||||
"640": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"1280": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"2560": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"10240": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"20480": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"40960": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"81920": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"122880": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
},
|
||||
"163840": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"num_warps": 8
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"triton_version": "3.8.0",
|
||||
"80": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"640": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"1280": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"num_warps": 8
|
||||
},
|
||||
"2560": {
|
||||
"BLOCK_SIZE_M": 8,
|
||||
"num_warps": 4
|
||||
},
|
||||
"5120": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 8
|
||||
},
|
||||
"10240": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 8
|
||||
},
|
||||
"20480": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 8
|
||||
},
|
||||
"40960": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"81920": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"122880": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
},
|
||||
"163840": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"num_warps": 2
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_prefill_checkpoint_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
chunk_states_ptr,
|
||||
ssm_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
state_indices_ptr,
|
||||
chunk_idx_ptr,
|
||||
stride_x_dim,
|
||||
stride_x_token,
|
||||
stride_conv_block,
|
||||
stride_conv_dim,
|
||||
stride_conv_token,
|
||||
stride_chunk_states,
|
||||
stride_ssm_block,
|
||||
DIM: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
SSM_ROW_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
checkpoint_offset = tl.load(checkpoint_offsets_ptr + row)
|
||||
valid = checkpoint_offset > 0
|
||||
dst = tl.load(state_indices_ptr + row).to(tl.int64)
|
||||
|
||||
conv_mask = valid & (cols < DIM * STATE_LEN)
|
||||
dim_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
token_idx = (
|
||||
tl.load(query_start_loc_ptr + row) + checkpoint_offset - STATE_LEN + history_idx
|
||||
).to(tl.int64)
|
||||
conv = tl.load(
|
||||
x_ptr + dim_idx * stride_x_dim + token_idx * stride_x_token, mask=conv_mask
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ dst * stride_conv_block
|
||||
+ dim_idx * stride_conv_dim
|
||||
+ history_idx * stride_conv_token,
|
||||
conv,
|
||||
mask=conv_mask,
|
||||
)
|
||||
|
||||
ssm_mask = valid & (cols < SSM_ROW_SIZE)
|
||||
src = tl.load(chunk_idx_ptr + row).to(tl.int64)
|
||||
ssm = tl.load(chunk_states_ptr + src * stride_chunk_states + cols, mask=ssm_mask)
|
||||
tl.store(ssm_state_ptr + dst * stride_ssm_block + cols, ssm, mask=ssm_mask)
|
||||
|
||||
|
||||
def store_prefill_checkpoint(
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
chunk_idx: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
chunk_states: torch.Tensor,
|
||||
ssm_state: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
) -> None:
|
||||
"""Write each prefill row's mid-prefill checkpoint into its paged block.
|
||||
|
||||
Rows without a checkpoint (offset 0) write nothing.
|
||||
|
||||
Args:
|
||||
checkpoint: Per-row offsets and destination blocks.
|
||||
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
|
||||
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
|
||||
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
|
||||
chunk_states: SSD states at every chunk end.
|
||||
ssm_state: Paged SSM state.
|
||||
query_start_loc: Prefill query start locations.
|
||||
|
||||
"""
|
||||
dim = x.size(0)
|
||||
state_len = conv_state.size(-1)
|
||||
ssm_row_size = chunk_states[0].numel()
|
||||
block_size = 1024
|
||||
grid = (
|
||||
chunk_idx.numel(),
|
||||
triton.cdiv(max(dim * state_len, ssm_row_size), block_size),
|
||||
)
|
||||
_store_prefill_checkpoint_kernel[grid](
|
||||
x,
|
||||
conv_state,
|
||||
chunk_states,
|
||||
ssm_state,
|
||||
query_start_loc,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
chunk_idx,
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
chunk_states.stride(0),
|
||||
ssm_state.stride(0),
|
||||
DIM=dim,
|
||||
STATE_LEN=state_len,
|
||||
SSM_ROW_SIZE=ssm_row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
@@ -84,6 +84,13 @@ class QuantizeMethodBase(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
|
||||
"""Materialize a serialized quantized weight for requantization."""
|
||||
raise NotImplementedError(
|
||||
f"The quantization method {type(self)} does not implement "
|
||||
"dequantize_weight. Please open an issue."
|
||||
)
|
||||
|
||||
|
||||
def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool:
|
||||
"""Not all quant methods have embedding implemented, so we need to check that
|
||||
@@ -294,13 +301,16 @@ def resolve_quant_method(
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
|
||||
base_quant_method = quant_config.get_quant_method(layer, prefix)
|
||||
if quant_config.online_quantization_config is None:
|
||||
# No online configuration: retain the checkpoint method.
|
||||
return base_quant_method
|
||||
# Online quantization currently supports only LinearBase and RoutedExperts.
|
||||
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
|
||||
if not isinstance(layer, (LinearBase, RoutedExperts)):
|
||||
# Online quantization only supports linear and routed MoE layers.
|
||||
return base_quant_method
|
||||
|
||||
quant_config.online_quantization_config.packed_modules_mapping = (
|
||||
@@ -310,16 +320,36 @@ def resolve_quant_method(
|
||||
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
|
||||
)
|
||||
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
|
||||
layer, prefix
|
||||
type(layer), prefix
|
||||
)
|
||||
|
||||
if checkpoint_is_quantized:
|
||||
if online_target is not None:
|
||||
raise ValueError(
|
||||
f"Cannot apply requested online quantization {online_target[3]} to "
|
||||
f"pre-quantized layer {prefix}: {base_quant_method} was already "
|
||||
"selected by the checkpoint quantization config."
|
||||
if online_target is None:
|
||||
# The checkpoint quant method is applied as there is no online override.
|
||||
return base_quant_method
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
raise NotImplementedError(
|
||||
"Requantizing checkpoint-quantized MoE layers is not supported."
|
||||
)
|
||||
return base_quant_method
|
||||
|
||||
online_quant_method = quant_config.online_quantization_config.get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
|
||||
assert base_quant_method is not None and not isinstance(
|
||||
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
|
||||
)
|
||||
|
||||
assert isinstance(online_quant_method, OnlineLinearBase)
|
||||
online_quant_method.set_requantization_source(base_quant_method)
|
||||
|
||||
# The online method dequantizes the checkpoint method before requantizing.
|
||||
return online_quant_method
|
||||
|
||||
if online_target is None:
|
||||
# The layer is unquantized and no online target applies.
|
||||
return base_quant_method
|
||||
|
||||
# Quantize an unquantized layer with the online method.
|
||||
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
|
||||
|
||||
+4
-2
@@ -450,10 +450,12 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
if w2_bias is not None:
|
||||
replace_parameter(layer, "w2_bias", w2_bias)
|
||||
|
||||
# CPU fused_experts_cpu requires zero points even for symmetric quant.
|
||||
# CPU fused_experts_cpu and the RDNA3 HIP kernel require zero points
|
||||
# even for symmetric quant (the oracle synthesizes them).
|
||||
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
|
||||
if (
|
||||
not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU
|
||||
not self.symmetric
|
||||
or self.wna16_backend in (WNA16MoEBackend.CPU, WNA16MoEBackend.RDNA3)
|
||||
) and self.wna16_backend != WNA16MoEBackend.EMULATION:
|
||||
assert w13_qzeros is not None and w2_qzeros is not None
|
||||
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
|
||||
|
||||
@@ -12,7 +12,7 @@ from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.config.quantization import QuantSpec
|
||||
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
init_fp8_linear_kernel,
|
||||
@@ -63,6 +63,9 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
trtllm_nvfp4_hidden_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
FP8_SCALE_SENTINEL,
|
||||
process_fp8_input_tensor_strategy_moe,
|
||||
@@ -76,6 +79,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP4_DTYPE,
|
||||
@@ -89,6 +93,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4DynamicToken,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
@@ -723,12 +728,18 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
|
||||
kv_cache_quant_algo: str | None = None,
|
||||
exclude_modules: list[str] | None = None,
|
||||
group_size: int = 16,
|
||||
*,
|
||||
quantization_args: QuantizationConfigArgs | None = None,
|
||||
) -> None:
|
||||
if exclude_modules is None:
|
||||
exclude_modules = []
|
||||
super().__init__(exclude_modules)
|
||||
self.quant_method = quant_method
|
||||
self.is_checkpoint_nvfp4_serialized = is_checkpoint_nvfp4_serialized
|
||||
moe_spec = quantization_args.moe if quantization_args is not None else None
|
||||
self.moe_activation_override: QuantKey | None = (
|
||||
moe_spec.activation if moe_spec is not None else None
|
||||
)
|
||||
if is_checkpoint_nvfp4_serialized:
|
||||
self.group_size = group_size
|
||||
self.kv_cache_quant_algo = kv_cache_quant_algo
|
||||
@@ -814,6 +825,7 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
|
||||
kv_cache_quant_method,
|
||||
exclude_modules,
|
||||
group_size,
|
||||
quantization_args=original_config.get("_online_quantization_args"),
|
||||
)
|
||||
|
||||
|
||||
@@ -834,18 +846,36 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
) -> None:
|
||||
super().__init__(moe_config)
|
||||
self.quant_config = quant_config
|
||||
# W4A16 mode fires for W4A16_NVFP4 on-disk checkpoints. With
|
||||
# activation_key=None every W4A4 backend's _supports_quant_scheme
|
||||
# rejects itself (they all require (kNvfp4Static, kNvfp4Dynamic)
|
||||
# exactly); only Marlin survives. Marlin's MoE path drops
|
||||
# activation scales in convert_to_nvfp4_moe_kernel_format, so no
|
||||
# other change is needed.
|
||||
self.use_a16 = quant_config.quant_method == "W4A16_NVFP4"
|
||||
activation_key = None if self.use_a16 else kNvfp4Dynamic
|
||||
|
||||
if quant_config.moe_activation_override is not None:
|
||||
if self.use_a16:
|
||||
raise ValueError(
|
||||
"NVFP4 activation overrides require a W4A4 checkpoint."
|
||||
)
|
||||
activation_key = quant_config.moe_activation_override
|
||||
if activation_key not in (kNvfp4Dynamic, kNvfp4DynamicToken):
|
||||
raise ValueError(
|
||||
"Unsupported ModelOpt NVFP4 MoE activation override: "
|
||||
f"{activation_key}. Use quantization_config.moe.activation="
|
||||
"'nvfp4_per_token' or omit the override."
|
||||
)
|
||||
self.per_token_activation = activation_key == kNvfp4DynamicToken
|
||||
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kNvfp4Static,
|
||||
activation_key=None if self.use_a16 else kNvfp4Dynamic,
|
||||
activation_key=activation_key,
|
||||
)
|
||||
if (
|
||||
self.per_token_activation
|
||||
and self.nvfp4_backend != NvFp4MoeBackend.FLASHINFER_TRTLLM
|
||||
):
|
||||
raise ValueError(
|
||||
"Per-token NVFP4 activation for pre-quantized weights "
|
||||
"requires the FlashInfer TRTLLM MoE backend."
|
||||
)
|
||||
|
||||
self.use_global_sf = is_global_sf_supported_for_nvfp4_backend(
|
||||
self.nvfp4_backend
|
||||
@@ -983,6 +1013,10 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
self._build_moe_kernel(layer)
|
||||
return
|
||||
|
||||
if self.per_token_activation:
|
||||
layer.w13_input_scale.data.fill_(1.0)
|
||||
layer.w2_input_scale.data.fill_(1.0)
|
||||
|
||||
# Use a single gscale for w13.
|
||||
if self.moe.is_act_and_mul and not torch.allclose(
|
||||
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
|
||||
@@ -1015,6 +1049,10 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
a2_scale=layer.w2_input_scale,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
use_a16=self.use_a16,
|
||||
trtllm_hidden_alignment=trtllm_nvfp4_hidden_alignment(
|
||||
per_token_activation=self.per_token_activation,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
),
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
@@ -1038,6 +1076,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
per_token_activation=self.per_token_activation,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
@@ -1587,6 +1626,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
group_size: int | None,
|
||||
**kwargs: Any,
|
||||
) -> "ModelOptMixedPrecisionConfig":
|
||||
quantization_args = original_config.get("_online_quantization_args")
|
||||
if "quantization" in original_config:
|
||||
quantized_layers = original_config["quantization"].get(
|
||||
"quantized_layers", {}
|
||||
@@ -1625,6 +1665,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
kv_cache_quant_algo=kv_cache_quant_method,
|
||||
exclude_modules=[],
|
||||
group_size=group_size,
|
||||
quantization_args=quantization_args,
|
||||
)
|
||||
# Sibling config for layers that declare quant_algo: "W4A16_NVFP4".
|
||||
# get_quant_method resolves this sub-config to the (kNvfp4Static, None)
|
||||
@@ -1637,6 +1678,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
kv_cache_quant_algo=kv_cache_quant_method,
|
||||
exclude_modules=[],
|
||||
group_size=group_size,
|
||||
quantization_args=quantization_args,
|
||||
)
|
||||
|
||||
mxfp8_config = ModelOptMxFp8Config(
|
||||
@@ -2572,6 +2614,17 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
layer._nvfp4_group_size_for_gather = self.ctx.group_size
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def dequantize_weight(self, layer: torch.nn.Module) -> torch.Tensor:
|
||||
"""Reconstruct serialized weights for online requantization."""
|
||||
if self.wkey.key is kMxfp8Static:
|
||||
return dequant_mxfp8_to_bf16(
|
||||
layer.weight.contiguous(), layer.weight_scale.contiguous()
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"ModelOpt weight dequantization is only supported for MXFP8."
|
||||
)
|
||||
|
||||
def apply(self, layer, x, bias=None):
|
||||
return self.fmt.apply(
|
||||
layer,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
@@ -60,6 +60,7 @@ from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
get_layer_name_after_index,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Static128BlockSym,
|
||||
@@ -104,6 +105,15 @@ _ONLINE_MOE_METHODS: dict[QuantKey, type] = {
|
||||
kNvfp4Static: Nvfp4OnlineMoEMethod,
|
||||
}
|
||||
|
||||
# Quantizers for full-precision fused shared-expert weights, keyed by the routed
|
||||
# experts' weight QuantKey. Each shard is quantized to the checkpoint layout on
|
||||
# arrival, so only block-local schemes are supported.
|
||||
ONLINE_SHARED_EXPERT_QUANTIZERS: dict[
|
||||
QuantKey, Callable[[torch.Tensor], tuple[torch.Tensor, torch.Tensor]]
|
||||
] = {
|
||||
kMxfp4Static: mxfp4_quantize,
|
||||
}
|
||||
|
||||
|
||||
def _find_matching_targets(
|
||||
prefix: str,
|
||||
@@ -156,6 +166,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
self.args = args
|
||||
self.ignored_layers: list[str] = args.ignore
|
||||
self.quantized_layers: dict[str, tuple[str, str, str | None]] = {}
|
||||
self.online_quantization_time = 0.0
|
||||
|
||||
@property
|
||||
def quantized_layer_summaries(self) -> list[str]:
|
||||
@@ -216,14 +227,14 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
self,
|
||||
spec: QuantSpec | None,
|
||||
table: dict[QuantKey, type],
|
||||
layer: torch.nn.Module,
|
||||
layer_cls: type[torch.nn.Module],
|
||||
) -> type | None:
|
||||
"""Resolve the online method class for a layer's quantization spec.
|
||||
|
||||
Args:
|
||||
spec: Quantization specification to resolve.
|
||||
table: Mapping from weight quantization keys to method classes.
|
||||
layer: Layer that will use the resolved method.
|
||||
layer_cls: Type of the layer that will use the resolved method.
|
||||
|
||||
Returns:
|
||||
The matching method class, or None when ``spec`` has no weight
|
||||
@@ -238,7 +249,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
cls = table.get(spec.weight)
|
||||
if cls is None:
|
||||
raise ValueError(
|
||||
f"online quantization for {type(layer).__name__} with "
|
||||
f"online quantization for {layer_cls.__name__} with "
|
||||
f"weight={spec.weight} is not supported; supported weight "
|
||||
f"keys: {sorted(str(k) for k in table)}"
|
||||
)
|
||||
@@ -253,12 +264,12 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
return cls
|
||||
|
||||
def resolve_quant_method_cls(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
self, layer_cls: type[torch.nn.Module], prefix: str
|
||||
) -> tuple[OnlineQuantizationSource, str, str | None, QuantSpec, type] | None:
|
||||
"""Resolve quantization metadata and method class without instantiating it.
|
||||
|
||||
Args:
|
||||
layer: Layer for which to resolve online quantization.
|
||||
layer_cls: Type of layer for which to resolve online quantization.
|
||||
prefix: Fully qualified layer name.
|
||||
|
||||
Returns:
|
||||
@@ -270,17 +281,17 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
quant_spec: QuantSpec | None
|
||||
if self.args.targets is not None:
|
||||
resolved_pattern = self._resolve_targets_quant_method_metadata(
|
||||
prefix, layer
|
||||
prefix, layer_cls
|
||||
)
|
||||
if resolved_pattern is None:
|
||||
return None
|
||||
source, quant_key_str, target_pattern, quant_spec, table = resolved_pattern
|
||||
else:
|
||||
if isinstance(layer, LinearBase):
|
||||
if issubclass(layer_cls, LinearBase):
|
||||
source = OnlineQuantizationSource.linear
|
||||
quant_spec = self.args.linear
|
||||
table = _ONLINE_LINEAR_METHODS
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
elif issubclass(layer_cls, RoutedExperts):
|
||||
source = OnlineQuantizationSource.moe
|
||||
quant_spec = self.args.moe
|
||||
table = _ONLINE_MOE_METHODS
|
||||
@@ -297,14 +308,14 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
quant_key_str = str(quant_spec)
|
||||
target_pattern = None
|
||||
|
||||
quant_method_cls = self._get_method_cls(quant_spec, table, layer)
|
||||
quant_method_cls = self._get_method_cls(quant_spec, table, layer_cls)
|
||||
if quant_method_cls is None:
|
||||
return None
|
||||
assert quant_spec is not None
|
||||
return source, quant_key_str, target_pattern, quant_spec, quant_method_cls
|
||||
|
||||
def _resolve_targets_quant_method_metadata(
|
||||
self, prefix: str, layer: torch.nn.Module
|
||||
self, prefix: str, layer_cls: type[torch.nn.Module]
|
||||
) -> (
|
||||
tuple[OnlineQuantizationSource, str, str, QuantSpec, dict[QuantKey, type]]
|
||||
| None
|
||||
@@ -313,7 +324,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
|
||||
Args:
|
||||
prefix: Fully qualified layer name.
|
||||
layer: Layer matched against configured target patterns.
|
||||
layer_cls: Type of layer matched against configured target patterns.
|
||||
|
||||
Returns:
|
||||
A tuple of source, quantization key string, target pattern, spec,
|
||||
@@ -348,22 +359,22 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
target_pattern = matches[0]
|
||||
quant_key_str = self.args.targets[target_pattern]
|
||||
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
|
||||
if isinstance(layer, LinearBase):
|
||||
if issubclass(layer_cls, LinearBase):
|
||||
quant_spec = shorthand.linear
|
||||
table = _ONLINE_LINEAR_METHODS
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
elif issubclass(layer_cls, RoutedExperts):
|
||||
quant_spec = shorthand.moe
|
||||
table = _ONLINE_MOE_METHODS
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Layer {prefix} was matched by quantization_config.targets "
|
||||
f"({target_pattern}), but online quantization is not supported for "
|
||||
f"{type(layer).__name__}."
|
||||
f"{layer_cls.__name__}."
|
||||
)
|
||||
if quant_spec is None:
|
||||
raise ValueError(
|
||||
f"targets pattern {target_pattern} = {quant_key_str} does "
|
||||
f"not define a QuantSpec for {type(layer).__name__} layers "
|
||||
f"not define a QuantSpec for {layer_cls.__name__} layers "
|
||||
f"(matched at {prefix})."
|
||||
)
|
||||
return (
|
||||
@@ -378,7 +389,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
# `targets` takes precedence over `moe` and `linear` and is exclusive.
|
||||
resolved = self.resolve_quant_method_cls(layer, prefix)
|
||||
resolved = self.resolve_quant_method_cls(type(layer), prefix)
|
||||
if resolved is not None:
|
||||
source, quant_key_str, target_pattern, _, quant_method_cls = resolved
|
||||
self.quantized_layers[prefix] = (
|
||||
|
||||
@@ -12,8 +12,6 @@ if TYPE_CHECKING:
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
@@ -34,11 +32,13 @@ from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
amax_for_moe_weight_quant,
|
||||
amax_for_tp_weight_quant,
|
||||
create_fp8_quant_key,
|
||||
@@ -121,10 +121,18 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
|
||||
uses_meta_device: bool = True
|
||||
activation_quant_key: QuantKey | None
|
||||
|
||||
def __init__(self):
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
self.requantization_source: QuantizeMethodBase | None = None
|
||||
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
|
||||
|
||||
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
|
||||
"""Configure serialized-weight conversion before online quantization."""
|
||||
self.requantization_source = source_method
|
||||
self.uses_meta_device = False
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -136,6 +144,24 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
if self.requantization_source is not None:
|
||||
existing_parameter_names = set(layer._parameters)
|
||||
self.requantization_source.create_weights(
|
||||
layer,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
input_size,
|
||||
output_size,
|
||||
params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
self.requantization_source_parameters = {
|
||||
name: parameter
|
||||
for name, parameter in layer._parameters.items()
|
||||
if name not in existing_parameter_names and parameter is not None
|
||||
}
|
||||
return
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
layer.logical_widths = output_partition_sizes
|
||||
@@ -159,11 +185,28 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
|
||||
initialize_online_processing(layer)
|
||||
|
||||
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
|
||||
"""Release checkpoint parameters after successful requantization."""
|
||||
for name, source_parameter in self.requantization_source_parameters.items():
|
||||
if layer._parameters.get(name) is source_parameter:
|
||||
delattr(layer, name)
|
||||
self.requantization_source_parameters.clear()
|
||||
|
||||
def get_weight_for_quantization(self, layer: Module) -> torch.Tensor:
|
||||
"""Return checkpoint weights materialized for online quantization."""
|
||||
if self.requantization_source is None:
|
||||
return layer.weight
|
||||
return self.requantization_source.dequantize_weight(layer)
|
||||
|
||||
|
||||
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
|
||||
activation_quant_key = (
|
||||
kFp8DynamicTokenSym if cutlass_fp8_supported() else kFp8DynamicTensorSym
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
@@ -172,11 +215,6 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
self.use_marlin = False
|
||||
self.marlin_input_dtype = None
|
||||
self.weight_quant_key = kFp8StaticTensorSym
|
||||
# Use per-token quantization for better perf if dynamic and cutlass
|
||||
if cutlass_fp8_supported():
|
||||
self.activation_quant_key = kFp8DynamicTokenSym
|
||||
else:
|
||||
self.activation_quant_key = kFp8DynamicTensorSym
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -198,6 +236,8 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
# TODO: init_fp8_linear_kernel should support activation_quant_key=None
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
@@ -213,10 +253,11 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
return
|
||||
|
||||
layer.input_scale = None
|
||||
amax = weight_amax(layer.weight).reshape(1)
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
amax = weight_amax(weight).reshape(1)
|
||||
amax = amax_for_tp_weight_quant(amax, _is_tp_sharded(layer))
|
||||
weight_scale = _fp8_scale(amax)
|
||||
qweight, _ = ops.scaled_fp8_quant(layer.weight, scale=weight_scale)
|
||||
qweight, _ = ops.scaled_fp8_quant(weight, scale=weight_scale)
|
||||
|
||||
# Update layer with new values.
|
||||
replace_parameter(layer, "weight", qweight.t().data)
|
||||
@@ -265,13 +306,11 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online blockwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
|
||||
activation_quant_key = kFp8Dynamic128Sym
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight_block_size = [128, 128]
|
||||
self.activation_quant_key = create_fp8_quant_key(
|
||||
static=False,
|
||||
group_shape=GroupShape(1, self.weight_block_size[0]),
|
||||
)
|
||||
self.weight_quant_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(*self.weight_block_size)
|
||||
)
|
||||
@@ -297,6 +336,8 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
)
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
# TODO: init_fp8_linear_kernel should support activation_quant_key=None
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
@@ -312,14 +353,18 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
|
||||
layer.input_scale = None
|
||||
block_size = self.weight_block_size
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
|
||||
qweight, weight_scale_inv = per_block_cast_to_fp8(
|
||||
layer.weight, block_size=block_size, use_ue8m0=False
|
||||
weight, block_size=block_size, use_ue8m0=False
|
||||
)
|
||||
|
||||
replace_parameter(layer, "weight", qweight.data)
|
||||
replace_parameter(layer, "weight_scale_inv", weight_scale_inv.data)
|
||||
|
||||
if self.requantization_source is not None and hasattr(layer, "weight_scale"):
|
||||
del layer.weight_scale
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
@@ -395,18 +440,20 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
return
|
||||
|
||||
layer.input_scale = None
|
||||
amax = weight_amax(layer.weight, dim=-1, keepdim=True)
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
amax = weight_amax(weight, dim=-1, keepdim=True)
|
||||
amax = amax_for_tp_weight_quant(
|
||||
amax, _is_tp_sharded(layer, reduces_output_dim=False)
|
||||
)
|
||||
weight_scale = _fp8_channel_scale(amax)
|
||||
qweight = _fp8_quant_per_channel(layer.weight, weight_scale)
|
||||
qweight = _fp8_quant_per_channel(weight, weight_scale)
|
||||
|
||||
replace_parameter(layer, "weight", qweight.t())
|
||||
replace_parameter(layer, "weight_scale", weight_scale)
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
expose_input_quant_key(layer, self.fp8_linear)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader.reload.layerwise import (
|
||||
initialize_online_processing,
|
||||
)
|
||||
@@ -24,6 +25,12 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
|
||||
"""Reject requantization from a checkpoint-quantized MoE method."""
|
||||
raise NotImplementedError(
|
||||
"Requantizing checkpoint-quantized MoE layers is not supported."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
@@ -119,7 +119,8 @@ class Mxfp4OnlineLinearMethod(OnlineLinearBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
weight_fp4, weight_scale = mxfp4_quantize(layer.weight.contiguous())
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
weight_fp4, weight_scale = mxfp4_quantize(weight.contiguous())
|
||||
|
||||
layer.input_scale = None
|
||||
replace_parameter(layer, "weight", weight_fp4.data)
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
from torch.nn import Module
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
@@ -42,6 +43,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
FP8 with block-32 scales) during weight loading.
|
||||
"""
|
||||
|
||||
activation_quant_key: QuantKey | None = None
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -75,7 +78,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
weight_fp8, weight_scale = mxfp8_e4m3_quantize(layer.weight.contiguous())
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
weight_fp8, weight_scale = mxfp8_e4m3_quantize(weight.contiguous())
|
||||
|
||||
layer.input_scale = None
|
||||
replace_parameter(layer, "weight", weight_fp8.data)
|
||||
|
||||
@@ -19,6 +19,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
trtllm_nvfp4_hidden_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
)
|
||||
@@ -87,6 +90,8 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
(SM100) only.
|
||||
"""
|
||||
|
||||
activation_quant_key = kNvfp4DynamicToken
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -100,9 +105,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kNvfp4Static,
|
||||
activation_key=kNvfp4DynamicToken,
|
||||
activation_key=self.activation_quant_key,
|
||||
)
|
||||
|
||||
@property
|
||||
def per_token_activation(self) -> bool:
|
||||
return self.activation_quant_key == kNvfp4DynamicToken
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
@@ -156,6 +165,10 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
w2_scale_2=layer.w2_weight_scale_2,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
trtllm_hidden_alignment=trtllm_nvfp4_hidden_alignment(
|
||||
per_token_activation=self.per_token_activation,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
),
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
@@ -176,7 +189,7 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
per_token_activation=True,
|
||||
per_token_activation=self.per_token_activation,
|
||||
)
|
||||
else:
|
||||
# Reload creates new scale tensors; derived kernel scales must use
|
||||
|
||||
@@ -8,9 +8,13 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_quark_ocp_mx_group_size(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
@@ -70,18 +74,100 @@ def is_shared_expert_quant_fse_compatible(
|
||||
A compatibility flag and, when incompatible, the reason.
|
||||
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
|
||||
|
||||
if projection_names is None:
|
||||
projection_names = ["gate_up_proj", "down_proj"]
|
||||
|
||||
if quant_config is None:
|
||||
return True, None
|
||||
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
|
||||
online_quant_config = quant_config.online_quantization_config
|
||||
shared_expert_targets = None
|
||||
if online_quant_config is not None:
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
|
||||
online_quant_config.packed_modules_mapping = quant_config.packed_modules_mapping
|
||||
shared_expert_targets = [
|
||||
online_quant_config.resolve_quant_method_cls(
|
||||
LinearBase, f"{shared_expert_prefix}.{projection_name}"
|
||||
)
|
||||
for projection_name in projection_names
|
||||
]
|
||||
if all(target is None for target in shared_expert_targets):
|
||||
shared_expert_targets = None
|
||||
|
||||
if shared_expert_targets is not None:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
ONLINE_SHARED_EXPERT_QUANTIZERS,
|
||||
)
|
||||
|
||||
# NOTE: online shared experts quantization check is only implemented for quark
|
||||
# quant method at the moment. This can be extended here for other quant methods.
|
||||
# Requirement is:
|
||||
# - Retrieve routed weight quant key
|
||||
# - Retrieve routed activation quant key
|
||||
# and ensure the online quantization set on shared expert match them.
|
||||
if isinstance(quant_config, QuarkConfig):
|
||||
routed_weight_key, routed_activation_key, _ = (
|
||||
quant_config.get_quant_method_target(expert_prefix, RoutedExperts)
|
||||
)
|
||||
else:
|
||||
reason = (
|
||||
"online quantization FSE compatibility check is not implemented for "
|
||||
f"{type(quant_config).__name__}"
|
||||
)
|
||||
logger.warning_once(reason)
|
||||
return False, reason
|
||||
|
||||
if routed_weight_key not in ONLINE_SHARED_EXPERT_QUANTIZERS:
|
||||
return (
|
||||
False,
|
||||
"online shared-expert quantization is not supported for routed "
|
||||
f"expert weight key {routed_weight_key}",
|
||||
)
|
||||
|
||||
for shared_expert_target in shared_expert_targets:
|
||||
if shared_expert_target is None:
|
||||
return (
|
||||
False,
|
||||
"online quantization targets only part of the shared expert at "
|
||||
f"{shared_expert_prefix}",
|
||||
)
|
||||
|
||||
_, _, _, shared_quant_spec, shared_method_cls = shared_expert_target
|
||||
if shared_method_cls in (None, UnquantizedLinearMethod):
|
||||
return (
|
||||
False,
|
||||
"online quantization excludes FSE weights at "
|
||||
f"{shared_expert_prefix}",
|
||||
)
|
||||
assert issubclass(shared_method_cls, OnlineLinearBase)
|
||||
|
||||
# NOTE: We can not rely on shared_quant_spec.activation_quant_key
|
||||
# as _ONLINE_SHORTHANDS: dict[str, QuantSpec] in quantization.py
|
||||
# does not define the default activation quant key.
|
||||
# `online_quant_config.resolve_quant_method_cls` should return the
|
||||
# activation quant key as well. Only some online methods define it
|
||||
# on the class, so compare the weight key first.
|
||||
if (
|
||||
shared_quant_spec.weight != routed_weight_key
|
||||
or shared_method_cls.activation_quant_key != routed_activation_key
|
||||
):
|
||||
return (
|
||||
False,
|
||||
"online shared-expert quantization keys do not match the "
|
||||
"routed expert keys",
|
||||
)
|
||||
return True, None
|
||||
|
||||
if isinstance(quant_config, DeepseekV4FP8Config):
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
@@ -344,6 +344,7 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
w2_scale_2: torch.Tensor,
|
||||
a2_scale: torch.Tensor,
|
||||
is_act_and_mul: bool,
|
||||
trtllm_hidden_alignment: int = 256,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
@@ -398,7 +399,11 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
|
||||
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
|
||||
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
|
||||
w13, w13_scale, w2, w2_scale
|
||||
w13,
|
||||
w13_scale,
|
||||
w2,
|
||||
w2_scale,
|
||||
min_alignment=trtllm_hidden_alignment,
|
||||
)
|
||||
if layer.moe_config.hidden_dim_unpadded is None:
|
||||
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
|
||||
|
||||
@@ -280,6 +280,20 @@ def align_fp4_moe_weights_for_fi(
|
||||
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
|
||||
|
||||
|
||||
def trtllm_nvfp4_hidden_alignment(
|
||||
per_token_activation: bool, is_act_and_mul: bool
|
||||
) -> int:
|
||||
"""Hidden-dim alignment for the TRTLLM-Gen NVFP4 MoE cubins.
|
||||
|
||||
The per-token (dynamic activation scale) non-gated variant has far fewer
|
||||
tile configs for 256-aligned K and its heuristic default tactic fails with
|
||||
"No valid config found" (e.g. Nemotron's 2688 -> 2816), so pad to 512.
|
||||
"""
|
||||
if per_token_activation and not is_act_and_mul:
|
||||
return 512
|
||||
return 256
|
||||
|
||||
|
||||
def align_fp4_moe_hidden_dim_for_fi(
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
|
||||
@@ -142,6 +142,7 @@ def _silu_mul_quant_fp8_packed_kernel(
|
||||
input_ptr,
|
||||
output_q_ptr,
|
||||
output_scale_ptr,
|
||||
expert_ends_ptr,
|
||||
M,
|
||||
input_stride_m,
|
||||
output_q_stride_m,
|
||||
@@ -158,6 +159,7 @@ def _silu_mul_quant_fp8_packed_kernel(
|
||||
PACKS_PER_CTA: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
HAS_CLAMP: tl.constexpr,
|
||||
EXPERT_ALIGNMENT: tl.constexpr,
|
||||
):
|
||||
GROUPS_PER_PACK: tl.constexpr = 4
|
||||
hidden_size: tl.constexpr = N // 2
|
||||
@@ -166,6 +168,13 @@ def _silu_mul_quant_fp8_packed_kernel(
|
||||
row_start = tl.program_id(1).to(tl.int64) * BLOCK_M
|
||||
row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M
|
||||
|
||||
if EXPERT_ALIGNMENT:
|
||||
expert = tl.program_id(2)
|
||||
previous_end = tl.load(expert_ends_ptr + expert - 1, expert > 0, other=0)
|
||||
expert_start = tl.cdiv(previous_end, EXPERT_ALIGNMENT) * EXPERT_ALIGNMENT
|
||||
row_start += expert_start.to(tl.int64)
|
||||
M = tl.load(expert_ends_ptr + expert)
|
||||
|
||||
groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK
|
||||
elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE
|
||||
col_start = pack_tile * elems_per_cta
|
||||
@@ -254,7 +263,19 @@ def silu_mul_quant_fp8_packed_triton(
|
||||
clamp_limit: float | None = None,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
*,
|
||||
expert_ends: torch.Tensor | None = None,
|
||||
expert_alignment: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fuse activation and FP8 quantization, optionally skipping expert padding.
|
||||
|
||||
``expert_ends`` uses DeepGEMM's prefix-sum layout: each entry is the
|
||||
exclusive end of an expert's live rows, including earlier alignment gaps.
|
||||
An expert starts at the preceding end rounded up to ``expert_alignment``.
|
||||
Padding outputs are left unwritten and must not be consumed as live rows.
|
||||
Without ``expert_ends``, the original dense launch configuration is used
|
||||
and the metadata loads are compiled out.
|
||||
"""
|
||||
assert input.dim() == 2
|
||||
assert input.is_contiguous()
|
||||
|
||||
@@ -293,13 +314,29 @@ def silu_mul_quant_fp8_packed_triton(
|
||||
|
||||
grid_n = triton.cdiv(packs_per_row, packs_per_cta)
|
||||
grid_m = min(triton.cdiv(M, BM), 4096)
|
||||
grid = (grid_n, grid_m)
|
||||
grid: tuple[int, ...] = (grid_n, grid_m)
|
||||
if expert_ends is not None:
|
||||
assert expert_ends.ndim == 1 and expert_ends.numel() > 0
|
||||
assert expert_ends.dtype == torch.int32
|
||||
assert expert_ends.device == input.device and expert_ends.is_contiguous()
|
||||
assert expert_alignment > 0
|
||||
num_experts = expert_ends.numel()
|
||||
# Bound empty CTAs while retaining enough parallelism for full experts.
|
||||
max_ctas = 4096 if group_size < 128 else 8192
|
||||
grid_m = min(
|
||||
triton.cdiv(M, BM * num_experts),
|
||||
max(1, triton.cdiv(max_ctas, grid_n * num_experts)),
|
||||
)
|
||||
grid = (grid_n, grid_m, num_experts)
|
||||
else:
|
||||
assert expert_alignment == 0
|
||||
|
||||
has_clamp = clamp_limit is not None
|
||||
_silu_mul_quant_fp8_packed_kernel[grid](
|
||||
input,
|
||||
output_q,
|
||||
output_scale_packed,
|
||||
expert_ends,
|
||||
M,
|
||||
input.stride(0),
|
||||
output_q.stride(0),
|
||||
@@ -316,6 +353,7 @@ def silu_mul_quant_fp8_packed_triton(
|
||||
PACKS_PER_CTA=packs_per_cta,
|
||||
BLOCK_M=BM,
|
||||
HAS_CLAMP=has_clamp,
|
||||
EXPERT_ALIGNMENT=expert_alignment,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
@@ -328,6 +366,7 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
y_ptr, # [M, N]
|
||||
y_q_ptr, # [M, N // 2]
|
||||
y_s_ptr, # [M, (N // 2) // GROUP_SIZE]
|
||||
expert_ends_ptr,
|
||||
M, # num tokens
|
||||
N, # intermediate size
|
||||
# Stride
|
||||
@@ -345,8 +384,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
GROUP_SIZE: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
EXPERT_ALIGNMENT: tl.constexpr,
|
||||
):
|
||||
# TODO(varun) : Add expert_ids so we may early-exit no-op thread blocks.
|
||||
"""Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
|
||||
the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills
|
||||
the outputs tensors at the right positions.
|
||||
@@ -357,56 +396,77 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
|
||||
m_offset = pid_m.to(tl.int64) * BLOCK_M
|
||||
n_offset = pid_n.to(tl.int64) * BLOCK_N
|
||||
if m_offset >= M:
|
||||
return
|
||||
row_step = tl.num_programs(0).to(tl.int64) * BLOCK_M
|
||||
num_iterations = 1
|
||||
if EXPERT_ALIGNMENT:
|
||||
expert = tl.program_id(2)
|
||||
previous_end = tl.load(expert_ends_ptr + expert - 1, expert > 0, other=0)
|
||||
expert_start = tl.cdiv(previous_end, EXPERT_ALIGNMENT) * EXPERT_ALIGNMENT
|
||||
m_offset += expert_start.to(tl.int64)
|
||||
M = tl.load(expert_ends_ptr + expert)
|
||||
num_iterations = tl.cdiv(tl.maximum(M - m_offset, 0), row_step)
|
||||
|
||||
offs_n = tl.arange(0, BLOCK_N).to(tl.int64)
|
||||
offs_m = tl.arange(0, BLOCK_M).to(tl.int64)
|
||||
|
||||
base_y_ptr = y_ptr + m_offset * N + n_offset
|
||||
for _ in range(num_iterations):
|
||||
base_y_ptr = y_ptr + m_offset * N + n_offset
|
||||
|
||||
act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :]
|
||||
act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :]
|
||||
|
||||
act_in = tl.load(act_in_ptrs)
|
||||
mul_in = tl.load(act_in_ptrs + N_2)
|
||||
if EXPERT_ALIGNMENT:
|
||||
live = m_offset + offs_m < M
|
||||
act_in = tl.load(act_in_ptrs, live[:, None], other=0)
|
||||
mul_in = tl.load(act_in_ptrs + N_2, live[:, None], other=0)
|
||||
else:
|
||||
act_in = tl.load(act_in_ptrs)
|
||||
mul_in = tl.load(act_in_ptrs + N_2)
|
||||
|
||||
# silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the
|
||||
# input dtype, run silu in fp32 then narrow, and do the mul at input
|
||||
# precision so HAS_CLAMP True/False share the same multiplication path.
|
||||
if HAS_CLAMP:
|
||||
act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to(
|
||||
y_ptr.dtype.element_ty
|
||||
)
|
||||
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
|
||||
y_ptr.dtype.element_ty
|
||||
)
|
||||
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
|
||||
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
|
||||
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
|
||||
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
|
||||
act_in = act_in.to(tl.float32)
|
||||
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
|
||||
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
|
||||
y = (glu * up).to(tl.float32)
|
||||
# silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the
|
||||
# input dtype, run silu in fp32 then narrow, and do the mul at input
|
||||
# precision so HAS_CLAMP True/False share the same multiplication path.
|
||||
if HAS_CLAMP:
|
||||
act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to(
|
||||
y_ptr.dtype.element_ty
|
||||
)
|
||||
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
|
||||
y_ptr.dtype.element_ty
|
||||
)
|
||||
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
|
||||
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
|
||||
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
|
||||
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
|
||||
act_in = act_in.to(tl.float32)
|
||||
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
|
||||
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
|
||||
y = (glu * up).to(tl.float32)
|
||||
|
||||
# quant
|
||||
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
|
||||
scale_raw = _absmax * (1.0 / fp8_max)
|
||||
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
|
||||
y_s = tl.reshape(y_s, (BLOCK_M, 1))
|
||||
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
|
||||
# quant
|
||||
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
|
||||
scale_raw = _absmax * (1.0 / fp8_max)
|
||||
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
|
||||
y_s = tl.reshape(y_s, (BLOCK_M, 1))
|
||||
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
|
||||
|
||||
# store y_q
|
||||
base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset
|
||||
y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :]
|
||||
tl.store(y_q_ptrs, y_q)
|
||||
# store y_q
|
||||
base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset
|
||||
y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :]
|
||||
if EXPERT_ALIGNMENT:
|
||||
tl.store(y_q_ptrs, y_q, live[:, None])
|
||||
else:
|
||||
tl.store(y_q_ptrs, y_q)
|
||||
|
||||
# store y_s
|
||||
group_id = n_offset // GROUP_SIZE
|
||||
base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset
|
||||
y_s_ptrs = base_y_s_ptr + offs_m
|
||||
y_s = tl.reshape(y_s, (BLOCK_M,))
|
||||
tl.store(y_s_ptrs, y_s)
|
||||
# store y_s
|
||||
group_id = n_offset // GROUP_SIZE
|
||||
base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset
|
||||
y_s_ptrs = base_y_s_ptr + offs_m
|
||||
y_s = tl.reshape(y_s, (BLOCK_M,))
|
||||
if EXPERT_ALIGNMENT:
|
||||
tl.store(y_s_ptrs, y_s, live)
|
||||
else:
|
||||
tl.store(y_s_ptrs, y_s)
|
||||
|
||||
m_offset += row_step
|
||||
|
||||
|
||||
def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
@@ -418,9 +478,15 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
group_size: int = 128,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
*,
|
||||
expert_ends: torch.Tensor | None = None,
|
||||
expert_alignment: int = 0,
|
||||
):
|
||||
"""Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
|
||||
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
|
||||
|
||||
Optional expert endpoints follow DeepGEMM's prefix-sum layout. Padding
|
||||
values and scales are left unwritten and must not be consumed as live rows.
|
||||
"""
|
||||
GROUP_SIZE = group_size
|
||||
assert input.ndim == 2
|
||||
@@ -458,13 +524,27 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
# Force even division so we can avoid edgecases within the kernel.
|
||||
assert M % BLOCK_M == 0
|
||||
assert N_2 % BLOCK_N == 0
|
||||
grid = (M // BLOCK_M, N_2 // BLOCK_N)
|
||||
grid: tuple[int, ...] = (M // BLOCK_M, N_2 // BLOCK_N)
|
||||
if expert_ends is not None:
|
||||
assert expert_ends.ndim == 1 and expert_ends.numel() > 0
|
||||
assert expert_ends.dtype == torch.int32
|
||||
assert expert_ends.device == input.device and expert_ends.is_contiguous()
|
||||
assert expert_alignment > 0 and expert_alignment % BLOCK_M == 0
|
||||
num_experts = expert_ends.numel()
|
||||
grid_m = min(
|
||||
triton.cdiv(M, BLOCK_M * num_experts),
|
||||
max(1, triton.cdiv(8192, grid[1] * num_experts)),
|
||||
)
|
||||
grid = (grid_m, grid[1], num_experts)
|
||||
else:
|
||||
assert expert_alignment == 0
|
||||
|
||||
has_clamp = clamp_limit is not None
|
||||
_silu_mul_per_token_group_quant_fp8_colmajor[grid](
|
||||
input,
|
||||
output,
|
||||
output_scales,
|
||||
expert_ends,
|
||||
M,
|
||||
N,
|
||||
output_scales.stride(-1),
|
||||
@@ -479,6 +559,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
GROUP_SIZE,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
expert_alignment,
|
||||
)
|
||||
|
||||
return output, output_scales
|
||||
|
||||
@@ -133,6 +133,11 @@ def is_layer_gptq_quantized(
|
||||
|
||||
quantized_layers = flatten_list(quantized_layers)
|
||||
|
||||
# Empty means no metadata could be read at all, not that the checkpoint
|
||||
# holds no quantized layer, and loading GPTQ weights unquantized cannot work.
|
||||
if not quantized_layers:
|
||||
return True
|
||||
|
||||
# Fused layers like gate_up_proj or qkv_proj will not be fused
|
||||
# in the safetensors checkpoint. So, we convert the name
|
||||
# from the fused version to unfused + check to make sure that
|
||||
|
||||
@@ -19,6 +19,10 @@ from vllm.model_executor.layers.quantization.utils.humming.moe import (
|
||||
make_humming_moe_quant_config,
|
||||
select_humming_moe_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming.priority import (
|
||||
prefers_humming,
|
||||
prioritize_humming,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming.schema import (
|
||||
check_and_fallback_input_schema,
|
||||
humming_is_layer_skipped,
|
||||
@@ -41,6 +45,8 @@ __all__ = [
|
||||
"convert_linear_layer_to_humming_standard",
|
||||
"get_humming_linear_compute_config",
|
||||
"prepare_humming_linear_layer_config",
|
||||
"prefers_humming",
|
||||
"prioritize_humming",
|
||||
"make_humming_moe_kernel",
|
||||
"select_humming_moe_experts",
|
||||
"input_schema_to_quant_key",
|
||||
|
||||
@@ -87,7 +87,7 @@ def prepare_humming_linear_layer_config(
|
||||
shape_n_stacks = layer.output_partition_sizes
|
||||
|
||||
# Step 1: convert weight and input schemas to humming standard format
|
||||
source_tensors = dict(layer.named_parameters())
|
||||
source_tensors = dict(layer.named_parameters(recurse=False))
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=source_tensors,
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
@@ -143,7 +143,7 @@ def prepare_humming_linear_layer_config(
|
||||
if "bias" in tensors:
|
||||
zero_bias = torch.zeros_like(tensors["bias"])
|
||||
layer.register_buffer("zero_bias", zero_bias)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
for name, _ in list(layer.named_parameters(recurse=False)):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Humming kernel selection priority."""
|
||||
|
||||
from enum import Enum
|
||||
from typing import TypeVar
|
||||
|
||||
_KernelT = TypeVar("_KernelT", bound=type | Enum)
|
||||
|
||||
# Compute capabilities where Humming outranks Marlin.
|
||||
_HUMMING_PREFERRED_CAPABILITIES = frozenset({90})
|
||||
|
||||
|
||||
def prefers_humming(compute_capability: int | None = None) -> bool:
|
||||
"""Whether Humming outranks Marlin. A missing compute capability uses the
|
||||
current device."""
|
||||
if compute_capability is None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda():
|
||||
cc = current_platform.get_device_capability()
|
||||
compute_capability = cc.to_int() if cc is not None else None
|
||||
return compute_capability in _HUMMING_PREFERRED_CAPABILITIES
|
||||
|
||||
|
||||
def prioritize_humming(
|
||||
kernels: list[_KernelT],
|
||||
compute_capability: int | None = None,
|
||||
) -> list[_KernelT]:
|
||||
"""Move Humming directly ahead of Marlin where Humming is preferred.
|
||||
|
||||
Every other entry keeps its relative order and the input list is not
|
||||
modified. Match kernel class names or MoE backend enum names.
|
||||
"""
|
||||
if not prefers_humming(compute_capability):
|
||||
return kernels
|
||||
|
||||
names = [
|
||||
(kernel.name if isinstance(kernel, Enum) else kernel.__name__).lower()
|
||||
for kernel in kernels
|
||||
]
|
||||
humming = next((i for i, name in enumerate(names) if "humming" in name), None)
|
||||
marlin = next((i for i, name in enumerate(names) if "marlin" in name), None)
|
||||
if humming is None or marlin is None or humming < marlin:
|
||||
return kernels
|
||||
kernels = kernels.copy()
|
||||
kernels.insert(marlin, kernels.pop(humming))
|
||||
return kernels
|
||||
@@ -322,6 +322,22 @@ def check_and_fallback_input_schema(
|
||||
input_group_size = input_schema.input_scale_group_size
|
||||
input_scale_dtype = input_schema.input_scale_dtype
|
||||
|
||||
def is_compatible(candidate: "HummingInputSchema") -> bool:
|
||||
check_weight_schema = weight_schema
|
||||
if (
|
||||
candidate.a_dtype in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
|
||||
and weight_schema.b_dtype == humming_dtypes.float4e2m1
|
||||
and weight_group_size == 16
|
||||
and weight_schema.bs_dtype
|
||||
in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
|
||||
):
|
||||
# Humming rejects folded NVFP4 scales despite A16 software dequant
|
||||
# supporting them. Relax only this check, preserving kernel metadata.
|
||||
check_weight_schema = dataclasses.replace(
|
||||
weight_schema, bs_dtype=humming_dtypes.float8e4m3
|
||||
)
|
||||
return candidate.is_compatible_with(check_weight_schema, param_dtype)
|
||||
|
||||
def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
|
||||
if dtype is None or dtype.num_bits == 16:
|
||||
return False
|
||||
@@ -333,7 +349,7 @@ def check_and_fallback_input_schema(
|
||||
is_fp8_deprecated = dtype.num_bits < 16 and 100 <= sm_version < 120
|
||||
return is_int4_deprecated or is_int8_deprecated or is_fp8_deprecated
|
||||
|
||||
if input_schema.is_compatible_with(weight_schema, param_dtype):
|
||||
if is_compatible(input_schema):
|
||||
if not allow_fallback:
|
||||
if is_deprecated(a_dtype):
|
||||
msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
|
||||
@@ -390,7 +406,7 @@ def check_and_fallback_input_schema(
|
||||
input_scale_dtype=scale_dtype,
|
||||
input_quant_mode=quant_mode,
|
||||
)
|
||||
if candidate.is_compatible_with(weight_schema, param_dtype):
|
||||
if is_compatible(candidate):
|
||||
return candidate
|
||||
|
||||
raise ValueError(
|
||||
|
||||
@@ -120,7 +120,14 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
|
||||
K = scale.shape[-1]
|
||||
pad_k = -K % 4
|
||||
scale = torch.nn.functional.pad(scale, (0, pad_k))
|
||||
elif current_platform.is_device_capability_family(100):
|
||||
elif current_platform.is_device_capability_family(
|
||||
100
|
||||
) or current_platform.is_device_capability_family(120):
|
||||
# Native MXFP (cap >= 10.0) requires the persistent kernel. On
|
||||
# SM12x, epilogue_subtile=1 also keeps the heuristic away from a
|
||||
# triton_kernels 3.5.1 codegen bug (SWAP_XW with a subtiled
|
||||
# epilogue); block_m is pinned per call in
|
||||
# gpt_oss_triton_kernels_moe to fit the 99KB shared memory.
|
||||
constraints = {
|
||||
"is_persistent": True,
|
||||
"epilogue_subtile": 1,
|
||||
|
||||
@@ -128,16 +128,13 @@ def _can_use_flashinfer_cutedsl_bf16(
|
||||
if (
|
||||
k <= 0
|
||||
or n <= 0
|
||||
or x.numel() == 0
|
||||
or weight.shape[1] != k
|
||||
or k % 128 != 0
|
||||
or x.data_ptr() % 32 != 0
|
||||
or weight.data_ptr() % 32 != 0
|
||||
):
|
||||
return False
|
||||
|
||||
m = x.numel() // k
|
||||
if not 1 <= m <= 32:
|
||||
return False
|
||||
return bias is None or (
|
||||
bias.is_cuda
|
||||
and bias.device == x.device
|
||||
|
||||
Reference in New Issue
Block a user