[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,855 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_test_weights, per_token_cast_to_fp8
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
dequantize_nvfp4_to_dtype,
|
||||
)
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_dp_group,
|
||||
get_pcp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.utils.import_utils import (
|
||||
has_aiter,
|
||||
has_deep_ep,
|
||||
has_deep_ep_v2,
|
||||
has_deep_gemm,
|
||||
has_mori,
|
||||
)
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
|
||||
from .mk_objects import (
|
||||
TestMoEQuantConfig,
|
||||
expert_info,
|
||||
make_fused_experts,
|
||||
prepare_finalize_info,
|
||||
)
|
||||
from .parallel_utils import ProcessGroupInfo
|
||||
|
||||
|
||||
def _describe_tensor(t: torch.Tensor | None, name: str) -> str:
|
||||
if t is None:
|
||||
return f"{name} : None"
|
||||
else:
|
||||
return f"{name} : {t.shape} {t.dtype} {t.device}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
Ms: list[int] | int
|
||||
K: int
|
||||
N: int
|
||||
E: int
|
||||
topks: list[int] | int
|
||||
dtype: torch.dtype
|
||||
quant_config: TestMoEQuantConfig | None
|
||||
|
||||
prepare_finalize_type: mk.FusedMoEPrepareAndFinalize
|
||||
fused_experts_type: mk.FusedMoEExperts
|
||||
|
||||
world_size: int
|
||||
|
||||
activation: MoEActivation = MoEActivation.SILU
|
||||
|
||||
torch_trace_dir_path: str | None = None
|
||||
|
||||
# Force AiterExperts's hidden_pad/intermediate_pad computation
|
||||
# (`experts/rocm_aiter_moe.py`) to diverge from the padded K/N sizes above.
|
||||
# None (default) preserves today's behavior: FusedMoEConfig defaults both
|
||||
# to the (unpadded) K/intermediate_size_per_partition, so hidden_pad and
|
||||
# intermediate_pad come out to 0.
|
||||
# See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
|
||||
hidden_dim_unpadded: int | None = None
|
||||
intermediate_size_per_partition_unpadded: int | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.quant_config is None:
|
||||
self.quant_config = TestMoEQuantConfig(None, False, False, None)
|
||||
|
||||
def describe(self) -> str:
|
||||
s = ""
|
||||
s += "== Config:\n"
|
||||
s += f" world_size={self.world_size}\n"
|
||||
s += f" PF={self.prepare_finalize_type.__name__}\n"
|
||||
s += f" FE={self.fused_experts_type.__name__}\n"
|
||||
s += f" E={self.E}\n"
|
||||
s += f" Ms={self.Ms}\n"
|
||||
s += f" N={self.N}\n"
|
||||
s += f" K={self.K}\n"
|
||||
s += f" topk={self.topks}\n"
|
||||
s += f" dtype={self.dtype}\n"
|
||||
s += " Quant:\n"
|
||||
if self.quant_config is not None:
|
||||
s += f" q_dtype={self.quant_dtype}\n"
|
||||
s += f" q_block_shape={self.quant_block_shape}\n"
|
||||
s += f" q_per_out_ch_quant={self.is_per_out_ch_quant}\n"
|
||||
s += f" q_per_act_token={self.is_per_act_token_quant}\n"
|
||||
else:
|
||||
s += " quant=None\n"
|
||||
return s
|
||||
|
||||
@property
|
||||
def M(self) -> int:
|
||||
assert isinstance(self.Ms, int)
|
||||
return self.Ms
|
||||
|
||||
@property
|
||||
def quant_dtype(self) -> torch.dtype | str | None:
|
||||
assert self.quant_config is not None
|
||||
return self.quant_config.quant_dtype
|
||||
|
||||
@property
|
||||
def is_per_act_token_quant(self) -> bool:
|
||||
assert self.quant_config is not None
|
||||
return self.quant_config.per_act_token_quant
|
||||
|
||||
@property
|
||||
def is_per_tensor_act_quant(self) -> bool:
|
||||
return not self.is_per_act_token_quant and self.quant_block_shape is None
|
||||
|
||||
@property
|
||||
def is_per_out_ch_quant(self) -> bool:
|
||||
assert self.quant_config is not None
|
||||
return self.quant_config.per_out_ch_quant
|
||||
|
||||
@property
|
||||
def quant_block_shape(self) -> list[int] | None:
|
||||
assert self.quant_config is not None
|
||||
return self.quant_config.block_shape
|
||||
|
||||
@property
|
||||
def topk(self) -> int:
|
||||
assert isinstance(self.topks, int)
|
||||
return self.topks
|
||||
|
||||
@property
|
||||
def num_local_experts(self) -> int:
|
||||
return self.E // self.world_size
|
||||
|
||||
def make_env_data(self) -> tuple[VllmConfig, dict[Any, Any]]:
|
||||
"""Make env data for vllm launch."""
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
enforce_eager=True,
|
||||
is_moe=True,
|
||||
enable_sleep_mode=False,
|
||||
sleep_mode_backend="cumem",
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
vllm_config.parallel_config.data_parallel_size = self.world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
|
||||
env_dict = {
|
||||
"VLLM_USE_DEEP_GEMM": str(int(self.needs_deep_gemm())),
|
||||
}
|
||||
|
||||
vllm_config.parallel_config.all2all_backend = self.all2all_backend()
|
||||
|
||||
return vllm_config, env_dict
|
||||
|
||||
def fp8_quant_key_pair(self) -> tuple[QuantKey, QuantKey]:
|
||||
"""Derive the (weight_quant_key, activation_quant_key) pair an FP8
|
||||
quant config of this shape corresponds to (either OCP or FNUZ FP8,
|
||||
see ``current_platform.fp8_dtype()``)."""
|
||||
if self.quant_block_shape is not None:
|
||||
return kFp8Static128BlockSym, kFp8Dynamic128Sym
|
||||
if self.is_per_out_ch_quant:
|
||||
return (
|
||||
kFp8StaticChannelSym,
|
||||
kFp8DynamicTokenSym
|
||||
if self.is_per_act_token_quant
|
||||
else kFp8StaticTensorSym,
|
||||
)
|
||||
return (
|
||||
kFp8StaticTensorSym,
|
||||
kFp8DynamicTensorSym
|
||||
if self.is_per_act_token_quant
|
||||
else kFp8StaticTensorSym,
|
||||
)
|
||||
|
||||
def fe_supports_quant_scheme(self) -> bool:
|
||||
"""Check if the fused experts class supports this quant config.
|
||||
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
|
||||
if self.quant_config is None or self.quant_dtype is None:
|
||||
return True
|
||||
if not is_fp8(self.quant_dtype):
|
||||
return True
|
||||
w_key, a_key = self.fp8_quant_key_pair()
|
||||
fe_cls = self.fused_experts_type
|
||||
if hasattr(fe_cls, "_supports_quant_scheme"):
|
||||
try:
|
||||
return fe_cls._supports_quant_scheme(w_key, a_key)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
return True
|
||||
|
||||
def is_fp8_block_quantized(self):
|
||||
return is_fp8(self.quant_dtype) and self.quant_block_shape is not None
|
||||
|
||||
def is_batched_prepare_finalize(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
return mk.FusedMoEActivationFormat.BatchedExperts == info.activation_format
|
||||
|
||||
def is_batched_fused_experts(self):
|
||||
info = expert_info(self.fused_experts_type)
|
||||
return mk.FusedMoEActivationFormat.BatchedExperts == info.activation_format
|
||||
|
||||
def is_standard_fused_experts(self):
|
||||
info = expert_info(self.fused_experts_type)
|
||||
return mk.FusedMoEActivationFormat.Standard == info.activation_format
|
||||
|
||||
def fe_supported_types(self):
|
||||
info = expert_info(self.fused_experts_type)
|
||||
return info.supported_dtypes
|
||||
|
||||
def pf_supported_types(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
return info.supported_dtypes
|
||||
|
||||
def is_block_quant_supported(self):
|
||||
info = expert_info(self.fused_experts_type)
|
||||
return info.blocked_quantization_support
|
||||
|
||||
def supports_apply_weight_on_input(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
return info.supports_apply_weight_on_input
|
||||
|
||||
def needs_deep_gemm(self):
|
||||
info = expert_info(self.fused_experts_type)
|
||||
return info.needs_deep_gemm
|
||||
|
||||
def needs_deep_ep(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
return (
|
||||
info.backend == "deepep_high_throughput"
|
||||
or info.backend == "deepep_low_latency"
|
||||
)
|
||||
|
||||
def needs_deep_ep_v2(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
return info.backend == "deepep_v2"
|
||||
|
||||
def needs_aiter(self):
|
||||
info = expert_info(self.fused_experts_type)
|
||||
return info.needs_aiter
|
||||
|
||||
def needs_mori(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
return info.backend in ("mori_high_throughput", "mori_low_latency")
|
||||
|
||||
def all2all_backend(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
return info.backend
|
||||
|
||||
def is_valid(self) -> tuple[bool, str | None]:
|
||||
# Check prepare-finalize and fused-experts compatibility
|
||||
if self.is_batched_prepare_finalize():
|
||||
if not self.is_batched_fused_experts():
|
||||
return False, "Mismatched format."
|
||||
else:
|
||||
if not self.is_standard_fused_experts():
|
||||
return False, "Mismatched format."
|
||||
|
||||
# Check quantization sanity
|
||||
if (
|
||||
int(self.is_per_act_token_quant)
|
||||
+ int(self.is_per_tensor_act_quant)
|
||||
+ int(self.quant_block_shape is not None)
|
||||
) > 1:
|
||||
# invalid quant config
|
||||
return False, f"Bad quant_config {self.quant_config}."
|
||||
|
||||
# check type support
|
||||
if self.quant_dtype is None:
|
||||
if (
|
||||
self.dtype not in self.pf_supported_types()
|
||||
or self.dtype not in self.fe_supported_types()
|
||||
):
|
||||
return False, (
|
||||
f"Unsupported type {self.dtype} not in "
|
||||
f"{self.pf_supported_types()} and "
|
||||
f"{self.fe_supported_types()}."
|
||||
)
|
||||
else:
|
||||
if (
|
||||
self.quant_dtype not in self.pf_supported_types()
|
||||
or self.quant_dtype not in self.fe_supported_types()
|
||||
):
|
||||
return False, (
|
||||
f"Unsupported quant type {self.quant_dtype} "
|
||||
f"not in {self.pf_supported_types()} and "
|
||||
f"{self.fe_supported_types()}."
|
||||
)
|
||||
|
||||
# Check quant scheme compatibility with fused experts class
|
||||
if not self.fe_supports_quant_scheme():
|
||||
return False, (
|
||||
f"FE {self.fused_experts_type.__name__} does not support "
|
||||
f"quant scheme (per_out_ch={self.is_per_out_ch_quant}, "
|
||||
f"per_act_token={self.is_per_act_token_quant}, "
|
||||
f"block={self.quant_block_shape})"
|
||||
)
|
||||
|
||||
# Check activation support; NotImplementedError means no opinion.
|
||||
try:
|
||||
if not self.fused_experts_type._supports_activation(self.activation):
|
||||
return False, (
|
||||
f"FE {self.fused_experts_type.__name__} does not support "
|
||||
f"activation {self.activation}"
|
||||
)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
|
||||
# Check block quantization support
|
||||
is_block_quantized = self.quant_block_shape is not None
|
||||
if is_block_quantized and self.quant_dtype is None:
|
||||
return False, "No block quantization support."
|
||||
|
||||
if is_block_quantized and not self.is_block_quant_supported():
|
||||
return False, "Mismatched block quantization support."
|
||||
|
||||
# deep_gemm only works with block-quantized
|
||||
if self.needs_deep_gemm() and not is_block_quantized:
|
||||
return False, "Needs DeepGEMM but not block quantized."
|
||||
|
||||
# Check dependencies (turn into asserts?)
|
||||
if self.needs_deep_ep() and not has_deep_ep():
|
||||
return False, "Needs DeepEP, but DeepEP not available."
|
||||
if self.needs_deep_ep_v2() and not has_deep_ep_v2():
|
||||
return False, "Needs DeepEP v2, but DeepEP v2 not available."
|
||||
if self.needs_deep_gemm() and not has_deep_gemm():
|
||||
return (
|
||||
False,
|
||||
"Needs DeepGEMM, but the current vLLM environment does not provide it.",
|
||||
)
|
||||
if self.needs_aiter() and not has_aiter(): # noqa: SIM103
|
||||
return False, "Needs Aiter, but Aiter not available."
|
||||
if self.needs_mori() and not has_mori(): # noqa: SIM103
|
||||
return False, "Needs MoRI, but MoRI not available."
|
||||
if self.needs_mori() and not rocm_aiter_ops.is_fused_moe_enabled():
|
||||
return False, (
|
||||
"Mori requires AITER's fused-moe backend to be enabled "
|
||||
"(VLLM_ROCM_USE_AITER=1 and VLLM_ROCM_USE_AITER_MOE=1)."
|
||||
)
|
||||
|
||||
try:
|
||||
if not self.fused_experts_type._supports_current_device():
|
||||
return (
|
||||
False,
|
||||
f"{self.fused_experts_type} not supported on the current device.",
|
||||
)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
|
||||
return True, None
|
||||
|
||||
|
||||
@dataclass
|
||||
class WeightTensors:
|
||||
w1: torch.Tensor
|
||||
w2: torch.Tensor
|
||||
w1_scale: torch.Tensor | None
|
||||
w2_scale: torch.Tensor | None
|
||||
w1_gs: torch.Tensor | None = None
|
||||
w2_gs: torch.Tensor | None = None
|
||||
|
||||
def describe(self):
|
||||
s = ""
|
||||
s += "== Weight Tensors: \n"
|
||||
s += f" - {_describe_tensor(self.w1, 'w1')} \n"
|
||||
s += f" - {_describe_tensor(self.w2, 'w2')} \n"
|
||||
s += f" - {_describe_tensor(self.w1_scale, 'w1_scale')} \n"
|
||||
s += f" - {_describe_tensor(self.w2_scale, 'w2_scale')} \n"
|
||||
s += f" - {_describe_tensor(self.w1_gs, 'w1_gs')} \n"
|
||||
s += f" - {_describe_tensor(self.w2_gs, 'w2_gs')} \n"
|
||||
return s
|
||||
|
||||
def is_quantized(self) -> bool:
|
||||
# or w1_scale is not None?
|
||||
return (
|
||||
is_fp8(self.w1.dtype)
|
||||
or self.w1.dtype == torch.uint8
|
||||
or self.w1.dtype == torch.int8
|
||||
)
|
||||
|
||||
def to_current_device(self):
|
||||
device = torch.accelerator.current_device_index()
|
||||
self.w1 = self.w1.to(device=device)
|
||||
self.w2 = self.w2.to(device=device)
|
||||
|
||||
if self.w1_scale is not None:
|
||||
self.w1_scale = self.w1_scale.to(device=device)
|
||||
if self.w2_scale is not None:
|
||||
self.w2_scale = self.w2_scale.to(device=device)
|
||||
|
||||
if self.w1_gs is not None:
|
||||
self.w1_gs = self.w1_gs.to(device=device)
|
||||
if self.w2_gs is not None:
|
||||
self.w2_gs = self.w2_gs.to(device=device)
|
||||
|
||||
def slice_weights(self, rank: int, num_local_experts: int) -> "WeightTensors":
|
||||
s = rank * num_local_experts
|
||||
e = s + num_local_experts
|
||||
w1 = self.w1[s:e, :, :]
|
||||
w2 = self.w2[s:e, :, :]
|
||||
w1_scale = self.w1_scale[s:e, :, :] if self.w1_scale is not None else None
|
||||
w2_scale = self.w2_scale[s:e, :, :] if self.w2_scale is not None else None
|
||||
w1_gs = self.w1_gs[s:e] if self.w1_gs is not None else None
|
||||
w2_gs = self.w2_gs[s:e] if self.w2_gs is not None else None
|
||||
|
||||
return WeightTensors(w1, w2, w1_scale, w2_scale, w1_gs, w2_gs)
|
||||
|
||||
@staticmethod
|
||||
def make(config: Config) -> "WeightTensors":
|
||||
(_, w1, w1_scale, w1_gs), (_, w2, w2_scale, w2_gs) = make_test_weights(
|
||||
e=config.E,
|
||||
n=config.N,
|
||||
k=config.K,
|
||||
in_dtype=config.dtype,
|
||||
quant_dtype=config.quant_dtype,
|
||||
block_shape=config.quant_block_shape,
|
||||
# or config.is_per_out_ch_quant
|
||||
per_out_ch_quant=config.is_per_act_token_quant,
|
||||
)
|
||||
return WeightTensors(
|
||||
w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale, w1_gs=w1_gs, w2_gs=w2_gs
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RankTensors:
|
||||
hidden_states: torch.Tensor
|
||||
hidden_states_scale: torch.Tensor | None
|
||||
|
||||
topk_weights: torch.Tensor
|
||||
topk_ids: torch.Tensor
|
||||
expert_map: torch.Tensor | None
|
||||
|
||||
def describe(self):
|
||||
s = ""
|
||||
s += "== Rank Tensors: \n"
|
||||
s += f" - {_describe_tensor(self.hidden_states, 'HS')} \n"
|
||||
s += f" - {_describe_tensor(self.hidden_states_scale, 'HS_scale')} \n"
|
||||
s += f" - {_describe_tensor(self.topk_weights, 'topk_weights')} \n"
|
||||
s += f" - {_describe_tensor(self.topk_ids, 'topk_ids')} \n"
|
||||
s += f" - {_describe_tensor(self.expert_map, 'expert_map')} \n"
|
||||
return s
|
||||
|
||||
@staticmethod
|
||||
def make_hidden_states(
|
||||
config: Config,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Return hidden_states."""
|
||||
m, k, dtype = (config.M, config.K, config.dtype)
|
||||
device = torch.accelerator.current_device_index()
|
||||
a = torch.randn((m, k), device=device, dtype=dtype) / 15.0
|
||||
|
||||
if config.quant_dtype is None:
|
||||
return a, None
|
||||
|
||||
# We dequant and use that as hidden_states so the tests are stable.
|
||||
# quantizing and dequantizing yield slightly different results
|
||||
# depending on the hardware. Here we, quantize and dequantize
|
||||
# first - so further quantize and dequantize will yield the same
|
||||
# values.
|
||||
if config.is_per_tensor_act_quant:
|
||||
a_q, a_scales = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=False)
|
||||
return a_q.float().mul(a_scales).to(dtype), a_scales
|
||||
|
||||
if config.is_per_act_token_quant:
|
||||
a_q, a_scales = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True)
|
||||
return a_q.float().mul(a_scales).to(dtype), None
|
||||
|
||||
assert config.quant_block_shape is not None
|
||||
block_k = config.quant_block_shape[1]
|
||||
a_q, a_scales = per_token_cast_to_fp8(a, block_size=block_k)
|
||||
return a_q.float().view((-1, block_k)).mul(a_scales.view(-1, 1)).view(m, k).to(
|
||||
dtype
|
||||
), None
|
||||
|
||||
@staticmethod
|
||||
def make(config: Config, pgi: ProcessGroupInfo):
|
||||
dtype = config.dtype
|
||||
topk, m, _ = (config.topk, config.M, config.K)
|
||||
hidden_states, hidden_states_scale = RankTensors.make_hidden_states(config)
|
||||
|
||||
num_local_experts, global_num_experts = (config.num_local_experts, config.E)
|
||||
score = torch.randn((m, global_num_experts), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(hidden_states, score, topk, False)
|
||||
|
||||
# distribute topk_ids evenly
|
||||
device = torch.accelerator.current_device_index()
|
||||
for mi in range(m):
|
||||
topk_ids[mi] = torch.randperm(config.E)[:topk]
|
||||
topk_ids = topk_ids.to(device=device)
|
||||
|
||||
expert_map = None
|
||||
if config.world_size > 1:
|
||||
expert_map = torch.full(
|
||||
(global_num_experts,), fill_value=-1, dtype=torch.int32
|
||||
)
|
||||
s = pgi.rank * num_local_experts
|
||||
e = s + num_local_experts
|
||||
expert_map[s:e] = torch.tensor(list(range(num_local_experts)))
|
||||
expert_map = expert_map.to(device=device, dtype=torch.int32)
|
||||
|
||||
return RankTensors(
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_scale=hidden_states_scale,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
|
||||
|
||||
def reference_moe_impl(
|
||||
config: Config, weights: WeightTensors, rank_tensors: RankTensors
|
||||
) -> torch.Tensor:
|
||||
if config.quant_dtype == "nvfp4":
|
||||
quant_blocksize = 16
|
||||
dtype = config.dtype
|
||||
|
||||
w1_q = weights.w1
|
||||
w1_blockscale = weights.w1_scale
|
||||
w1_gs = weights.w1_gs
|
||||
|
||||
w2_q = weights.w2
|
||||
w2_blockscale = weights.w2_scale
|
||||
w2_gs = weights.w2_gs
|
||||
|
||||
a_global_scale = (
|
||||
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX)
|
||||
/ torch.amax(rank_tensors.hidden_states.flatten(), dim=-1)
|
||||
).to(torch.float32)
|
||||
|
||||
assert w1_gs is not None
|
||||
assert w2_gs is not None
|
||||
assert w1_blockscale is not None
|
||||
assert w2_blockscale is not None
|
||||
|
||||
assert w1_blockscale.shape[1] % 128 == 0
|
||||
assert w1_blockscale.shape[2] % 4 == 0
|
||||
assert w2_blockscale.shape[1] % 128 == 0
|
||||
assert w2_blockscale.shape[2] % 4 == 0
|
||||
|
||||
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(
|
||||
rank_tensors.hidden_states, a_global_scale
|
||||
)
|
||||
|
||||
a = dequantize_nvfp4_to_dtype(
|
||||
a_fp4,
|
||||
a_scale_interleaved,
|
||||
a_global_scale,
|
||||
dtype=dtype,
|
||||
device=a_fp4.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
|
||||
e = w1_q.shape[0]
|
||||
n = w1_q.shape[1] // 2
|
||||
k = w2_q.shape[1]
|
||||
|
||||
w1 = torch.zeros((e, 2 * n, k), device="cuda", dtype=dtype)
|
||||
w2 = torch.zeros((e, k, n), device="cuda", dtype=dtype)
|
||||
|
||||
for idx in range(0, e):
|
||||
w1[idx] = dequantize_nvfp4_to_dtype(
|
||||
w1_q[idx],
|
||||
w1_blockscale[idx],
|
||||
w1_gs[idx],
|
||||
dtype=dtype,
|
||||
device=w1_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
w2[idx] = dequantize_nvfp4_to_dtype(
|
||||
w2_q[idx],
|
||||
w2_blockscale[idx],
|
||||
w2_gs[idx],
|
||||
dtype=dtype,
|
||||
device=w2_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
a_scale = None
|
||||
w1_scale = None
|
||||
w2_scale = None
|
||||
quant_dtype = None
|
||||
per_act_token_quant = False
|
||||
block_shape = None
|
||||
else:
|
||||
a = rank_tensors.hidden_states
|
||||
a_scale = rank_tensors.hidden_states_scale
|
||||
w1 = weights.w1
|
||||
w1_scale = weights.w1_scale
|
||||
w2 = weights.w2
|
||||
w2_scale = weights.w2_scale
|
||||
quant_dtype = config.quant_dtype
|
||||
per_act_token_quant = config.is_per_act_token_quant
|
||||
block_shape = config.quant_block_shape
|
||||
|
||||
return torch_experts(
|
||||
a=a,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weight=rank_tensors.topk_weights,
|
||||
topk_ids=rank_tensors.topk_ids,
|
||||
global_num_experts=config.E,
|
||||
expert_map=None,
|
||||
activation=config.activation,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a_scale,
|
||||
quant_dtype=quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
apply_router_weights_on_input=config.topk == 1
|
||||
and config.supports_apply_weight_on_input(),
|
||||
)
|
||||
|
||||
|
||||
def _make_gscale(num_experts: int) -> torch.Tensor:
|
||||
return torch.ones(
|
||||
(num_experts,),
|
||||
device=torch.accelerator.current_device_index(),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
def make_modular_kernel(
|
||||
config: Config,
|
||||
vllm_config: VllmConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> mk.FusedMoEKernel:
|
||||
# make moe config
|
||||
moe_parallel_config: FusedMoEParallelConfig = FusedMoEParallelConfig.make(
|
||||
tp_size_=get_tensor_model_parallel_world_size(),
|
||||
pcp_size_=get_pcp_group().world_size,
|
||||
dp_size_=get_dp_group().world_size,
|
||||
sp_size_=1,
|
||||
vllm_parallel_config=vllm_config.parallel_config,
|
||||
)
|
||||
|
||||
moe = FusedMoEConfig(
|
||||
num_experts=config.E,
|
||||
experts_per_token=config.topk,
|
||||
hidden_dim=config.K,
|
||||
intermediate_size=config.N,
|
||||
num_local_experts=config.num_local_experts,
|
||||
num_logical_experts=config.E,
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
in_dtype=config.dtype,
|
||||
max_num_tokens=next_power_of_2(config.M),
|
||||
activation=config.activation,
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=RoutingMethodType.DeepSeekV3,
|
||||
hidden_dim_unpadded=config.hidden_dim_unpadded,
|
||||
intermediate_size_per_partition_unpadded=(
|
||||
config.intermediate_size_per_partition_unpadded
|
||||
),
|
||||
)
|
||||
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
moe=moe,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
)
|
||||
assert prepare_finalize is not None
|
||||
|
||||
fused_experts = make_fused_experts(
|
||||
config.fused_experts_type,
|
||||
moe,
|
||||
quant_config,
|
||||
prepare_finalize.num_dispatchers(),
|
||||
config.N,
|
||||
)
|
||||
|
||||
modular_kernel = mk.FusedMoEKernel(
|
||||
prepare_finalize=prepare_finalize,
|
||||
fused_experts=fused_experts,
|
||||
)
|
||||
|
||||
return modular_kernel
|
||||
|
||||
|
||||
def _shuffle_weights_for_aiter(rank_weights: WeightTensors) -> WeightTensors:
|
||||
"""Pre-shuffle weights so AITER selects its prebuilt `preshuffle_on` module.
|
||||
|
||||
Production shuffles in `process_weights_after_loading`; this harness builds its
|
||||
tensors directly, so without this every call asks for a `preshuffle_off` module
|
||||
that is not prebuilt, and JIT-compiles a kernel the image already ships.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
w1, w2 = rocm_aiter_ops.shuffle_weights(rank_weights.w1, rank_weights.w2)
|
||||
w1.is_shuffled = True
|
||||
w2.is_shuffled = True
|
||||
return replace(rank_weights, w1=w1, w2=w2)
|
||||
|
||||
|
||||
def _maybe_convert_weights_for_experts(
|
||||
config: Config,
|
||||
rank_weights: WeightTensors,
|
||||
) -> WeightTensors:
|
||||
"""Convert weights to expert-specific format (e.g., TrtLLM BlockMajorK)."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
)
|
||||
|
||||
fe_type = config.fused_experts_type
|
||||
fe_name = getattr(fe_type, "__name__", "")
|
||||
|
||||
# AITER's prebuilt modules carry a gfx950 instance table even on gfx942 and reject
|
||||
# intermediate sizes that are not a multiple of 256, but only for the fp8 per-tensor
|
||||
# and per-token schemes. Those keep JIT-compiling until
|
||||
# https://github.com/ROCm/aiter/issues/5766 is fixed.
|
||||
if issubclass(fe_type, AiterExperts) and (
|
||||
config.quant_dtype is None or config.quant_block_shape is not None
|
||||
):
|
||||
return _shuffle_weights_for_aiter(rank_weights)
|
||||
|
||||
backend: Fp8MoeBackend | None = None
|
||||
if fe_name == "TrtLlmFp8ExpertsModular":
|
||||
backend = Fp8MoeBackend.FLASHINFER_TRTLLM
|
||||
elif fe_name == "FlashInferExperts":
|
||||
backend = Fp8MoeBackend.FLASHINFER_CUTLASS
|
||||
|
||||
if backend is None or not rank_weights.is_quantized():
|
||||
return rank_weights
|
||||
|
||||
mock_layer = SimpleNamespace(
|
||||
weight_block_size=config.quant_block_shape,
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=True,
|
||||
intermediate_size_per_partition=config.N,
|
||||
),
|
||||
activation=SimpleNamespace(is_gated=True),
|
||||
)
|
||||
|
||||
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
fp8_backend=backend,
|
||||
layer=mock_layer,
|
||||
w13=rank_weights.w1,
|
||||
w2=rank_weights.w2,
|
||||
w13_scale=rank_weights.w1_scale,
|
||||
w2_scale=rank_weights.w2_scale,
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
|
||||
return WeightTensors(
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
w1_gs=rank_weights.w1_gs,
|
||||
w2_gs=rank_weights.w2_gs,
|
||||
)
|
||||
|
||||
|
||||
def run_modular_kernel(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
config: Config,
|
||||
weights: WeightTensors,
|
||||
rank_tensors: RankTensors,
|
||||
) -> torch.Tensor:
|
||||
assert isinstance(config.Ms, int)
|
||||
assert isinstance(config.topks, int)
|
||||
|
||||
# weights for rank
|
||||
rank_weights = weights.slice_weights(pgi.rank, config.num_local_experts)
|
||||
rank_weights = _maybe_convert_weights_for_experts(config, rank_weights)
|
||||
|
||||
if config.quant_dtype == "nvfp4":
|
||||
gscale = _make_gscale(config.num_local_experts)
|
||||
else:
|
||||
gscale = None
|
||||
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
config.quant_dtype,
|
||||
w1_scale=rank_weights.w1_scale,
|
||||
w2_scale=rank_weights.w2_scale,
|
||||
a1_scale=rank_tensors.hidden_states_scale,
|
||||
g1_alphas=(1 / rank_weights.w1_gs) if rank_weights.w1_gs is not None else None,
|
||||
g2_alphas=(1 / rank_weights.w2_gs) if rank_weights.w2_gs is not None else None,
|
||||
a1_gscale=gscale,
|
||||
a2_gscale=gscale,
|
||||
block_shape=config.quant_block_shape,
|
||||
per_act_token_quant=config.is_per_act_token_quant,
|
||||
per_out_ch_quant=config.is_per_out_ch_quant,
|
||||
)
|
||||
|
||||
mk = make_modular_kernel(config, vllm_config, quant_config)
|
||||
|
||||
# impls might update the tensor in place
|
||||
hidden_states = rank_tensors.hidden_states.clone()
|
||||
|
||||
topk_ids = rank_tensors.topk_ids.to(mk.prepare_finalize.topk_indices_dtype())
|
||||
|
||||
mk_kwargs = {
|
||||
"hidden_states": hidden_states,
|
||||
"w1": rank_weights.w1,
|
||||
"w2": rank_weights.w2,
|
||||
"topk_weights": rank_tensors.topk_weights,
|
||||
"topk_ids": topk_ids,
|
||||
"activation": config.activation,
|
||||
"expert_map": rank_tensors.expert_map,
|
||||
"global_num_experts": config.E,
|
||||
"apply_router_weight_on_input": config.topk == 1
|
||||
and config.supports_apply_weight_on_input(),
|
||||
}
|
||||
|
||||
num_tokens = rank_tensors.hidden_states.shape[0]
|
||||
num_tokens_across_dp = torch.tensor(
|
||||
[num_tokens] * config.world_size, device="cuda", dtype=torch.int
|
||||
)
|
||||
|
||||
torch.distributed.barrier()
|
||||
|
||||
with set_forward_context(
|
||||
None,
|
||||
vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
):
|
||||
out = mk.apply(**mk_kwargs)
|
||||
|
||||
return out
|
||||
@@ -1,499 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
# Fused experts and PrepareFinalize imports
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe import TritonExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
|
||||
BatchedDeepGemmExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmExperts
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
NaiveBatchedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
cutlass_fp4_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
cutlass_fp8_supported,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.flashinfer import (
|
||||
has_flashinfer_cutlass_fused_moe,
|
||||
has_flashinfer_nvlink_one_sided,
|
||||
has_flashinfer_trtllm_fused_moe,
|
||||
)
|
||||
from vllm.utils.import_utils import (
|
||||
has_aiter,
|
||||
has_deep_ep,
|
||||
has_deep_ep_v2,
|
||||
has_deep_gemm,
|
||||
has_mori,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestMoEQuantConfig:
|
||||
quant_dtype: torch.dtype | str | None
|
||||
per_out_ch_quant: bool
|
||||
per_act_token_quant: bool
|
||||
block_shape: list[int] | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PrepareFinalizeInfo:
|
||||
activation_format: mk.FusedMoEActivationFormat
|
||||
supported_dtypes: list[torch.dtype | str]
|
||||
blocked_quantization_support: bool
|
||||
backend: str | None
|
||||
supports_apply_weight_on_input: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExpertInfo:
|
||||
activation_format: mk.FusedMoEActivationFormat
|
||||
supported_dtypes: list[torch.dtype | str]
|
||||
blocked_quantization_support: bool
|
||||
needs_matching_quant: bool = False
|
||||
needs_deep_gemm: bool = False
|
||||
needs_aiter: bool = False
|
||||
|
||||
|
||||
PREPARE_FINALIZE_INFO: dict[
|
||||
mk.FusedMoEPrepareAndFinalizeModular, PrepareFinalizeInfo
|
||||
] = {}
|
||||
EXPERT_INFO: dict[mk.FusedMoEExpertsModular, ExpertInfo] = {}
|
||||
MK_ALL_PREPARE_FINALIZE_TYPES: list[mk.FusedMoEPrepareAndFinalizeModular] = []
|
||||
MK_MULTI_GPU_PREPARE_FINALIZE_TYPES: list[mk.FusedMoEPrepareAndFinalizeModular] = []
|
||||
MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES: list[mk.FusedMoEPrepareAndFinalizeModular] = []
|
||||
MK_FUSED_EXPERT_TYPES: list[mk.FusedMoEExpertsModular] = []
|
||||
|
||||
standard_format = mk.FusedMoEActivationFormat.Standard
|
||||
batched_format = mk.FusedMoEActivationFormat.BatchedExperts
|
||||
common_float_types: list[torch.dtype | str] = [
|
||||
current_platform.fp8_dtype(),
|
||||
torch.bfloat16,
|
||||
torch.float16,
|
||||
torch.float32,
|
||||
]
|
||||
common_float_and_int_types = common_float_types + [torch.int8]
|
||||
nvfp4_types = ["nvfp4"]
|
||||
mxfp4_types = ["mxfp4"]
|
||||
fp8_types = [current_platform.fp8_dtype()]
|
||||
|
||||
|
||||
def register_prepare_and_finalize(
|
||||
kind,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
supported_dtypes: list[torch.dtype | str],
|
||||
blocked_quantization_support: bool,
|
||||
backend: str | None,
|
||||
force_multigpu: bool = False,
|
||||
supports_apply_weight_on_input: bool = True,
|
||||
):
|
||||
global PREPARE_FINALIZE_INFO
|
||||
global MK_ALL_PREPARE_FINALIZE_TYPES
|
||||
global MK_MULTI_GPU_PREPARE_FINALIZE_TYPES
|
||||
global MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES
|
||||
assert kind not in PREPARE_FINALIZE_INFO
|
||||
|
||||
PREPARE_FINALIZE_INFO[kind] = PrepareFinalizeInfo(
|
||||
activation_format,
|
||||
supported_dtypes,
|
||||
blocked_quantization_support,
|
||||
backend,
|
||||
supports_apply_weight_on_input,
|
||||
)
|
||||
MK_ALL_PREPARE_FINALIZE_TYPES.append(kind)
|
||||
if backend is not None or force_multigpu:
|
||||
MK_MULTI_GPU_PREPARE_FINALIZE_TYPES.append(kind)
|
||||
else:
|
||||
MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES.append(kind)
|
||||
|
||||
|
||||
def register_experts(
|
||||
kind,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
supported_dtypes: list[torch.dtype | str],
|
||||
blocked_quantization_support: bool,
|
||||
needs_matching_quant: bool = False,
|
||||
needs_deep_gemm: bool = False,
|
||||
needs_aiter: bool = False,
|
||||
):
|
||||
global EXPERT_INFO
|
||||
global MK_FUSED_EXPERT_TYPES
|
||||
assert kind not in EXPERT_INFO
|
||||
|
||||
EXPERT_INFO[kind] = ExpertInfo(
|
||||
activation_format,
|
||||
supported_dtypes,
|
||||
blocked_quantization_support,
|
||||
needs_matching_quant,
|
||||
needs_deep_gemm,
|
||||
needs_aiter,
|
||||
)
|
||||
|
||||
MK_FUSED_EXPERT_TYPES.append(kind)
|
||||
|
||||
|
||||
def prepare_finalize_info(kind) -> PrepareFinalizeInfo:
|
||||
info = PREPARE_FINALIZE_INFO.get(kind)
|
||||
assert info is not None
|
||||
return info
|
||||
|
||||
|
||||
def expert_info(kind) -> ExpertInfo:
|
||||
info = EXPERT_INFO.get(kind)
|
||||
assert info is not None
|
||||
return info
|
||||
|
||||
|
||||
register_prepare_and_finalize(
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
standard_format,
|
||||
common_float_types,
|
||||
blocked_quantization_support=True,
|
||||
backend=None,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
BatchedTritonExperts,
|
||||
batched_format,
|
||||
common_float_types,
|
||||
blocked_quantization_support=True,
|
||||
needs_matching_quant=True,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
TritonExperts,
|
||||
standard_format,
|
||||
common_float_and_int_types,
|
||||
blocked_quantization_support=True,
|
||||
needs_matching_quant=True,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
NaiveBatchedExperts,
|
||||
batched_format,
|
||||
common_float_and_int_types,
|
||||
blocked_quantization_support=True,
|
||||
)
|
||||
|
||||
# Disable on blackwell for now
|
||||
if has_deep_ep() and not current_platform.has_device_capability(100):
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import (
|
||||
DeepEPHTPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ll import (
|
||||
DeepEPLLPrepareAndFinalize,
|
||||
)
|
||||
|
||||
register_prepare_and_finalize(
|
||||
DeepEPHTPrepareAndFinalize,
|
||||
standard_format,
|
||||
common_float_types,
|
||||
blocked_quantization_support=True,
|
||||
backend="deepep_high_throughput",
|
||||
)
|
||||
|
||||
register_prepare_and_finalize(
|
||||
DeepEPLLPrepareAndFinalize,
|
||||
batched_format,
|
||||
common_float_types,
|
||||
blocked_quantization_support=True,
|
||||
backend="deepep_low_latency",
|
||||
)
|
||||
|
||||
if has_deep_ep_v2() and current_platform.has_device_capability(100):
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
|
||||
DeepEPV2PrepareAndFinalize,
|
||||
)
|
||||
|
||||
register_prepare_and_finalize(
|
||||
DeepEPV2PrepareAndFinalize,
|
||||
standard_format,
|
||||
common_float_types,
|
||||
blocked_quantization_support=True,
|
||||
backend="deepep_v2",
|
||||
)
|
||||
|
||||
if has_mori():
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
|
||||
MoriPrepareAndFinalize,
|
||||
)
|
||||
|
||||
register_prepare_and_finalize(
|
||||
MoriPrepareAndFinalize,
|
||||
standard_format,
|
||||
fp8_types,
|
||||
blocked_quantization_support=True,
|
||||
backend="mori_high_throughput",
|
||||
supports_apply_weight_on_input=False,
|
||||
)
|
||||
|
||||
if has_flashinfer_cutlass_fused_moe() and current_platform.has_device_capability(100):
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two_sided import ( # noqa: E501
|
||||
FlashInferNVLinkTwoSidedPrepareAndFinalize,
|
||||
)
|
||||
|
||||
register_prepare_and_finalize(
|
||||
FlashInferNVLinkTwoSidedPrepareAndFinalize,
|
||||
standard_format,
|
||||
nvfp4_types + fp8_types,
|
||||
blocked_quantization_support=True,
|
||||
backend=None,
|
||||
force_multigpu=True,
|
||||
supports_apply_weight_on_input=False,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
FlashInferExperts,
|
||||
standard_format,
|
||||
nvfp4_types + fp8_types,
|
||||
blocked_quantization_support=True,
|
||||
# Note: this is a hack to get it to run for now
|
||||
)
|
||||
else:
|
||||
FlashInferCutlassMoEPrepareAndFinalize = None
|
||||
FlashInferExperts = None
|
||||
|
||||
if (
|
||||
has_flashinfer_nvlink_one_sided()
|
||||
and has_flashinfer_cutlass_fused_moe()
|
||||
and current_platform.has_device_capability(100)
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_one_sided import ( # noqa: E501
|
||||
FlashInferNVLinkOneSidedPrepareAndFinalize,
|
||||
)
|
||||
|
||||
register_prepare_and_finalize(
|
||||
FlashInferNVLinkOneSidedPrepareAndFinalize,
|
||||
standard_format,
|
||||
nvfp4_types,
|
||||
blocked_quantization_support=False,
|
||||
backend="flashinfer_nvlink_one_sided",
|
||||
supports_apply_weight_on_input=False,
|
||||
)
|
||||
|
||||
if has_flashinfer_cutlass_fused_moe() and current_platform.has_device_capability(100):
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
|
||||
TrtLlmNvFp4ExpertsModular,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
TrtLlmNvFp4ExpertsModular,
|
||||
standard_format,
|
||||
nvfp4_types,
|
||||
blocked_quantization_support=False,
|
||||
)
|
||||
|
||||
if has_flashinfer_trtllm_fused_moe() and current_platform.has_device_capability(100):
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
|
||||
TrtLlmFp8ExpertsModular,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
TrtLlmFp8ExpertsModular,
|
||||
standard_format,
|
||||
fp8_types,
|
||||
blocked_quantization_support=True,
|
||||
)
|
||||
|
||||
if has_aiter():
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterFp8BlockExperts,
|
||||
AiterFp8ChannelExperts,
|
||||
AiterFp8TensorExperts,
|
||||
AiterMxfp4Experts,
|
||||
AiterUnquantizedExperts,
|
||||
)
|
||||
|
||||
# Registration dtypes gate which quant configs the modular-kernel test
|
||||
# harness will ever pair with a given class (see Config.is_valid()):
|
||||
# they must match what each subclass's _supports_quant_scheme() actually
|
||||
# declares, or the harness will generate a case the class rejects.
|
||||
for aiter_experts_cls, aiter_supported_dtypes in (
|
||||
(AiterUnquantizedExperts, common_float_types),
|
||||
(AiterFp8BlockExperts, fp8_types),
|
||||
(AiterFp8TensorExperts, fp8_types),
|
||||
(AiterFp8ChannelExperts, fp8_types),
|
||||
(AiterMxfp4Experts, mxfp4_types),
|
||||
):
|
||||
register_experts(
|
||||
aiter_experts_cls,
|
||||
standard_format,
|
||||
aiter_supported_dtypes,
|
||||
blocked_quantization_support=True,
|
||||
needs_aiter=True,
|
||||
)
|
||||
else:
|
||||
AiterUnquantizedExperts = None
|
||||
AiterFp8BlockExperts = None
|
||||
AiterFp8TensorExperts = None
|
||||
AiterFp8ChannelExperts = None
|
||||
AiterMxfp4Experts = None
|
||||
|
||||
if has_deep_gemm() and is_deep_gemm_supported():
|
||||
register_experts(
|
||||
BatchedDeepGemmExperts,
|
||||
batched_format,
|
||||
fp8_types,
|
||||
blocked_quantization_support=True,
|
||||
needs_matching_quant=False,
|
||||
needs_deep_gemm=True,
|
||||
)
|
||||
register_experts(
|
||||
DeepGemmExperts,
|
||||
standard_format,
|
||||
fp8_types,
|
||||
blocked_quantization_support=True,
|
||||
needs_matching_quant=False,
|
||||
needs_deep_gemm=True,
|
||||
)
|
||||
register_experts(
|
||||
TritonOrDeepGemmExperts,
|
||||
standard_format,
|
||||
common_float_and_int_types,
|
||||
blocked_quantization_support=True,
|
||||
needs_matching_quant=True,
|
||||
needs_deep_gemm=True,
|
||||
)
|
||||
|
||||
if cutlass_fp8_supported():
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
CutlassBatchedExpertsFp8,
|
||||
CutlassExpertsFp8,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
CutlassExpertsFp8,
|
||||
standard_format,
|
||||
fp8_types,
|
||||
blocked_quantization_support=False,
|
||||
)
|
||||
register_experts(
|
||||
CutlassBatchedExpertsFp8,
|
||||
batched_format,
|
||||
fp8_types,
|
||||
blocked_quantization_support=False,
|
||||
)
|
||||
else:
|
||||
CutlassBatchedExpertsFp8 = None
|
||||
CutlassExpertsFp8 = None
|
||||
|
||||
if cutlass_fp4_supported():
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsFp4,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
CutlassExpertsFp4,
|
||||
standard_format,
|
||||
nvfp4_types,
|
||||
blocked_quantization_support=True,
|
||||
)
|
||||
else:
|
||||
CutlassExpertsFp4 = None
|
||||
|
||||
MK_QUANT_CONFIGS: list[TestMoEQuantConfig | None] = [
|
||||
None,
|
||||
# per-channel / per-column weights and per-tensor activations
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
per_out_ch_quant=True,
|
||||
per_act_token_quant=False,
|
||||
block_shape=None,
|
||||
),
|
||||
# per-channel / per-column weights and per-token activations
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
per_out_ch_quant=True,
|
||||
per_act_token_quant=True,
|
||||
block_shape=None,
|
||||
),
|
||||
# per-tensor weights and per-tensor activations
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
per_out_ch_quant=False,
|
||||
per_act_token_quant=False,
|
||||
block_shape=None,
|
||||
),
|
||||
# per-tensor weights and per-token activations
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
per_out_ch_quant=False,
|
||||
per_act_token_quant=True,
|
||||
block_shape=None,
|
||||
),
|
||||
# block-quantized weights and 128 block per-token activations
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
per_out_ch_quant=False,
|
||||
per_act_token_quant=False,
|
||||
block_shape=[128, 128],
|
||||
),
|
||||
# TODO (varun) : Should we test the following combinations ?
|
||||
# block-quantized weights and per-token activations
|
||||
# block-quantized weights and per-tensor activations
|
||||
]
|
||||
|
||||
if cutlass_fp4_supported() or has_flashinfer_cutlass_fused_moe():
|
||||
MK_QUANT_CONFIGS += [
|
||||
TestMoEQuantConfig(
|
||||
quant_dtype="nvfp4",
|
||||
per_out_ch_quant=False,
|
||||
per_act_token_quant=False,
|
||||
block_shape=None,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def make_fused_experts(
|
||||
fused_experts_type: mk.FusedMoEExpertsModular,
|
||||
moe: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
num_dispatchers: int,
|
||||
N: int,
|
||||
) -> mk.FusedMoEExpertsModular:
|
||||
if (
|
||||
fused_experts_type.activation_format()
|
||||
== mk.FusedMoEActivationFormat.BatchedExperts
|
||||
):
|
||||
kwargs = {
|
||||
"moe_config": moe,
|
||||
"quant_config": quant_config,
|
||||
"max_num_tokens": moe.max_num_tokens,
|
||||
"num_dispatchers": num_dispatchers,
|
||||
}
|
||||
else:
|
||||
kwargs = {
|
||||
"moe_config": moe,
|
||||
"quant_config": quant_config,
|
||||
}
|
||||
|
||||
torch.set_printoptions(threshold=0, edgeitems=0, linewidth=10000)
|
||||
|
||||
print(f"Making {fused_experts_type.__class__.__name__} {kwargs} ...")
|
||||
experts = fused_experts_type(**kwargs)
|
||||
|
||||
torch.set_printoptions(threshold=1000, edgeitems=5, linewidth=80)
|
||||
|
||||
return experts
|
||||
@@ -1,183 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Concatenate
|
||||
|
||||
import torch
|
||||
from torch.multiprocessing import spawn # pyright: ignore[reportPrivateImportUsage]
|
||||
from typing_extensions import ParamSpec
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
## Parallel Processes Utils
|
||||
|
||||
P = ParamSpec("P")
|
||||
|
||||
# On ROCm the first AITER CK MoE launch JIT-compiles two stage modules that the
|
||||
# AITER wheel does not prebuild, ~10 min each (ROCm/aiter#5766).
|
||||
LAUNCH_TIMEOUT_S = 1800 if current_platform.is_rocm() else 900
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ProcessGroupInfo:
|
||||
world_size: int
|
||||
world_local_size: int
|
||||
rank: int
|
||||
node_rank: int
|
||||
local_rank: int
|
||||
device: torch.device
|
||||
|
||||
|
||||
def _set_vllm_config(
|
||||
vllm_config: VllmConfig, world_size: int, rank: int, local_rank: int
|
||||
):
|
||||
import tempfile
|
||||
|
||||
temp_file = tempfile.mkstemp()[1]
|
||||
|
||||
# When DP is enabled, processes are organized as:
|
||||
# rank = dp_rank * tp_pp_world_size + tp_pp_rank
|
||||
tp_pp_world_size = vllm_config.parallel_config.world_size
|
||||
vllm_config.parallel_config.data_parallel_rank = rank // tp_pp_world_size
|
||||
tp_pp_rank = rank % tp_pp_world_size
|
||||
vllm_config.parallel_config.rank = tp_pp_rank
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
init_distributed_environment(
|
||||
world_size=tp_pp_world_size,
|
||||
rank=tp_pp_rank,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
local_rank=local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
|
||||
initialize_model_parallel(
|
||||
tensor_model_parallel_size=vllm_config.parallel_config.tensor_parallel_size,
|
||||
pipeline_model_parallel_size=vllm_config.parallel_config.pipeline_parallel_size,
|
||||
)
|
||||
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
|
||||
cpu_group = torch.distributed.split_group(
|
||||
split_ranks=[list(range(world_size))],
|
||||
group_desc="moe_test_cpu",
|
||||
)
|
||||
else:
|
||||
cpu_group = torch.distributed.new_group(
|
||||
list(range(world_size)), backend="gloo"
|
||||
)
|
||||
return cpu_group
|
||||
|
||||
|
||||
def _worker_parallel_launch(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
world_local_size: int,
|
||||
node_rank: int,
|
||||
init_method: str,
|
||||
worker: Callable[..., None],
|
||||
vllm_config: VllmConfig | None,
|
||||
env_dict: dict | None,
|
||||
worker_kwargs: dict[str, Any],
|
||||
*args: Any,
|
||||
) -> None:
|
||||
rank = node_rank * world_local_size + local_rank
|
||||
device = torch.device("cuda", local_rank)
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.distributed.init_process_group(
|
||||
backend="cpu:gloo,cuda:nccl",
|
||||
init_method=init_method,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device_id=device,
|
||||
)
|
||||
barrier = torch.tensor([rank], device=device)
|
||||
torch.distributed.all_reduce(barrier)
|
||||
|
||||
if env_dict is not None:
|
||||
os.environ.update(env_dict)
|
||||
|
||||
cpu_group = None
|
||||
if vllm_config is not None:
|
||||
cpu_group = _set_vllm_config(vllm_config, world_size, rank, local_rank)
|
||||
|
||||
def _run_worker():
|
||||
worker(
|
||||
ProcessGroupInfo(
|
||||
world_size=world_size,
|
||||
world_local_size=world_local_size,
|
||||
rank=rank,
|
||||
node_rank=node_rank,
|
||||
local_rank=local_rank,
|
||||
device=device,
|
||||
),
|
||||
vllm_config,
|
||||
cpu_group,
|
||||
*args,
|
||||
**worker_kwargs,
|
||||
)
|
||||
|
||||
try:
|
||||
if vllm_config is not None:
|
||||
with set_current_vllm_config(vllm_config):
|
||||
_run_worker()
|
||||
else:
|
||||
_run_worker()
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
traceback.print_exc()
|
||||
raise
|
||||
finally:
|
||||
torch.accelerator.synchronize()
|
||||
if vllm_config is not None:
|
||||
cleanup_dist_env_and_memory()
|
||||
else:
|
||||
torch.distributed.destroy_process_group()
|
||||
|
||||
|
||||
def parallel_launch_with_config(
|
||||
world_size: int,
|
||||
worker: Callable[Concatenate[ProcessGroupInfo, VllmConfig, Any, P], None],
|
||||
vllm_config: VllmConfig,
|
||||
env_dict: dict[Any, Any] | None,
|
||||
*args: P.args,
|
||||
**kwargs: P.kwargs,
|
||||
) -> None:
|
||||
context = spawn(
|
||||
_worker_parallel_launch,
|
||||
args=(
|
||||
world_size,
|
||||
world_size,
|
||||
0,
|
||||
f"tcp://{os.getenv('LOCALHOST', 'localhost')}:{get_open_port()}",
|
||||
worker,
|
||||
vllm_config,
|
||||
env_dict,
|
||||
kwargs,
|
||||
)
|
||||
+ args,
|
||||
nprocs=world_size,
|
||||
join=False,
|
||||
)
|
||||
# A worker stuck in communicator setup never exits, so bound the wait.
|
||||
deadline = time.monotonic() + LAUNCH_TIMEOUT_S
|
||||
while not context.join(timeout=max(0.0, deadline - time.monotonic())):
|
||||
if time.monotonic() >= deadline:
|
||||
for process in context.processes:
|
||||
process.kill()
|
||||
for process in context.processes:
|
||||
process.join()
|
||||
raise TimeoutError(
|
||||
f"Parallel workers did not finish within {LAUNCH_TIMEOUT_S}s"
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,690 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import (
|
||||
batched_moe,
|
||||
make_dummy_moe_config,
|
||||
make_quantized_test_activations,
|
||||
make_test_weights,
|
||||
naive_batched_moe,
|
||||
)
|
||||
from tests.kernels.quant_utils import native_batched_masked_quant_matmul
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
invoke_moe_batched_triton_kernel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Triton MoE kernels require CUDA/ROCm/XPU.",
|
||||
)
|
||||
|
||||
MNK_FACTORS = [
|
||||
(1, 128, 128),
|
||||
(1, 512, 512),
|
||||
(1, 1024, 2048),
|
||||
(32, 128, 128),
|
||||
(32, 512, 512),
|
||||
(32, 1024, 2048),
|
||||
(45, 128, 2048),
|
||||
(45, 1024, 128),
|
||||
(64, 512, 512),
|
||||
(64, 1024, 2048),
|
||||
(222, 128, 2048),
|
||||
(222, 1024, 2048),
|
||||
]
|
||||
NUM_EXPERTS = [8, 64]
|
||||
TOP_KS = [1, 2, 6]
|
||||
|
||||
DTYPES = [torch.bfloat16]
|
||||
|
||||
if not current_platform.is_fp8_fnuz():
|
||||
DTYPES.append(torch.float8_e4m3fn)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchedMMConfig:
|
||||
in_dtype: torch.dtype
|
||||
quant_dtype: torch.dtype | None
|
||||
out_dtype: torch.dtype
|
||||
num_experts: int
|
||||
max_tokens_per_expert: int
|
||||
K: int
|
||||
N: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchedMMTensors:
|
||||
A: torch.Tensor # [E, max_tokens, K]
|
||||
B: torch.Tensor # [E, K, N] - column major
|
||||
C: torch.Tensor # [E, max_tokens, N]
|
||||
num_expert_tokens: torch.Tensor # [E]
|
||||
|
||||
@staticmethod
|
||||
def make_tensors(config: BatchedMMConfig):
|
||||
A = (
|
||||
torch.randn(
|
||||
(config.num_experts, config.max_tokens_per_expert, config.K),
|
||||
device=DEVICE,
|
||||
dtype=config.in_dtype,
|
||||
)
|
||||
/ 10
|
||||
)
|
||||
B = torch.randn(
|
||||
(config.num_experts, config.N, config.K),
|
||||
device=DEVICE,
|
||||
dtype=config.in_dtype,
|
||||
)
|
||||
C = torch.zeros(
|
||||
(config.num_experts, config.max_tokens_per_expert, config.N),
|
||||
device=DEVICE,
|
||||
dtype=config.out_dtype,
|
||||
)
|
||||
|
||||
num_expert_tokens = torch.randint(
|
||||
low=0,
|
||||
high=config.max_tokens_per_expert,
|
||||
size=(config.num_experts,),
|
||||
device=DEVICE,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
return BatchedMMTensors(A, B, C, num_expert_tokens)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts", [8, 32])
|
||||
@pytest.mark.parametrize("max_tokens_per_expert", [32, 224, 512])
|
||||
@pytest.mark.parametrize("K", [128, 1024])
|
||||
@pytest.mark.parametrize("N", [128, 1024])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("block_shape", [None, [128, 128]])
|
||||
@pytest.mark.parametrize("per_act_token_quant", [False, True])
|
||||
# Tuned configs can pick a BLOCK_SIZE_K wider than the quantization block.
|
||||
@pytest.mark.parametrize("fp8_block_k", [32, 256])
|
||||
def test_batched_mm(
|
||||
num_experts: int,
|
||||
max_tokens_per_expert: int,
|
||||
K: int,
|
||||
N: int,
|
||||
dtype: torch.dtype,
|
||||
block_shape: list[int] | None,
|
||||
per_act_token_quant: bool,
|
||||
fp8_block_k: int,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
set_random_seed(7)
|
||||
|
||||
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
|
||||
|
||||
if (
|
||||
dtype == torch.float8_e4m3fn
|
||||
and current_platform.is_cuda_alike()
|
||||
and not current_platform.has_device_capability(89)
|
||||
):
|
||||
pytest.skip(
|
||||
"Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89"
|
||||
)
|
||||
|
||||
if (per_act_token_quant or block_shape is not None) and not use_fp8_w8a8:
|
||||
pytest.skip("Don't test blocking for non-quantized types.")
|
||||
|
||||
if per_act_token_quant and block_shape is not None:
|
||||
pytest.skip("Skip illegal quantization test.")
|
||||
|
||||
if fp8_block_k != 32 and not use_fp8_w8a8:
|
||||
pytest.skip("BLOCK_SIZE_K is only varied for fp8.")
|
||||
|
||||
if dtype.itemsize == 1:
|
||||
act_dtype = torch.bfloat16
|
||||
quant_dtype = dtype
|
||||
else:
|
||||
act_dtype = dtype
|
||||
quant_dtype = None
|
||||
|
||||
num_expert_tokens = torch.randint(
|
||||
low=0,
|
||||
high=max_tokens_per_expert,
|
||||
size=(num_experts,),
|
||||
device=DEVICE,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
A, A_q, A_scale = make_quantized_test_activations(
|
||||
num_experts,
|
||||
max_tokens_per_expert,
|
||||
K,
|
||||
in_dtype=act_dtype,
|
||||
quant_dtype=quant_dtype,
|
||||
block_shape=block_shape,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
)
|
||||
|
||||
(B, B_q, B_scale, _), _ = make_test_weights(
|
||||
num_experts,
|
||||
N // 2,
|
||||
K,
|
||||
in_dtype=act_dtype,
|
||||
quant_dtype=quant_dtype,
|
||||
block_shape=block_shape,
|
||||
per_out_ch_quant=per_act_token_quant,
|
||||
)
|
||||
|
||||
out_shape = (num_experts, max_tokens_per_expert, N)
|
||||
test_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
|
||||
ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
|
||||
q_ref_output = torch.zeros(out_shape, dtype=act_dtype, device=DEVICE)
|
||||
|
||||
compute_tl_dtype = {
|
||||
torch.float16: tl.float16,
|
||||
torch.bfloat16: tl.bfloat16,
|
||||
torch.float32: tl.float32,
|
||||
}[test_output.dtype]
|
||||
|
||||
assert A_q.dtype == B_q.dtype
|
||||
|
||||
invoke_moe_batched_triton_kernel(
|
||||
A_q,
|
||||
B_q,
|
||||
test_output,
|
||||
num_expert_tokens,
|
||||
compute_tl_dtype,
|
||||
# Quantization data
|
||||
A_scale,
|
||||
B_scale,
|
||||
None,
|
||||
# Quantization schemes
|
||||
use_fp8_w8a8,
|
||||
False,
|
||||
False,
|
||||
config={
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 16 if dtype.itemsize > 1 else fp8_block_k,
|
||||
},
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
|
||||
ref_output = native_batched_masked_quant_matmul(
|
||||
A,
|
||||
B,
|
||||
ref_output,
|
||||
num_expert_tokens,
|
||||
)
|
||||
|
||||
q_ref_output = native_batched_masked_quant_matmul(
|
||||
A_q,
|
||||
B_q,
|
||||
q_ref_output,
|
||||
num_expert_tokens,
|
||||
A_scale,
|
||||
B_scale,
|
||||
block_shape,
|
||||
per_act_token_quant,
|
||||
)
|
||||
|
||||
rtol, atol = {
|
||||
torch.float16: (6e-2, 6e-2),
|
||||
torch.bfloat16: (6e-2, 6e-2),
|
||||
torch.float32: (1e-2, 1e-2),
|
||||
}[test_output.dtype]
|
||||
|
||||
torch.testing.assert_close(ref_output, q_ref_output, atol=atol, rtol=rtol)
|
||||
torch.testing.assert_close(test_output, q_ref_output, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("m", "n", "k"), MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", NUM_EXPERTS)
|
||||
@pytest.mark.parametrize("topk", TOP_KS)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("per_act_token_quant", [False, True])
|
||||
@pytest.mark.parametrize("block_shape", [None, [128, 128]])
|
||||
@pytest.mark.parametrize("input_scales", [False])
|
||||
def test_fused_moe_batched_experts(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
e: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
per_act_token_quant: bool,
|
||||
block_shape: list[int] | None,
|
||||
input_scales: bool,
|
||||
workspace_init,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
set_random_seed(7)
|
||||
|
||||
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
|
||||
|
||||
if (
|
||||
dtype == torch.float8_e4m3fn
|
||||
and current_platform.is_cuda_alike()
|
||||
and not current_platform.has_device_capability(89)
|
||||
):
|
||||
pytest.skip(
|
||||
"Triton limitation: fp8e4nv data type is not supported on CUDA arch < 89"
|
||||
)
|
||||
|
||||
if topk > e:
|
||||
pytest.skip("topk > e")
|
||||
|
||||
if not use_fp8_w8a8 and (per_act_token_quant or block_shape is not None):
|
||||
pytest.skip("Skip quantization test for non-quantized type")
|
||||
|
||||
if per_act_token_quant and block_shape is not None:
|
||||
pytest.skip("Skip illegal quantization test.")
|
||||
|
||||
a = torch.randn((m, k), device=DEVICE, dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device=DEVICE, dtype=torch.bfloat16)
|
||||
|
||||
if dtype.itemsize == 1:
|
||||
act_dtype = torch.bfloat16
|
||||
quant_dtype = dtype
|
||||
else:
|
||||
act_dtype = dtype
|
||||
quant_dtype = None
|
||||
|
||||
(w1_16, w1, w1_s, _), (w2_16, w2, w2_s, _) = make_test_weights(
|
||||
e,
|
||||
n,
|
||||
k,
|
||||
block_shape=block_shape,
|
||||
in_dtype=act_dtype,
|
||||
quant_dtype=quant_dtype,
|
||||
per_out_ch_quant=per_act_token_quant,
|
||||
)
|
||||
|
||||
if input_scales and quant_dtype is not None:
|
||||
a1_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32)
|
||||
a2_scale = torch.tensor(1, device=DEVICE, dtype=torch.float32)
|
||||
else:
|
||||
a1_scale = None
|
||||
a2_scale = None
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
topk_weight, topk_ids, _ = fused_topk(a, score, topk, False)
|
||||
|
||||
baseline_output = torch_experts(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
quant_dtype=quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
|
||||
batched_output = naive_batched_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
quant_dtype=quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
|
||||
triton_output = batched_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
quant_dtype=quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(batched_output, baseline_output, atol=3e-2, rtol=2e-2)
|
||||
|
||||
torch.testing.assert_close(triton_output, batched_output, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
# USE_TD in moe_mmk
|
||||
|
||||
# K % 8 == 0 (bf16, 16-byte alignment)
|
||||
_TD_SHAPES = [
|
||||
(4, 1, 128, 256),
|
||||
(8, 64, 512, 512),
|
||||
(4, 256, 1024, 2048),
|
||||
]
|
||||
|
||||
|
||||
def _td_supported() -> bool:
|
||||
return current_platform.is_xpu() or (
|
||||
current_platform.is_cuda() and current_platform.has_device_capability(90)
|
||||
)
|
||||
|
||||
|
||||
def _run_td(A, B, num_expert_tokens, use_td: bool):
|
||||
out = torch.zeros(
|
||||
A.shape[0],
|
||||
A.shape[1],
|
||||
B.shape[1],
|
||||
dtype=torch.bfloat16,
|
||||
device=A.device,
|
||||
)
|
||||
import triton
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
batched_triton_kernel,
|
||||
)
|
||||
|
||||
if use_td:
|
||||
from vllm.triton_utils.allocation import set_triton_allocator
|
||||
|
||||
set_triton_allocator(A.device)
|
||||
|
||||
E = A.shape[0]
|
||||
max_tokens = A.shape[1]
|
||||
K = A.shape[2]
|
||||
N = B.shape[1]
|
||||
BM = BN = BK = 64
|
||||
grid = (E, triton.cdiv(max_tokens, BM) * triton.cdiv(N, BN))
|
||||
batched_triton_kernel[grid](
|
||||
A,
|
||||
B,
|
||||
out,
|
||||
num_expert_tokens,
|
||||
tl.bfloat16,
|
||||
max_tokens,
|
||||
K,
|
||||
N,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
A.stride(0),
|
||||
A.stride(1),
|
||||
A.stride(2),
|
||||
B.stride(0),
|
||||
B.stride(2),
|
||||
B.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
out.stride(2),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
BLOCK_M=BM,
|
||||
BLOCK_N=BN,
|
||||
BLOCK_K=BK,
|
||||
USE_TD=use_td,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts,max_tokens_per_expert,K,N", _TD_SHAPES)
|
||||
def test_batched_mm_td_matches_plain(num_experts, max_tokens_per_expert, K, N):
|
||||
if not _td_supported():
|
||||
pytest.skip("TD requires XPU or CUDA sm_90+")
|
||||
set_random_seed(42)
|
||||
device = current_platform.device_type
|
||||
A = (
|
||||
torch.randn(
|
||||
num_experts, max_tokens_per_expert, K, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
/ 10
|
||||
)
|
||||
B = torch.randn(num_experts, N, K, device=device, dtype=torch.bfloat16)
|
||||
num_expert_tokens = torch.randint(
|
||||
1,
|
||||
max_tokens_per_expert + 1,
|
||||
size=(num_experts,),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
out_plain = _run_td(A, B, num_expert_tokens, False)
|
||||
out_td = _run_td(A, B, num_expert_tokens, True)
|
||||
|
||||
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
|
||||
|
||||
|
||||
def test_batched_mm_td_zero_expert_tokens():
|
||||
if not _td_supported():
|
||||
pytest.skip("TD requires XPU or CUDA sm_90+")
|
||||
set_random_seed(42)
|
||||
device = current_platform.device_type
|
||||
E, M, K, N = 8, 32, 256, 256
|
||||
A = torch.randn(E, M, K, device=device, dtype=torch.bfloat16) / 10
|
||||
B = torch.randn(E, N, K, device=device, dtype=torch.bfloat16)
|
||||
num_expert_tokens = torch.zeros(E, device=device, dtype=torch.int32)
|
||||
num_expert_tokens[::2] = M
|
||||
|
||||
out_plain = _run_td(A, B, num_expert_tokens, False)
|
||||
out_td = _run_td(A, B, num_expert_tokens, True)
|
||||
|
||||
for e in range(E):
|
||||
if num_expert_tokens[e].item() == 0:
|
||||
assert out_plain[e].abs().max().item() == 0.0
|
||||
assert out_td[e].abs().max().item() == 0.0
|
||||
|
||||
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
|
||||
|
||||
|
||||
# BatchedTritonExperts device enablement (XPU)
|
||||
def test_batched_triton_experts_supports_current_device():
|
||||
assert BatchedTritonExperts._supports_current_device()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
MoEActivation.SITU,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
],
|
||||
)
|
||||
def test_batched_triton_supports_masked_activations(
|
||||
monkeypatch, activation: MoEActivation
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
|
||||
assert BatchedTritonExperts._supports_activation(activation)
|
||||
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
assert not BatchedTritonExperts._supports_activation(activation)
|
||||
|
||||
|
||||
def _torch_experts_with_shared_activation(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, hidden_dim = a.shape
|
||||
topk = topk_ids.shape[1]
|
||||
routed_input = a[:, None, :].expand(-1, topk, -1).reshape(-1, hidden_dim)
|
||||
routed_experts = topk_ids.reshape(-1)
|
||||
routed_output = torch.zeros_like(routed_input)
|
||||
|
||||
for expert in range(w1.shape[0]):
|
||||
expert_mask = routed_experts == expert
|
||||
if not expert_mask.any():
|
||||
continue
|
||||
projected = routed_input[expert_mask] @ w1[expert].T
|
||||
activated = torch.empty(
|
||||
projected.shape[0], w2.shape[2], dtype=a.dtype, device=a.device
|
||||
)
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
activated,
|
||||
projected,
|
||||
activation_config=activation_config,
|
||||
)
|
||||
routed_output[expert_mask] = activated @ w2[expert].T
|
||||
|
||||
return (
|
||||
(
|
||||
routed_output.view(num_tokens, topk, hidden_dim).float()
|
||||
* topk_weights[..., None]
|
||||
)
|
||||
.sum(dim=1)
|
||||
.to(a.dtype)
|
||||
)
|
||||
|
||||
|
||||
_MASKED_BATCHED_ACTIVATION_CASES = [
|
||||
pytest.param(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
id="situ",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
|
||||
id="swigluoai-uninterleave",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
ApplyMoEActivationConfig(),
|
||||
id="swiglustep",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "activation_config"), _MASKED_BATCHED_ACTIVATION_CASES
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_batched_experts_masked_activation_end_to_end(
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
):
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("Masked activations are not enabled on XPU")
|
||||
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
set_random_seed(7)
|
||||
device = current_platform.device_type
|
||||
init_workspace_manager(torch.device(f"{device}:0"))
|
||||
m, n, k, e, topk = 16, 128, 128, 8, 2
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
|
||||
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
in_dtype=a.dtype,
|
||||
activation=activation,
|
||||
)
|
||||
moe_config.swiglu_limit = activation_config.clamp_limit
|
||||
moe_config.swiglu_alpha = activation_config.alpha
|
||||
moe_config.swiglu_beta = activation_config.beta
|
||||
moe_config.activation_situ_beta = activation_config.activation_situ_beta
|
||||
moe_config.activation_situ_linear_beta = (
|
||||
activation_config.activation_situ_linear_beta
|
||||
)
|
||||
|
||||
routing_weights = torch.softmax(score.float(), dim=-1)
|
||||
topk_weights, topk_ids = torch.topk(routing_weights, topk, dim=-1)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
baseline_output = _torch_experts_with_shared_activation(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation,
|
||||
activation_config,
|
||||
)
|
||||
triton_output = batched_moe(
|
||||
a, w1, w2, topk_weights, topk_ids, moe_config=moe_config
|
||||
)
|
||||
|
||||
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
|
||||
def test_batched_experts_end_to_end(m, n, k, e, topk):
|
||||
"""End-to-end BatchedTritonExperts via the reference (no-comms)
|
||||
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
|
||||
the device-enablement path."""
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
set_random_seed(7)
|
||||
device = current_platform.device_type
|
||||
init_workspace_manager(torch.device(f"{device}:0"))
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
|
||||
# w1 is the fused gate+up projection [E, 2N, K]; w2 is [E, K, N].
|
||||
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
topk_weight, topk_ids, _ = fused_topk(a, score, topk, False)
|
||||
|
||||
baseline_output = torch_experts(a, w1, w2, topk_weight, topk_ids)
|
||||
triton_output = batched_moe(a, w1, w2, topk_weight, topk_ids)
|
||||
|
||||
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def test_batched_triton_backend_mapping():
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
map_unquantized_backend,
|
||||
)
|
||||
|
||||
assert (
|
||||
map_unquantized_backend("batched_triton")
|
||||
== UnquantizedMoeBackend.BATCHED_TRITON
|
||||
)
|
||||
assert map_unquantized_backend("triton") == UnquantizedMoeBackend.TRITON
|
||||
@@ -1,572 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import (
|
||||
make_dummy_moe_config,
|
||||
make_test_quant_config,
|
||||
make_test_weights,
|
||||
modular_triton_fused_moe,
|
||||
)
|
||||
from tests.kernels.quant_utils import native_w8a8_block_matmul
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_experts,
|
||||
fused_topk,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
fp8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts import triton_moe
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||
_valid_deep_gemm_shape,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
get_mk_alignment_for_contiguous_layout,
|
||||
is_deep_gemm_e8m0_used,
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
|
||||
dg_available = has_deep_gemm()
|
||||
|
||||
if current_platform.get_device_capability() < (9, 0):
|
||||
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
|
||||
if current_platform.is_fp8_fnuz():
|
||||
pytest.skip(
|
||||
"Tests in this file require float8_e4m3fn and platform does not support",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
|
||||
# Test configurations
|
||||
DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32]
|
||||
# Deepseek-V3's intermediate size 18432, so N is 18432*2/8=4608 at TP8
|
||||
# and its hidden size is 7168.
|
||||
MNK_FACTORS = [
|
||||
(1, 128, 128),
|
||||
(1, 128, 7168),
|
||||
(1, 1024, 7168),
|
||||
(1, 4608, 128),
|
||||
(1, 4608, 7168),
|
||||
(83, 128, 128),
|
||||
(83, 512, 512),
|
||||
(83, 4608, 512),
|
||||
(83, 4608, 7168),
|
||||
(128, 512, 512),
|
||||
(128, 1024, 7168),
|
||||
(128, 4608, 7168),
|
||||
(2048, 128, 128),
|
||||
(2048, 1024, 7168),
|
||||
(2048, 4608, 512),
|
||||
(2048, 4608, 7168),
|
||||
(8192, 128, 128),
|
||||
(8192, 128, 7168),
|
||||
(8192, 1024, 7168),
|
||||
(8192, 4608, 7168),
|
||||
]
|
||||
|
||||
MNK_FACTORS_DG = [
|
||||
(128, 128, 128),
|
||||
(128, 128, 7168),
|
||||
(128, 1024, 7168),
|
||||
(128, 4608, 128),
|
||||
(128, 4608, 7168),
|
||||
(192, 512, 512),
|
||||
(192, 1024, 7168),
|
||||
(192, 4608, 7168),
|
||||
(1335, 128, 128),
|
||||
(1335, 1024, 7168),
|
||||
(1335, 4608, 512),
|
||||
(1335, 4608, 7168),
|
||||
(2048, 128, 128),
|
||||
(2048, 128, 7168),
|
||||
(2048, 1024, 7168),
|
||||
(2048, 4608, 7168),
|
||||
]
|
||||
|
||||
BLOCK_SIZE = [[128, 128]]
|
||||
E = [2, 8, 16] # [128, 256]
|
||||
TOP_KS = [1, 2, 6]
|
||||
SEEDS = [0]
|
||||
|
||||
|
||||
def torch_w8a8_block_fp8_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
w1_s,
|
||||
w2_s,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
block_shape,
|
||||
silu_fp32=False,
|
||||
expert_map=None,
|
||||
):
|
||||
"""Fused MoE with block-wise fp8 quantization using native torch.
|
||||
|
||||
silu_fp32=True computes the intermediate SiLU in fp32 and quantizes it
|
||||
directly, matching the modular Helion silu_and_mul_per_block_quant kernel;
|
||||
False rounds the SiLU output to bf16 first, matching fused_experts. Each
|
||||
kernel is checked against the reference variant matching its precision.
|
||||
"""
|
||||
B, D = a.shape
|
||||
topk = topk_ids.size(1)
|
||||
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
|
||||
topk_weight = topk_weight.view(-1)
|
||||
topk_ids = topk_ids.view(-1)
|
||||
if expert_map is not None:
|
||||
# w1/w2 hold this rank's experts; slots mapped to -1 contribute nothing.
|
||||
topk_ids = expert_map[topk_ids]
|
||||
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
# Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the
|
||||
# kernels use) so the reference is bit-identical here; removes ~0.06-0.10% fp8
|
||||
# boundary-flip divergence that otherwise accumulates over K. Matmul stays fp32.
|
||||
a_q, a_s = per_token_group_quant_fp8(a, block_k, dtype=current_platform.fp8_dtype())
|
||||
a_q = a_q.to(torch.float32)
|
||||
for i in range(w1.shape[0]):
|
||||
mask = topk_ids == i
|
||||
if mask.sum():
|
||||
inter_out = native_w8a8_block_matmul(
|
||||
a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype
|
||||
)
|
||||
act_out = SiluAndMul().forward_native(
|
||||
inter_out.float() if silu_fp32 else inter_out
|
||||
)
|
||||
act_out_q, act_out_s = per_token_group_quant_fp8(
|
||||
act_out, block_k, dtype=current_platform.fp8_dtype()
|
||||
)
|
||||
out[mask] = native_w8a8_block_matmul(
|
||||
act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype
|
||||
)
|
||||
return (
|
||||
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
|
||||
).sum(dim=1)
|
||||
|
||||
|
||||
# Skip all tests if CUDA is not available
|
||||
pytest.importorskip("torch.cuda")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_cuda():
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS)
|
||||
@pytest.mark.parametrize("E", E)
|
||||
@pytest.mark.parametrize("topk", TOP_KS)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZE)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_fused_moe(
|
||||
M, N, K, E, topk, block_size, dtype, seed, monkeypatch, workspace_init
|
||||
):
|
||||
if topk > E:
|
||||
pytest.skip(f"Skipping test; topk={topk} > E={E}")
|
||||
|
||||
torch.manual_seed(seed)
|
||||
|
||||
a = torch.randn((M, K), dtype=dtype) / 10
|
||||
score = torch.randn((M, E), dtype=dtype)
|
||||
|
||||
w1, w2, quant_config = make_test_quant_config(
|
||||
E,
|
||||
N,
|
||||
K,
|
||||
dtype,
|
||||
quant_dtype=torch.float8_e4m3fn,
|
||||
per_act_token_quant=False,
|
||||
block_shape=block_size,
|
||||
)
|
||||
|
||||
m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config)
|
||||
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||
|
||||
# Set the context to avoid lots of warning spam.
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = torch_w8a8_block_fp8_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
quant_config.w1_scale,
|
||||
quant_config.w2_scale,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
block_size,
|
||||
)
|
||||
|
||||
out = fused_experts(
|
||||
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
|
||||
)
|
||||
|
||||
m_out = m_fused_moe.apply(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
global_num_experts=w1.shape[0],
|
||||
)
|
||||
|
||||
# 0.039 only needed for M >= 8192
|
||||
tol = 0.035 if M < 8192 else 0.039
|
||||
|
||||
# The modular path fuses SiLU+quant in fp32 (silu_and_mul_per_block_quant),
|
||||
# while fused_experts/the reference round SiLU to bf16 first. On large K/N this
|
||||
# ~1-ULP gap pushes m_out past the base tol, so validate m_out against an
|
||||
# fp32-SiLU reference — keeping the tight base tolerance, no widened override.
|
||||
if current_platform.is_rocm() and K >= 4096 and N >= 1024:
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out_m = torch_w8a8_block_fp8_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
quant_config.w1_scale,
|
||||
quant_config.w2_scale,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
block_size,
|
||||
silu_fp32=True,
|
||||
)
|
||||
else:
|
||||
ref_out_m = ref_out
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=tol, rtol=tol)
|
||||
torch.testing.assert_close(m_out, ref_out_m, atol=tol, rtol=tol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_clamp_skips_fused_silu_quant(
|
||||
clamp_limit, monkeypatch, workspace_init, disable_deepgemm_ue8m0
|
||||
):
|
||||
"""A SwiGLU clamp must bypass the fused SiLU+quant kernel, which cannot clamp."""
|
||||
torch.manual_seed(0)
|
||||
M, N, K, E, topk = 16, 256, 512, 8, 2
|
||||
block_size = [128, 128]
|
||||
|
||||
a = torch.randn((M, K), dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((M, E), dtype=torch.bfloat16)
|
||||
(_, w1, w1_s, _), (_, w2, w2_s, _) = make_test_weights(
|
||||
E, N, K, torch.bfloat16, torch.float8_e4m3fn, block_shape=block_size
|
||||
)
|
||||
quant_config = fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
block_shape=block_size,
|
||||
gemm1_clamp_limit=clamp_limit,
|
||||
)
|
||||
m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||
|
||||
fused_calls = 0
|
||||
fused_op = triton_moe.ops.silu_and_mul_per_block_quant
|
||||
|
||||
def counting_fused_op(*args, **kwargs):
|
||||
nonlocal fused_calls
|
||||
fused_calls += 1
|
||||
return fused_op(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
triton_moe.ops, "silu_and_mul_per_block_quant", counting_fused_op
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
m_fused_moe.apply(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
global_num_experts=E,
|
||||
)
|
||||
|
||||
assert (fused_calls > 0) == (clamp_limit is None), (
|
||||
f"fused fast path ran {fused_calls} times with clamp_limit={clamp_limit}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("M", "N", "K"), [(83, 512, 512), (2048, 4608, 512)])
|
||||
@pytest.mark.parametrize("topk", [2, 6])
|
||||
@pytest.mark.parametrize("ep_size", [2, 4])
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_fused_moe_expert_map(M, N, K, topk, ep_size, workspace_init):
|
||||
"""One EP rank: local expert shard plus an expert_map with -1 entries."""
|
||||
E, block_size, dtype = 16, [128, 128], torch.bfloat16
|
||||
torch.manual_seed(0)
|
||||
|
||||
a = torch.randn((M, K), dtype=dtype) / 10
|
||||
score = torch.randn((M, E), dtype=dtype)
|
||||
w1, w2, quant_config = make_test_quant_config(
|
||||
E, N, K, dtype, quant_dtype=torch.float8_e4m3fn, block_shape=block_size
|
||||
)
|
||||
|
||||
local_e = E // ep_size
|
||||
e_ids = torch.randperm(E, dtype=torch.int32)[:local_e]
|
||||
e_map = torch.full((E,), -1, dtype=torch.int32)
|
||||
e_map[e_ids] = torch.arange(local_e, dtype=torch.int32)
|
||||
w1, w2 = w1[e_ids].contiguous(), w2[e_ids].contiguous()
|
||||
w1_s = quant_config.w1_scale[e_ids].contiguous()
|
||||
w2_s = quant_config.w2_scale[e_ids].contiguous()
|
||||
quant_config = fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_s, w2_scale=w2_s, block_shape=block_size
|
||||
)
|
||||
|
||||
m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = torch_w8a8_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, block_size, expert_map=e_map
|
||||
)
|
||||
m_out = m_fused_moe.apply(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=e_map,
|
||||
global_num_experts=E,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(m_out, ref_out, atol=0.035, rtol=0.035)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG)
|
||||
@pytest.mark.parametrize("E", E)
|
||||
@pytest.mark.parametrize("topk", TOP_KS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.skipif(not dg_available, reason="DeepGemm kernels not available.")
|
||||
@pytest.mark.skipif(is_deep_gemm_e8m0_used(), reason="Not E8M0 scale MOE")
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_deep_gemm_fused_moe(M, N, K, E, topk, seed, monkeypatch):
|
||||
if topk > E:
|
||||
pytest.skip(f"Skipping test: topk={topk} > E={E}")
|
||||
|
||||
if not _valid_deep_gemm_shape(M, N, K):
|
||||
pytest.skip(f"Skipping test: invalid size m={M}, n={N}, k={K}")
|
||||
|
||||
torch.manual_seed(seed)
|
||||
|
||||
block_size = get_mk_alignment_for_contiguous_layout()
|
||||
dtype = torch.bfloat16
|
||||
|
||||
a = torch.randn((M, K), dtype=dtype) / 10
|
||||
score = torch.randn((M, E), dtype=dtype)
|
||||
|
||||
(_, w1, w1_s, _), (_, w2, w2_s, _) = make_test_weights(
|
||||
E,
|
||||
N,
|
||||
K,
|
||||
dtype,
|
||||
torch.float8_e4m3fn,
|
||||
per_out_ch_quant=False,
|
||||
block_shape=block_size,
|
||||
)
|
||||
|
||||
# Note: for now use_compile will error out if the problem size is
|
||||
# large enough to trigger chunking. I'm leaving the flag and
|
||||
# setup code in case we are able to revisit this later.
|
||||
use_compile = False
|
||||
|
||||
use_cudagraph = N >= 1024 and K >= 1024 and current_platform.is_cuda_alike()
|
||||
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||
|
||||
quant_config = fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
block_shape=block_size,
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
deep_gemm_experts = mk.FusedMoEKernel(
|
||||
prepare_finalize=maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
fused_experts=TritonOrDeepGemmExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
),
|
||||
)
|
||||
|
||||
def deep_gemm_moe_fp8(a, w1, w2, w1_s, w2_s, topk_weights, topk_ids):
|
||||
return deep_gemm_experts.apply(
|
||||
hidden_states=a,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=E,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=False,
|
||||
)
|
||||
|
||||
# Set the context to avoid lots of warning spam.
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = torch_w8a8_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, block_size
|
||||
)
|
||||
|
||||
if use_compile:
|
||||
deep_gemm_moe_fp8_fn = torch.compile(
|
||||
deep_gemm_moe_fp8, backend="inductor", fullgraph=True
|
||||
)
|
||||
torch._dynamo.mark_dynamic(a, 0)
|
||||
torch._dynamo.mark_dynamic(topk_weights, 0)
|
||||
torch._dynamo.mark_dynamic(topk_ids, 0)
|
||||
else:
|
||||
deep_gemm_moe_fp8_fn = deep_gemm_moe_fp8
|
||||
|
||||
out = deep_gemm_moe_fp8_fn(a, w1, w2, w1_s, w2_s, topk_weights, topk_ids)
|
||||
|
||||
if use_cudagraph:
|
||||
out.fill_(0)
|
||||
stream = torch.cuda.Stream()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
out = deep_gemm_moe_fp8_fn(
|
||||
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
|
||||
|
||||
|
||||
def _dequant_block_fp8(
|
||||
w_fp8: torch.Tensor, w_s: torch.Tensor, block_shape: list[int]
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize a [N, K] fp8 weight with [Nb, Kb] block scales to fp32."""
|
||||
n, k = w_fp8.shape
|
||||
bn, bk = block_shape
|
||||
s = w_s.repeat_interleave(bn, dim=0).repeat_interleave(bk, dim=1)
|
||||
return w_fp8.to(torch.float32) * s[:n, :k]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1, 2, 3])
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_fused_moe_refined_block_scales(tp_rank, workspace_init):
|
||||
"""TP-misaligned blockwise FP8 MoE (e.g. Qwen4Exp, intermediate
|
||||
640 at TP=4 -> 160 per rank, not divisible by the checkpoint's 128 block).
|
||||
|
||||
The per-rank scale grid is refined from 128 to 32 (lossless: each 32-block
|
||||
lies inside one global 128-block) so the Triton kernel can consume exact
|
||||
per-shard scales. Simulates one TP rank's shard and checks:
|
||||
1. the refined+sharded scales dequantize exactly like the global scales;
|
||||
2. the Triton fused MoE kernel with block_shape=[32, 32] matches the
|
||||
native-torch blockwise reference on the same shard.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
dtype = torch.bfloat16
|
||||
M, E, topk = 8, 4, 2
|
||||
hidden = 256
|
||||
inter_full = 640
|
||||
tp_size = 4
|
||||
n_shard = inter_full // tp_size # 160
|
||||
ckpt_block = [128, 128]
|
||||
refined_block = [32, 32]
|
||||
factor = ckpt_block[0] // refined_block[0] # 4
|
||||
|
||||
a = torch.randn((M, hidden), dtype=dtype) / 10
|
||||
score = torch.randn((M, E), dtype=dtype)
|
||||
|
||||
(_, w1_full, w1_s_full, _), (_, w2_full, w2_s_full, _) = make_test_weights(
|
||||
E,
|
||||
inter_full,
|
||||
hidden,
|
||||
dtype,
|
||||
torch.float8_e4m3fn,
|
||||
per_out_ch_quant=False,
|
||||
block_shape=ckpt_block,
|
||||
)
|
||||
assert w1_s_full is not None
|
||||
assert w2_s_full is not None
|
||||
|
||||
# TP shard the weights (exact slicing, no scale involvement).
|
||||
lo, hi = tp_rank * n_shard, (tp_rank + 1) * n_shard
|
||||
w1 = torch.cat(
|
||||
[w1_full[:, lo:hi], w1_full[:, inter_full + lo : inter_full + hi]], dim=1
|
||||
)
|
||||
w2 = w2_full[:, :, lo:hi]
|
||||
|
||||
# Refine the global 128-block scales to 32 blocks, then take the shard's
|
||||
# slice -- mirrors the Fp8MoEMethod/RoutedExperts loading path.
|
||||
w1_s32 = w1_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
|
||||
factor, dim=-1
|
||||
)
|
||||
nb = n_shard // refined_block[0] # 5 local 32-blocks per projection
|
||||
gate_hi = w1_s32.shape[1] // 2
|
||||
w1_s = torch.cat(
|
||||
[
|
||||
w1_s32[:, tp_rank * nb : (tp_rank + 1) * nb],
|
||||
w1_s32[:, gate_hi + tp_rank * nb : gate_hi + (tp_rank + 1) * nb],
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
w2_s32 = w2_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
|
||||
factor, dim=-1
|
||||
)
|
||||
w2_s = w2_s32[:, :, tp_rank * nb : (tp_rank + 1) * nb]
|
||||
|
||||
# The refined per-shard scales must reproduce the global-scale dequant
|
||||
# exactly (dequant full-width, then slice to the shard).
|
||||
for e in range(E):
|
||||
d32 = _dequant_block_fp8(w1[e, :n_shard], w1_s[e, :nb], refined_block)
|
||||
d128 = _dequant_block_fp8(w1_full[e], w1_s_full[e], ckpt_block)[lo:hi]
|
||||
assert torch.equal(d32, d128)
|
||||
d32 = _dequant_block_fp8(w2[e], w2_s[e], refined_block)
|
||||
d128 = _dequant_block_fp8(w2_full[e], w2_s_full[e], ckpt_block)[:, lo:hi]
|
||||
assert torch.equal(d32, d128)
|
||||
|
||||
quant_config = fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
block_shape=refined_block,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = torch_w8a8_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, refined_block
|
||||
)
|
||||
out = fused_experts(
|
||||
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
|
||||
@@ -1,163 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_test_quant_config
|
||||
from tests.kernels.quant_utils import (
|
||||
native_per_token_group_quant_int8,
|
||||
native_w8a8_block_matmul,
|
||||
)
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_group_quant_int8,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.get_device_capability() < (7, 0):
|
||||
pytest.skip("INT8 Triton requires CUDA 7.0 or higher", allow_module_level=True)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
|
||||
DTYPES = [torch.bfloat16]
|
||||
|
||||
MNK_FACTORS = [
|
||||
(1, 128, 128),
|
||||
(1, 128, 7168),
|
||||
(1, 1024, 7168),
|
||||
(1, 4096, 512),
|
||||
(1, 4096, 7168),
|
||||
(33, 512, 512),
|
||||
(33, 128, 7168),
|
||||
(33, 1024, 7168),
|
||||
(33, 4096, 128),
|
||||
(33, 4096, 7168),
|
||||
(128, 128, 128),
|
||||
(128, 1024, 7168),
|
||||
(128, 4096, 512),
|
||||
(128, 4096, 7168),
|
||||
(222, 512, 512),
|
||||
(222, 1024, 7168),
|
||||
(222, 4096, 7168),
|
||||
(2048, 128, 128),
|
||||
(2048, 1024, 7168),
|
||||
(2048, 4096, 4096),
|
||||
]
|
||||
|
||||
E = [8, 24]
|
||||
TOP_KS = [2, 6]
|
||||
# BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]]
|
||||
BLOCK_SIZE = [[128, 128]]
|
||||
SEEDS = [0]
|
||||
|
||||
|
||||
# For test
|
||||
def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
|
||||
"""This function performs fused moe with block-wise quantization using
|
||||
native torch."""
|
||||
B, D = a.shape
|
||||
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
topk_weight = topk_weight.view(-1)
|
||||
topk_ids = topk_ids.view(-1)
|
||||
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
a_q, a_s = native_per_token_group_quant_int8(a, block_k, round_to_nearest=False)
|
||||
for i in range(w1.shape[0]):
|
||||
mask = topk_ids == i
|
||||
if mask.sum():
|
||||
inter_out = native_w8a8_block_matmul(
|
||||
a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype
|
||||
)
|
||||
act_out = SiluAndMul().forward_native(inter_out)
|
||||
act_out_q, act_out_s = native_per_token_group_quant_int8(
|
||||
act_out, block_k, round_to_nearest=False
|
||||
)
|
||||
act_out = act_out.to(torch.float32)
|
||||
out[mask] = native_w8a8_block_matmul(
|
||||
act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype
|
||||
)
|
||||
return (
|
||||
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
|
||||
).sum(dim=1)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_cuda():
|
||||
"""Sets the default CUDA device before each test in this module."""
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS)
|
||||
@pytest.mark.parametrize("E", E)
|
||||
@pytest.mark.parametrize("topk", TOP_KS)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZE)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
|
||||
"""Tests the fused_moe kernel with W8A8 INT8 block quantization against a
|
||||
native torch reference."""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
a = torch.randn((M, K), dtype=dtype) / 10
|
||||
score = torch.randn((M, E), dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||
|
||||
w1, w2, quant_config = make_test_quant_config(
|
||||
E,
|
||||
N,
|
||||
K,
|
||||
dtype,
|
||||
quant_dtype=torch.int8,
|
||||
per_act_token_quant=False,
|
||||
block_shape=block_size,
|
||||
)
|
||||
|
||||
# Set the context to avoid lots of warning spam.
|
||||
with set_current_vllm_config(vllm_config):
|
||||
out = fused_experts(
|
||||
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
|
||||
)
|
||||
ref_out = torch_w8a8_block_int8_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
quant_config.w1_scale,
|
||||
quant_config.w2_scale,
|
||||
score,
|
||||
topk,
|
||||
block_size,
|
||||
)
|
||||
|
||||
# Check results
|
||||
torch.testing.assert_close(out, ref_out, atol=0.065, rtol=0.065)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
def test_block_int8_quantization_truncates_fractional_values(dtype):
|
||||
"""Block INT8 quantization truncates, unlike per-token round-to-nearest."""
|
||||
device = current_platform.device_type
|
||||
x = torch.tensor(
|
||||
[[-127, -1.75, -0.75, 0.75, 1.75, 127] + [0] * 122],
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
expected = torch.tensor(
|
||||
[[-127, -1, 0, 0, 1, 127] + [0] * 122], dtype=torch.int8, device=device
|
||||
)
|
||||
|
||||
out_q, scale = per_token_group_quant_int8(x, group_size=128)
|
||||
ref_q, ref_s = native_per_token_group_quant_int8(
|
||||
x, group_size=128, round_to_nearest=False
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_q, expected, atol=0, rtol=0)
|
||||
torch.testing.assert_close(ref_q, expected, atol=0, rtol=0)
|
||||
torch.testing.assert_close(scale, torch.ones_like(scale), atol=0, rtol=0)
|
||||
torch.testing.assert_close(ref_s, scale, atol=0, rtol=0)
|
||||
@@ -1,149 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests compute_expert_num_tokens kernels."""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.utils import count_expert_num_tokens
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Triton MoE kernels require CUDA/ROCm/XPU.",
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TestTensors:
|
||||
topk_ids: torch.Tensor
|
||||
expert_map: torch.Tensor | None = None
|
||||
|
||||
def to_device(self, device: str):
|
||||
self.topk_ids = self.topk_ids.to(device=device)
|
||||
if self.expert_map is not None:
|
||||
self.expert_map = self.expert_map.to(device=device)
|
||||
|
||||
@staticmethod
|
||||
def make(
|
||||
num_tokens: int,
|
||||
num_topk: int,
|
||||
num_experts: int,
|
||||
device: str,
|
||||
topk_ids_dtype: torch.dtype,
|
||||
) -> "TestTensors":
|
||||
# make topk ids
|
||||
topk_ids = torch.empty((num_tokens, num_topk), device=device, dtype=torch.int64)
|
||||
for x in range(num_tokens):
|
||||
topk_ids[x] = torch.randperm(num_experts)[:num_topk]
|
||||
topk_ids = topk_ids.to(dtype=torch.int64)
|
||||
return TestTensors(topk_ids=topk_ids)
|
||||
|
||||
def with_ep_rank(
|
||||
self, ep_rank: int, num_global_experts: int, num_local_experts: int, device: str
|
||||
):
|
||||
# make an expert map
|
||||
expert_map = torch.empty((num_global_experts), device=device, dtype=torch.int32)
|
||||
expert_map.fill_(-1)
|
||||
s = ep_rank * num_local_experts
|
||||
e = s + num_local_experts
|
||||
expert_map[s:e] = torch.tensor(list(range(num_local_experts)), device=device)
|
||||
|
||||
return TestTensors(topk_ids=self.topk_ids.clone(), expert_map=expert_map)
|
||||
|
||||
|
||||
def ref_impl(tt: TestTensors, expert_num_tokens: torch.Tensor):
|
||||
# do the reference in cpu
|
||||
tt.to_device("cpu")
|
||||
expert_ids, counts = tt.topk_ids.unique(return_counts=True)
|
||||
|
||||
for eid, count in zip(expert_ids, counts):
|
||||
if eid != -1 and tt.expert_map is not None:
|
||||
eid = tt.expert_map[eid]
|
||||
|
||||
if eid == -1:
|
||||
continue
|
||||
|
||||
expert_num_tokens[eid] += count
|
||||
|
||||
|
||||
def do_test_compute_expert_num_tokens(
|
||||
num_tokens: int,
|
||||
num_topk: int,
|
||||
num_experts: int,
|
||||
ep_size: int,
|
||||
topk_ids_dtype: torch.dtype,
|
||||
):
|
||||
assert num_topk <= num_experts
|
||||
|
||||
tt = TestTensors.make(
|
||||
num_tokens, num_topk, num_experts, topk_ids_dtype=topk_ids_dtype, device="cpu"
|
||||
)
|
||||
|
||||
num_global_experts = num_experts
|
||||
assert num_global_experts % ep_size == 0
|
||||
num_local_experts = num_global_experts // ep_size
|
||||
for ep_rank in range(ep_size):
|
||||
tt_rank = tt.with_ep_rank(ep_rank, num_global_experts, num_local_experts, "cpu")
|
||||
|
||||
ref_expert_num_tokens = torch.zeros(
|
||||
(num_local_experts), device="cpu", dtype=torch.int32
|
||||
)
|
||||
ref_impl(tt_rank, ref_expert_num_tokens)
|
||||
ref_expert_num_tokens = ref_expert_num_tokens.to(DEVICE)
|
||||
|
||||
tt_rank.to_device(DEVICE)
|
||||
# Test with expert_map
|
||||
triton_expert_num_tokens_w_emap = count_expert_num_tokens(
|
||||
tt_rank.topk_ids, num_local_experts, tt_rank.expert_map
|
||||
)
|
||||
|
||||
# Test without expert map
|
||||
topk_ids = tt_rank.expert_map[tt_rank.topk_ids].to(topk_ids_dtype)
|
||||
triton_expert_num_tokens_wo_emap = count_expert_num_tokens(
|
||||
topk_ids, num_local_experts, expert_map=None
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
ref_expert_num_tokens, triton_expert_num_tokens_w_emap, atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ref_expert_num_tokens, triton_expert_num_tokens_wo_emap, atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 11, 127, 128, 3333, 7317])
|
||||
@pytest.mark.parametrize("num_topk", [2, 6, 8])
|
||||
@pytest.mark.parametrize("num_experts", [64])
|
||||
@pytest.mark.parametrize("ep_size", [1, 2, 4])
|
||||
@pytest.mark.parametrize("topk_ids_dtype", [torch.int64])
|
||||
def test_compute_expert_num_tokens(
|
||||
num_tokens: int,
|
||||
num_topk: int,
|
||||
num_experts: int,
|
||||
ep_size: int,
|
||||
topk_ids_dtype: torch.dtype,
|
||||
):
|
||||
do_test_compute_expert_num_tokens(
|
||||
num_tokens, num_topk, num_experts, ep_size, topk_ids_dtype
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("numel", list(range(1, 8192, 111)))
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("ep_size", [2])
|
||||
@pytest.mark.parametrize("topk_ids_dtype", [torch.int64])
|
||||
def test_compute_expert_num_tokens_from_numel(
|
||||
numel: int, num_experts: int, ep_size: int, topk_ids_dtype: torch.dtype
|
||||
):
|
||||
do_test_compute_expert_num_tokens(
|
||||
num_tokens=numel,
|
||||
num_topk=1,
|
||||
num_experts=num_experts,
|
||||
ep_size=ep_size,
|
||||
topk_ids_dtype=topk_ids_dtype,
|
||||
)
|
||||
@@ -1,608 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
|
||||
from vllm._custom_ops import (
|
||||
cpu_fused_moe,
|
||||
cpu_fused_moe_int8,
|
||||
cpu_prepack_moe_weight,
|
||||
cpu_prepack_moe_weight_int8,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
biased_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ArmCPUUnquantizedExperts,
|
||||
CPUUnquantizedExperts,
|
||||
PowerCPUUnquantizedExperts,
|
||||
X86CPUUnquantizedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
EXPERT_NUM = [
|
||||
8,
|
||||
]
|
||||
HIDDEN_DIM = [128, 2880]
|
||||
INTERMEDIATE_DIM = [128, 2880]
|
||||
BATCH_SIZE = [1, 64, 256]
|
||||
ACT = [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
]
|
||||
USE_BIAS = [False, True]
|
||||
ISA = ["vec"]
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
ISA.append("neon")
|
||||
if torch.cpu._is_amx_tile_supported():
|
||||
ISA.append("amx")
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC:
|
||||
ISA.append("vsx")
|
||||
|
||||
DTYPE = [torch.bfloat16]
|
||||
|
||||
|
||||
def _ref_moe_activation(
|
||||
input: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
) -> torch.Tensor:
|
||||
if activation == MoEActivation.SWIGLUOAI:
|
||||
gate = input[..., ::2].clamp(max=7.0)
|
||||
up = input[..., 1::2].clamp(min=-7.0, max=7.0)
|
||||
glu = gate * torch.sigmoid(gate * 1.702)
|
||||
return (up + 1) * glu
|
||||
|
||||
d = input.shape[-1] // 2
|
||||
gate, up = input[..., :d], input[..., d:]
|
||||
if activation == MoEActivation.SILU:
|
||||
return torch.nn.functional.silu(gate) * up
|
||||
if activation == MoEActivation.GELU:
|
||||
return torch.nn.functional.gelu(gate, approximate="none") * up
|
||||
if activation == MoEActivation.GELU_TANH:
|
||||
return torch.nn.functional.gelu(gate, approximate="tanh") * up
|
||||
raise ValueError(f"Unsupported activation: {activation}")
|
||||
|
||||
|
||||
def ref_fused_moe(
|
||||
input: torch.Tensor,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
) -> torch.Tensor:
|
||||
len_experts = w13.size(0)
|
||||
|
||||
cnts = topk_ids.new_zeros((topk_ids.shape[0], len_experts))
|
||||
cnts.scatter_(1, topk_ids.to(torch.int64), 1)
|
||||
tokens_per_expert = cnts.sum(dim=0)
|
||||
idxs = topk_ids.view(-1).argsort()
|
||||
|
||||
sorted_tokens = input[idxs // topk_ids.shape[1]]
|
||||
tokens_per_expert = tokens_per_expert.cpu().numpy()
|
||||
|
||||
outputs = []
|
||||
start_idx = 0
|
||||
|
||||
for i, num_tokens in enumerate(tokens_per_expert):
|
||||
end_idx = start_idx + num_tokens
|
||||
if num_tokens == 0:
|
||||
continue
|
||||
tokens_for_this_expert = sorted_tokens[start_idx:end_idx].float()
|
||||
curr_w13 = w13[i].float()
|
||||
curr_w2 = w2[i].float()
|
||||
|
||||
curr_w13_bias = None
|
||||
if w13_bias is not None:
|
||||
curr_w13_bias = w13_bias[i].float()
|
||||
|
||||
curr_w2_bias = None
|
||||
if w2_bias is not None:
|
||||
curr_w2_bias = w2_bias[i].float()
|
||||
|
||||
gate_up = torch.nn.functional.linear(
|
||||
tokens_for_this_expert, curr_w13, curr_w13_bias
|
||||
)
|
||||
# Note: to simulate the kernel implementation
|
||||
gate_up = _ref_moe_activation(gate_up, activation)
|
||||
gate_up = gate_up.to(dtype=input.dtype).float()
|
||||
expert_out = torch.nn.functional.linear(gate_up, curr_w2, curr_w2_bias)
|
||||
|
||||
outputs.append(expert_out)
|
||||
start_idx = end_idx
|
||||
|
||||
outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
|
||||
new_x = torch.empty_like(outs)
|
||||
|
||||
new_x[idxs] = outs
|
||||
final_out = (
|
||||
new_x.view(*topk_ids.shape, -1)
|
||||
.mul_(topk_weights.unsqueeze(dim=-1))
|
||||
.sum(dim=1)
|
||||
.type(input.dtype)
|
||||
)
|
||||
return final_out
|
||||
|
||||
|
||||
def quantize_per_channel(
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Symmetrically quantize each weight output channel."""
|
||||
weight_fp32 = weight.float()
|
||||
scale = weight_fp32.abs().amax(dim=-1).clamp_min(1e-12) / 127.0
|
||||
quantized = (
|
||||
(weight_fp32 / scale.unsqueeze(-1)).round().clamp(-127, 127).to(torch.int8)
|
||||
)
|
||||
return quantized, scale
|
||||
|
||||
|
||||
def quantize_per_token(
|
||||
input: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Symmetrically quantize each input token."""
|
||||
input_fp32 = input.float()
|
||||
scale = input_fp32.abs().amax(dim=-1, keepdim=True).clamp_min(1e-7) / 127.0
|
||||
quantized = (input_fp32 / scale).round().clamp(-127, 127).to(torch.int8)
|
||||
return quantized, scale
|
||||
|
||||
|
||||
def ref_fused_moe_int8(
|
||||
input: torch.Tensor,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
) -> torch.Tensor:
|
||||
"""Reference the two dynamically quantized INT8 GEMMs."""
|
||||
input_int8, input_scale = quantize_per_token(input)
|
||||
expert_num = w13.size(0)
|
||||
|
||||
counts = topk_ids.new_zeros((topk_ids.size(0), expert_num))
|
||||
counts.scatter_(1, topk_ids.to(torch.int64), 1)
|
||||
tokens_per_expert = counts.sum(dim=0).tolist()
|
||||
sorted_route_ids = topk_ids.view(-1).argsort()
|
||||
sorted_token_ids = sorted_route_ids // topk_ids.size(1)
|
||||
|
||||
outputs = []
|
||||
start_idx = 0
|
||||
for expert_idx, token_count in enumerate(tokens_per_expert):
|
||||
end_idx = start_idx + token_count
|
||||
if token_count == 0:
|
||||
continue
|
||||
|
||||
token_ids = sorted_token_ids[start_idx:end_idx]
|
||||
gate_up = torch.matmul(
|
||||
input_int8[token_ids].float(),
|
||||
w13[expert_idx].float().T,
|
||||
)
|
||||
gate_up *= input_scale[token_ids] * w13_scale[expert_idx]
|
||||
if w13_bias is not None:
|
||||
gate_up += w13_bias[expert_idx].float()
|
||||
|
||||
intermediate = _ref_moe_activation(gate_up, activation).to(input.dtype)
|
||||
intermediate_int8, intermediate_scale = quantize_per_token(intermediate)
|
||||
output = torch.matmul(
|
||||
intermediate_int8.float(),
|
||||
w2[expert_idx].float().T,
|
||||
)
|
||||
output *= intermediate_scale * w2_scale[expert_idx]
|
||||
if w2_bias is not None:
|
||||
output += w2_bias[expert_idx].float()
|
||||
|
||||
outputs.append(output)
|
||||
start_idx = end_idx
|
||||
|
||||
sorted_output = torch.cat(outputs, dim=0)
|
||||
routed_output = torch.empty_like(sorted_output)
|
||||
routed_output[sorted_route_ids] = sorted_output
|
||||
return (
|
||||
routed_output.view(*topk_ids.shape, -1)
|
||||
.mul_(topk_weights.unsqueeze(-1))
|
||||
.sum(dim=1)
|
||||
.to(input.dtype)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZE)
|
||||
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
|
||||
@pytest.mark.parametrize("intermediate_size", INTERMEDIATE_DIM)
|
||||
@pytest.mark.parametrize("use_bias", USE_BIAS)
|
||||
@pytest.mark.parametrize("dtype", DTYPE)
|
||||
@pytest.mark.parametrize("act", ACT)
|
||||
@pytest.mark.parametrize("isa", ISA)
|
||||
def test_cpu_fused_moe(
|
||||
default_vllm_config,
|
||||
batch_size: int,
|
||||
expert_num: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
use_bias: bool,
|
||||
dtype: torch.dtype,
|
||||
act: MoEActivation,
|
||||
isa: str,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
topk_num = max(expert_num // 2, 1)
|
||||
up_dim = 2 * intermediate_size
|
||||
|
||||
input = torch.randn((batch_size, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w13 = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w2 = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / (
|
||||
0.5 * intermediate_size**0.5
|
||||
)
|
||||
router_logits = torch.randn((batch_size, expert_num), dtype=dtype)
|
||||
w13_bias = None
|
||||
w2_bias = None
|
||||
if use_bias:
|
||||
w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
|
||||
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
score = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk_num)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
ref_output = ref_fused_moe(
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
act,
|
||||
)
|
||||
|
||||
packed_w13 = cpu_prepack_moe_weight(w13, isa)
|
||||
packed_w2 = cpu_prepack_moe_weight(w2, isa)
|
||||
output = torch.empty_like(input)
|
||||
cpu_fused_moe(
|
||||
output,
|
||||
input,
|
||||
packed_w13,
|
||||
packed_w2,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
act.value,
|
||||
isa,
|
||||
)
|
||||
|
||||
atol, rtol = get_default_atol(output), get_default_rtol(output)
|
||||
(
|
||||
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.get_cpu_architecture()
|
||||
not in (CpuArchEnum.ARM, CpuArchEnum.POWERPC),
|
||||
reason="Requires Arm or POWER CPU",
|
||||
)
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZE)
|
||||
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
|
||||
@pytest.mark.parametrize("intermediate_size", INTERMEDIATE_DIM)
|
||||
@pytest.mark.parametrize("use_bias", USE_BIAS)
|
||||
@pytest.mark.parametrize(
|
||||
"dtype",
|
||||
(
|
||||
[torch.float32, torch.float16, torch.bfloat16]
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
else [torch.float32, torch.bfloat16]
|
||||
),
|
||||
)
|
||||
@pytest.mark.parametrize("act", ACT)
|
||||
@pytest.mark.parametrize(
|
||||
"isa",
|
||||
["neon"] if current_platform.get_cpu_architecture() == CpuArchEnum.ARM else ["vsx"],
|
||||
)
|
||||
def test_cpu_fused_moe_int8(
|
||||
batch_size: int,
|
||||
expert_num: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
use_bias: bool,
|
||||
dtype: torch.dtype,
|
||||
act: MoEActivation,
|
||||
isa: str,
|
||||
):
|
||||
set_random_seed(0)
|
||||
topk_num = max(expert_num // 2, 1)
|
||||
up_dim = 2 * intermediate_size
|
||||
|
||||
input = torch.randn((batch_size, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w13_fp = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w2_fp = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / (
|
||||
0.5 * intermediate_size**0.5
|
||||
)
|
||||
w13, w13_scale = quantize_per_channel(w13_fp)
|
||||
w2, w2_scale = quantize_per_channel(w2_fp)
|
||||
|
||||
w13_bias = None
|
||||
w2_bias = None
|
||||
if use_bias:
|
||||
w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
|
||||
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
|
||||
router_logits = torch.randn((batch_size, expert_num), dtype=dtype)
|
||||
score = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(score, topk_num)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
ref_output = ref_fused_moe_int8(
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
act,
|
||||
)
|
||||
packed_w13 = cpu_prepack_moe_weight_int8(w13, isa)
|
||||
packed_w2 = cpu_prepack_moe_weight_int8(w2, isa)
|
||||
output = torch.empty_like(input)
|
||||
cpu_fused_moe_int8(
|
||||
output,
|
||||
input,
|
||||
packed_w13,
|
||||
packed_w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
act.value,
|
||||
isa,
|
||||
)
|
||||
|
||||
atol = rtol = 1e-1 if isa == "vsx" else 2e-2
|
||||
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
# moe_intermediate_size not a multiple of 32, e.g. what tensor-parallel
|
||||
# sharding of moe_intermediate_size=704 at tp=4 produces (704 // 4 == 176).
|
||||
UNALIGNED_INTERMEDIATE_DIM = 176
|
||||
|
||||
|
||||
class _StubMoELayer(torch.nn.Module):
|
||||
"""Minimal stand-in for the real MoE layer module, exposing just what
|
||||
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
|
||||
router configuration, and optionally w13_bias/w2_bias)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
w13_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.w13_weight = torch.nn.Parameter(w13_weight, requires_grad=False)
|
||||
self.w2_weight = torch.nn.Parameter(w2_weight, requires_grad=False)
|
||||
self.activation = activation
|
||||
if w13_bias is not None:
|
||||
self.w13_bias = torch.nn.Parameter(w13_bias, requires_grad=False)
|
||||
if w2_bias is not None:
|
||||
self.w2_bias = torch.nn.Parameter(w2_bias, requires_grad=False)
|
||||
self.use_grouped_topk = False
|
||||
self.renormalize = False
|
||||
self.scoring_func = "softmax"
|
||||
self.custom_routing_function = None
|
||||
|
||||
|
||||
def _make_moe_config(
|
||||
expert_num: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
topk_num: int,
|
||||
dtype: torch.dtype,
|
||||
act: MoEActivation,
|
||||
has_bias: bool = False,
|
||||
) -> FusedMoEConfig:
|
||||
return FusedMoEConfig(
|
||||
num_experts=expert_num,
|
||||
experts_per_token=topk_num,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_local_experts=expert_num,
|
||||
num_logical_experts=expert_num,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=act,
|
||||
in_dtype=dtype,
|
||||
device="cpu",
|
||||
routing_method=RoutingMethodType.Default,
|
||||
has_bias=has_bias,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("hidden_size", "intermediate_size", "act", "expected"),
|
||||
[
|
||||
(128, 128, MoEActivation.SILU, True),
|
||||
# Hidden size is not 32-aligned.
|
||||
(112, 128, MoEActivation.SILU, False),
|
||||
(128, 176, MoEActivation.SILU, True),
|
||||
# SwiGLUOAI's interleaved gate/up layout cannot be padded.
|
||||
(128, 176, MoEActivation.SWIGLUOAI, False),
|
||||
],
|
||||
)
|
||||
def test_cpu_vec_fused_moe_shape_support(
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
act: MoEActivation,
|
||||
expected: bool,
|
||||
):
|
||||
moe_config = _make_moe_config(
|
||||
expert_num=8,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
topk_num=4,
|
||||
dtype=torch.bfloat16,
|
||||
act=act,
|
||||
)
|
||||
supported, _ = CPUUnquantizedExperts.is_supported_config(
|
||||
CPUUnquantizedExperts,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
|
||||
@pytest.mark.parametrize("use_bias", USE_BIAS)
|
||||
@pytest.mark.parametrize("dtype", DTYPE)
|
||||
@pytest.mark.parametrize("isa", ISA)
|
||||
@pytest.mark.parametrize(
|
||||
"act",
|
||||
[MoEActivation.SILU, MoEActivation.GELU, MoEActivation.GELU_TANH],
|
||||
)
|
||||
def test_cpu_fused_moe_unaligned_intermediate_size(
|
||||
default_vllm_config,
|
||||
expert_num: int,
|
||||
hidden_size: int,
|
||||
use_bias: bool,
|
||||
dtype: torch.dtype,
|
||||
isa: str,
|
||||
act: MoEActivation,
|
||||
):
|
||||
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
|
||||
weights before prepacking."""
|
||||
set_random_seed(0)
|
||||
batch_size = 64
|
||||
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
|
||||
topk_num = max(expert_num // 2, 1)
|
||||
up_dim = 2 * intermediate_size
|
||||
|
||||
input = torch.randn((batch_size, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w13 = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w2 = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / (
|
||||
0.5 * intermediate_size**0.5
|
||||
)
|
||||
router_logits = torch.randn((batch_size, expert_num), dtype=dtype)
|
||||
w13_bias = None
|
||||
w2_bias = None
|
||||
if use_bias:
|
||||
w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
|
||||
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
# Route with the same helper the router uses internally, so the
|
||||
# reference only differs from the kernel in how the experts are
|
||||
# evaluated.
|
||||
topk_weight, topk_ids = _softmax_topk(router_logits, topk_num, False)
|
||||
|
||||
ref_output = ref_fused_moe(
|
||||
input, w13, w2, w13_bias, w2_bias, topk_weight, topk_ids, act
|
||||
)
|
||||
|
||||
layer = _StubMoELayer(
|
||||
w13.clone(),
|
||||
w2.clone(),
|
||||
act,
|
||||
w13_bias.clone() if w13_bias is not None else None,
|
||||
w2_bias.clone() if w2_bias is not None else None,
|
||||
)
|
||||
|
||||
moe_config = _make_moe_config(
|
||||
expert_num, hidden_size, intermediate_size, topk_num, dtype, act, use_bias
|
||||
)
|
||||
experts_cls = {
|
||||
"vec": CPUUnquantizedExperts,
|
||||
"amx": X86CPUUnquantizedExperts,
|
||||
"neon": ArmCPUUnquantizedExperts,
|
||||
"vsx": PowerCPUUnquantizedExperts,
|
||||
}[isa]
|
||||
supported, reason = experts_cls.is_supported_config(
|
||||
experts_cls,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported, (
|
||||
f"expected the padded intermediate size to be supported, got {reason}"
|
||||
)
|
||||
|
||||
# Mirror UnquantizedFusedMoEMethod._setup_kernel: the quant config is
|
||||
# built from the layer first, then the experts shuffle the layer into the
|
||||
# kernel's runtime format.
|
||||
quant_config = (
|
||||
biased_moe_quant_config(layer.w13_bias, layer.w2_bias)
|
||||
if use_bias
|
||||
else FusedMoEQuantConfig.make()
|
||||
)
|
||||
experts = experts_cls(moe_config, quant_config)
|
||||
assert experts.isa == isa
|
||||
experts.process_weights_after_loading(layer)
|
||||
|
||||
output = torch.empty_like(input)
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=input,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_weights=topk_weight,
|
||||
topk_ids=topk_ids,
|
||||
activation=act,
|
||||
global_num_experts=expert_num,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=torch.empty(0),
|
||||
workspace2=torch.empty(0),
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
atol, rtol = get_default_atol(output), get_default_rtol(output)
|
||||
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
|
||||
@@ -1,227 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for CPU INT4 W4A8 dynamic quantized fused MoE kernel (CPUExpertsInt4)."""
|
||||
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import (
|
||||
CPUExpertsInt4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.w4a8_int8 import (
|
||||
convert_to_w4a8_int8_moe_format,
|
||||
)
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if (
|
||||
not current_platform.is_cpu()
|
||||
or current_platform.get_cpu_architecture() != CpuArchEnum.ARM
|
||||
):
|
||||
pytest.skip("skipping Arm CPU-only tests", allow_module_level=True)
|
||||
|
||||
|
||||
# Tolerance for INT4 W4A8
|
||||
INT4_W4A8_ATOL = 2e-2
|
||||
INT4_W4A8_RTOL = 2e-2
|
||||
|
||||
|
||||
def _silu_and_mul(x: torch.Tensor) -> torch.Tensor:
|
||||
"""SwiGLU activation: SiLU(gate) * up."""
|
||||
d = x.shape[-1] // 2
|
||||
return F.silu(x[..., :d]) * x[..., d:]
|
||||
|
||||
|
||||
def _make_int4_moe_weights(
|
||||
E: int,
|
||||
N: int,
|
||||
K: int,
|
||||
group_size: int,
|
||||
has_bias: bool = False,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
"""Generate random INT4 MoE weights with random scales.
|
||||
|
||||
Args:
|
||||
E: Number of experts
|
||||
N: Intermediate size
|
||||
K: Hidden size
|
||||
group_size: Quantization group size (-1 for channel-wise)
|
||||
has_bias: Whether to include bias
|
||||
|
||||
Returns:
|
||||
(w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias)
|
||||
where *_ref are the dequantized float reference weights
|
||||
|
||||
"""
|
||||
# Generate INT4 weights as int8 values in [-8, 7]
|
||||
w13_int4 = torch.randint(-8, 8, (E, 2 * N, K), dtype=torch.int8)
|
||||
w2_int4 = torch.randint(-8, 8, (E, K, N), dtype=torch.int8)
|
||||
|
||||
# Determine number of scale columns
|
||||
def _n_scale_cols(in_features: int) -> int:
|
||||
return 1 if group_size == -1 else (in_features // group_size)
|
||||
|
||||
# Generate random scales
|
||||
scale_dtype = torch.float32 if group_size == -1 else torch.bfloat16
|
||||
w13_scales = torch.rand(E, 2 * N, _n_scale_cols(K), dtype=scale_dtype) * 0.01
|
||||
w2_scales = torch.rand(E, K, _n_scale_cols(N), dtype=scale_dtype) * 0.01
|
||||
|
||||
# Generate biases if needed
|
||||
w13_bias = None
|
||||
w2_bias = None
|
||||
if has_bias:
|
||||
w13_bias = torch.randn(E, 2 * N, dtype=torch.float32) * 0.01
|
||||
w2_bias = torch.randn(E, K, dtype=torch.float32) * 0.01
|
||||
|
||||
w13_packed, w2_packed, *_ = convert_to_w4a8_int8_moe_format(
|
||||
w13_weight=w13_int4,
|
||||
w2_weight=w2_int4,
|
||||
w13_weight_scale=w13_scales,
|
||||
w2_weight_scale=w2_scales,
|
||||
group_size=group_size,
|
||||
w13_bias=w13_bias if has_bias else None,
|
||||
w2_bias=w2_bias if has_bias else None,
|
||||
)
|
||||
|
||||
if group_size == -1:
|
||||
w13_scale = w13_scales.float()
|
||||
w2_scale = w2_scales.float()
|
||||
else:
|
||||
w13_scale = w13_scales.float().repeat_interleave(group_size, dim=-1)
|
||||
w2_scale = w2_scales.float().repeat_interleave(group_size, dim=-1)
|
||||
|
||||
w13_ref = w13_int4.float() * w13_scale
|
||||
w2_ref = w2_int4.float() * w2_scale
|
||||
if has_bias and w13_bias is not None:
|
||||
w13_ref = w13_ref + w13_bias.float().unsqueeze(-1)
|
||||
if has_bias and w2_bias is not None:
|
||||
w2_ref = w2_ref + w2_bias.float().unsqueeze(-1)
|
||||
|
||||
return w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias
|
||||
|
||||
|
||||
def ref_int4_moe(
|
||||
a: torch.Tensor,
|
||||
w13_ref: torch.Tensor,
|
||||
w2_ref: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Reference INT4 W4A8 fused MoE using dequantized weights.
|
||||
|
||||
Steps:
|
||||
1. Use dequantized float weights
|
||||
2. For each expert: matmul → SwiGLU → matmul
|
||||
3. Weighted sum across top-k experts
|
||||
"""
|
||||
B, D = a.shape
|
||||
topk = topk_ids.size(1)
|
||||
|
||||
a_exp = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float()
|
||||
out = torch.zeros(B * topk, w2_ref.shape[1], dtype=torch.float32)
|
||||
|
||||
topk_weight_flat = topk_weight.view(-1)
|
||||
topk_ids_flat = topk_ids.view(-1)
|
||||
|
||||
for i in range(w13_ref.shape[0]):
|
||||
mask = topk_ids_flat == i
|
||||
if mask.sum():
|
||||
# w13: [2N, K], input: [B, K] -> output: [B, 2N]
|
||||
gate_up = torch.matmul(a_exp[mask], w13_ref[i].transpose(0, 1))
|
||||
# SwiGLU activation
|
||||
hidden = _silu_and_mul(gate_up)
|
||||
# w2: [K, N], hidden: [B, N] -> output: [B, K]
|
||||
out[mask] = torch.matmul(hidden, w2_ref[i].transpose(0, 1))
|
||||
|
||||
return (
|
||||
(out.view(B, -1, w2_ref.shape[1]) * topk_weight_flat.view(B, -1, 1))
|
||||
.sum(dim=1)
|
||||
.to(a.dtype)
|
||||
)
|
||||
|
||||
|
||||
NUM_TOKENS = [1, 2, 64, 128]
|
||||
# (intermediate_size N, hidden_size K, num_experts E, topk, group_size)
|
||||
MoE_CONFIGS = [
|
||||
(256, 512, 8, 2, 128),
|
||||
(256, 512, 8, 2, 64),
|
||||
(256, 512, 8, 2, -1), # channel-wise
|
||||
(512, 256, 8, 4, 128),
|
||||
(512, 512, 8, 2, 128),
|
||||
(768, 2048, 8, 2, 128),
|
||||
(768, 2048, 16, 4, 64),
|
||||
]
|
||||
SEEDS = [0, 42]
|
||||
ACTIVATION_DTYPES = [torch.float32, torch.bfloat16, torch.float16]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("N,K,E,topk,group_size", MoE_CONFIGS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("activation_dtype", ACTIVATION_DTYPES)
|
||||
def test_cpu_int4_moe_kernel(M, N, K, E, topk, group_size, seed, activation_dtype):
|
||||
"""Test dynamic_4bit_int_moe kernel against dequantized torch reference."""
|
||||
set_random_seed(seed)
|
||||
activation = MoEActivation.SILU
|
||||
|
||||
# Generate input activations
|
||||
a = torch.randn(M, K, dtype=activation_dtype) / (K**0.5)
|
||||
|
||||
# Generate INT4 weights
|
||||
w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias = _make_int4_moe_weights(
|
||||
E, N, K, group_size, has_bias=False
|
||||
)
|
||||
|
||||
# Generate router logits and topk
|
||||
score = torch.randn(M, E, dtype=torch.bfloat16)
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
topk_ids = topk_ids.to(torch.long)
|
||||
|
||||
# Reference output using dequantized weights
|
||||
ref_out = ref_int4_moe(
|
||||
a,
|
||||
w13_ref,
|
||||
w2_ref,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
)
|
||||
|
||||
# Test dynamic_4bit_int_moe kernel
|
||||
apply_router_weight_on_input = False
|
||||
|
||||
out = torch.ops._C.dynamic_4bit_int_moe(
|
||||
a,
|
||||
topk_ids,
|
||||
topk_weight,
|
||||
w13_packed,
|
||||
w2_packed,
|
||||
K, # H (hidden_size / w2_out_features)
|
||||
N, # I (intermediate_size / w2_in_features)
|
||||
group_size,
|
||||
apply_router_weight_on_input,
|
||||
CPUExpertsInt4._activation_kind(activation),
|
||||
)
|
||||
|
||||
assert out.dtype == activation_dtype
|
||||
torch.testing.assert_close(
|
||||
ref_out,
|
||||
out,
|
||||
atol=INT4_W4A8_ATOL,
|
||||
rtol=INT4_W4A8_RTOL,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,586 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip(
|
||||
reason="Nvfp4 Requires compute capability of 10 or above.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
import torch
|
||||
from flashinfer import fp4_quantize
|
||||
from torch.nn import functional as F
|
||||
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_batched_moe import ( # noqa: E501
|
||||
flashinfer_cutedsl_moe_masked,
|
||||
)
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_cutedsl_grouped_gemm_nt_masked as cutedsl_gmm_masked,
|
||||
)
|
||||
from vllm.utils.flashinfer import (
|
||||
scaled_fp4_grouped_quantize,
|
||||
)
|
||||
|
||||
kE2M1ToFloat = torch.tensor(
|
||||
[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32
|
||||
)
|
||||
|
||||
FLOAT8_E4M3_MAX = 448.0
|
||||
FLOAT4_E2M1_MAX = 6.0
|
||||
|
||||
|
||||
def convert_swizzled_to_linear(a_sf_swizzled: torch.Tensor, m, k, block_size):
|
||||
m_tiles = (m + 128 - 1) // 128
|
||||
f = block_size * 4
|
||||
k_tiles = (k + f - 1) // f
|
||||
tmp = torch.reshape(a_sf_swizzled, (1, m_tiles, k_tiles, 32, 4, 4))
|
||||
tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5))
|
||||
out = tmp.reshape(m_tiles * 128, k_tiles * f // block_size)
|
||||
return out[0:m, 0:k]
|
||||
|
||||
|
||||
def dequantize_nvfp4_to_dtype(
|
||||
tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16
|
||||
):
|
||||
"""Dequantize the fp4 tensor back to high precision."""
|
||||
# Two fp4 values are packed into one uint8.
|
||||
assert tensor_fp4.dtype == torch.uint8
|
||||
m, packed_k = tensor_fp4.shape
|
||||
k = packed_k * 2
|
||||
tensor_f32 = break_fp4_bytes(tensor_fp4, dtype)
|
||||
tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size)
|
||||
tensor_sf = tensor_sf.view(torch.float8_e4m3fn)
|
||||
tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size)
|
||||
tensor_sf_dtype = tensor_sf.to(torch.float32) / global_scale
|
||||
|
||||
# scale the tensor
|
||||
out = (tensor_f32 * tensor_sf_dtype.unsqueeze(-1)).reshape(m, k)
|
||||
return out.to(dtype=dtype)
|
||||
|
||||
|
||||
def break_fp4_bytes(a, dtype):
|
||||
assert a.dtype == torch.uint8
|
||||
m, n = a.shape
|
||||
|
||||
# Vectorized nibble processing
|
||||
a_flat = a.flatten()
|
||||
high = (a_flat & 0xF0) >> 4 # Upper nibbles
|
||||
low = a_flat & 0x0F # Lower nibbles
|
||||
|
||||
# Combine nibbles for batch processing
|
||||
combined = torch.stack((low, high), dim=1).flatten()
|
||||
|
||||
# Vectorized sign and magnitude extraction
|
||||
signs = (combined & 0x08).to(torch.bool) # Sign bits
|
||||
abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices
|
||||
|
||||
# Device-aware lookup and sign application
|
||||
kE2M1 = kE2M1ToFloat.to(device=a.device)
|
||||
values = kE2M1[abs_vals] * torch.where(signs, -1.0, 1.0)
|
||||
|
||||
# Reshape to final form
|
||||
return values.reshape(m, n * 2).to(dtype=dtype)
|
||||
|
||||
|
||||
def generate_balanced_routing(
|
||||
hidden_states: torch.Tensor, num_experts: int, top_k: int
|
||||
):
|
||||
"""Generate routing weights and topk indices such that every expert is active.
|
||||
Returns routing_weights, topk_idx
|
||||
"""
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
# num_tokens = batch_size * seq_len
|
||||
|
||||
# First, assign at least one token per expert
|
||||
tokens_per_expert = torch.arange(num_tokens) % num_experts
|
||||
tokens_per_expert = tokens_per_expert[torch.randperm(num_tokens)] # shuffle
|
||||
|
||||
# Each token has top_k experts — start with one guaranteed expert
|
||||
topk_idx = torch.full((num_tokens, top_k), -1, dtype=torch.long)
|
||||
topk_idx[:, 0] = tokens_per_expert
|
||||
|
||||
# For remaining top_k - 1 experts, pick randomly (allowing repeats)
|
||||
if top_k > 1:
|
||||
random_choices = torch.randint(0, num_experts, (num_tokens, top_k - 1))
|
||||
topk_idx[:, 1:] = random_choices
|
||||
|
||||
# Normalize routing weights so each token's weights sum to 1
|
||||
routing_weights = torch.rand(num_tokens, top_k)
|
||||
routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
# Reshape back if needed
|
||||
routing_weights = routing_weights.view(num_tokens, top_k)
|
||||
topk_idx = topk_idx.view(num_tokens, top_k)
|
||||
|
||||
return routing_weights, topk_idx
|
||||
|
||||
|
||||
def prepare_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
):
|
||||
routing_weights, topk_idx = generate_balanced_routing(
|
||||
router_logits, num_experts, topk
|
||||
)
|
||||
|
||||
masked_m = []
|
||||
for i in range(num_experts):
|
||||
mask = topk_idx.view(-1) == i
|
||||
masked_m.append(mask.sum())
|
||||
|
||||
masked_m = torch.tensor(masked_m, dtype=torch.int32)
|
||||
# Initialize the hidden_states_3d with ones instead of empty to avoid nan
|
||||
# issue.
|
||||
hidden_states_3d = torch.ones(
|
||||
(num_experts, max(masked_m), hidden_states.shape[1]),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
for i in range(num_experts):
|
||||
hidden_states_3d[i, : masked_m[i], :] = hidden_states[topk_idx.view(-1) == i]
|
||||
|
||||
return hidden_states_3d, masked_m, topk_idx, routing_weights
|
||||
|
||||
|
||||
MNK_FACTORS = [
|
||||
(2, 1024, 1024),
|
||||
(2, 1024, 1536),
|
||||
(2, 3072, 1024),
|
||||
(2, 3072, 1536),
|
||||
(64, 1024, 1024),
|
||||
(64, 1024, 1536),
|
||||
(64, 3072, 1024),
|
||||
(64, 2048, 1024),
|
||||
(224, 1024, 1024),
|
||||
(224, 1024, 1536),
|
||||
]
|
||||
|
||||
|
||||
# Reference implementation of torch_moe
|
||||
def torch_moe(a, w1, w2, score, topk, expert_map):
|
||||
B, D = a.shape
|
||||
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
topk_weight = topk_weight.view(-1)
|
||||
topk_ids = topk_ids.view(-1)
|
||||
if expert_map is not None:
|
||||
topk_ids = expert_map[topk_ids]
|
||||
for i in range(w1.shape[0]):
|
||||
mask = topk_ids == i
|
||||
if mask.sum():
|
||||
out[mask] = SiluAndMul()(a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(
|
||||
0, 1
|
||||
)
|
||||
return (
|
||||
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
|
||||
).sum(dim=1)
|
||||
|
||||
|
||||
def torch_moe_nvfp4(a, w1, w2, topk, topk_weight, topk_ids):
|
||||
B, D = a.shape
|
||||
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
|
||||
topk_weight = topk_weight.view(-1)
|
||||
topk_ids = topk_ids.view(-1)
|
||||
|
||||
for i in range(w1.shape[0]):
|
||||
mask = topk_ids == i
|
||||
if mask.sum():
|
||||
m = w1[i].shape[0]
|
||||
assert m % 2 == 0
|
||||
# Note: w1 and w3 are swapped!
|
||||
w3_expert, w1_expert = w1[i][m // 2 :, :], w1[i][: m // 2, :]
|
||||
inter = F.silu(a[mask] @ w1_expert.t()) * (a[mask] @ w3_expert.t())
|
||||
inter_gs = torch.tensor(1.0).cuda()
|
||||
inter_q, inter_blockscale = fp4_quantize(inter, inter_gs)
|
||||
inter = dequantize_nvfp4_to_dtype(
|
||||
inter_q,
|
||||
inter_blockscale,
|
||||
inter_gs,
|
||||
dtype=inter.dtype,
|
||||
device=inter.device,
|
||||
block_size=16,
|
||||
).cuda()
|
||||
out[mask] = inter @ w2[i].transpose(0, 1)
|
||||
return (
|
||||
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
|
||||
).sum(dim=1)
|
||||
|
||||
|
||||
def grouped_gemm_ref(
|
||||
hidden_states_expanded: torch.Tensor,
|
||||
hidden_states_3d: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
topk_idx: torch.Tensor,
|
||||
masked_m: torch.Tensor,
|
||||
B: int,
|
||||
topk: int,
|
||||
num_experts: int,
|
||||
*,
|
||||
block_size: int = 16,
|
||||
) -> torch.Tensor:
|
||||
"""Computes the reference grouped GEMM (fp4 quantized per-expert loop),
|
||||
computes flashinfer grouped GEMM (for scale consistency),
|
||||
and returns ONLY the repacked reference output: out_ref.
|
||||
|
||||
Returns:
|
||||
out_ref: Tensor [num_experts, max_m, n_out]
|
||||
|
||||
"""
|
||||
device_hs = hidden_states_expanded.device
|
||||
device_w = weights.device
|
||||
out_dtype = weights.dtype
|
||||
n_out = weights.shape[1]
|
||||
|
||||
# Flattened reference output (B*topk, n_out)
|
||||
out = torch.zeros((B * topk, n_out), dtype=out_dtype, device=device_w)
|
||||
|
||||
# Per-expert reference compute loop
|
||||
for i in range(num_experts):
|
||||
mask = topk_idx.view(-1) == i
|
||||
if mask.any():
|
||||
lhs = hidden_states_expanded[mask]
|
||||
rhs = weights[i]
|
||||
|
||||
a_amax = lhs.abs().max().to(torch.float32).to(device_hs)
|
||||
b_amax = rhs.abs().max().to(torch.float32).to(device_w)
|
||||
|
||||
a_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / a_amax
|
||||
b_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / b_amax
|
||||
|
||||
lhsq, lhsq_sf = fp4_quantize(lhs, a_gs)
|
||||
rhsq, rhsq_sf = fp4_quantize(rhs, b_gs)
|
||||
|
||||
lhs_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
lhsq,
|
||||
lhsq_sf,
|
||||
a_gs,
|
||||
dtype=lhs.dtype,
|
||||
device=device_hs,
|
||||
block_size=block_size,
|
||||
)
|
||||
rhs_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
rhsq,
|
||||
rhsq_sf,
|
||||
b_gs,
|
||||
dtype=rhs.dtype,
|
||||
device=device_w,
|
||||
block_size=block_size,
|
||||
)
|
||||
|
||||
out[mask] = lhs_in_dtype @ rhs_in_dtype.t()
|
||||
|
||||
# Determine per-expert max_m
|
||||
max_m_val = int(masked_m.max().item())
|
||||
|
||||
# Repack into [num_experts, max_m, n_out]
|
||||
out_ref = torch.zeros(
|
||||
(num_experts, max_m_val, n_out),
|
||||
dtype=out.dtype,
|
||||
device=out.device,
|
||||
)
|
||||
expert_slot = [0] * num_experts
|
||||
|
||||
for i, expert_id in enumerate(topk_idx.view(-1).tolist()):
|
||||
slot = expert_slot[expert_id]
|
||||
if slot < max_m_val:
|
||||
out_ref[expert_id, slot, :] = out[i]
|
||||
expert_slot[expert_id] += 1
|
||||
else:
|
||||
raise IndexError(
|
||||
f"Expert {expert_id} exceeded max slots ({max_m_val}). "
|
||||
"Increase max_m or check masked_m."
|
||||
)
|
||||
|
||||
return out_ref
|
||||
|
||||
|
||||
def flashinfer_cutedsl_grouped_gemm_nt_masked(
|
||||
hidden_states: torch.Tensor, # 3d
|
||||
input_global_scale: torch.Tensor, # (l,)
|
||||
weights: torch.Tensor,
|
||||
w_global_scale: torch.Tensor, # (l,)
|
||||
masked_m: torch.Tensor,
|
||||
):
|
||||
# hidden_states: [l, m, k]
|
||||
# weights: [l, n, k]
|
||||
aq, aq_sf = scaled_fp4_grouped_quantize(
|
||||
hidden_states,
|
||||
masked_m.to(hidden_states.device),
|
||||
input_global_scale,
|
||||
)
|
||||
num_experts, n, k = weights.shape
|
||||
bq, bq_sf = scaled_fp4_grouped_quantize(
|
||||
weights,
|
||||
torch.full((num_experts,), n, device=weights.device, dtype=torch.int32),
|
||||
w_global_scale,
|
||||
)
|
||||
|
||||
out = torch.zeros(
|
||||
(num_experts, max(masked_m), n), dtype=weights.dtype, device=aq.device
|
||||
)
|
||||
out = out.permute(1, 2, 0) # requirement of kernel
|
||||
sf_vec_size = 16
|
||||
ab_dtype = "float4_e2m1fn"
|
||||
sf_dtype = "float8_e4m3fn"
|
||||
c_dtype = "bfloat16"
|
||||
alpha = 1.0 / (input_global_scale * w_global_scale).to(out.dtype).view(
|
||||
1, 1, num_experts
|
||||
)
|
||||
|
||||
def get_cute_dtype(input: torch.Tensor) -> str:
|
||||
if input.dtype == torch.bfloat16:
|
||||
return "bfloat16"
|
||||
elif input.dtype == torch.float16:
|
||||
return "float16"
|
||||
elif input.dtype == torch.float32:
|
||||
return "float32"
|
||||
else:
|
||||
raise ValueError(f"Unsupported cute dtype {input.dtype}")
|
||||
|
||||
cutedsl_gmm_masked(
|
||||
(aq, aq_sf),
|
||||
(bq, bq_sf),
|
||||
out,
|
||||
masked_m.to(aq.device),
|
||||
ab_dtype=ab_dtype,
|
||||
sf_dtype=sf_dtype,
|
||||
c_dtype=c_dtype,
|
||||
sf_vec_size=sf_vec_size,
|
||||
alpha=alpha,
|
||||
alpha_dtype=get_cute_dtype(alpha),
|
||||
)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bs, hidden_dim, inter_dim", [(2, 128, 256), (16, 128, 512)])
|
||||
@pytest.mark.parametrize("topk", [1, 2, 4])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_cutedsl_moe_masked(
|
||||
bs: int, hidden_dim: int, inter_dim: int, topk: int
|
||||
):
|
||||
torch.manual_seed(42)
|
||||
device = "cuda"
|
||||
num_experts = 8
|
||||
hidden_states = (
|
||||
torch.randn(bs, hidden_dim, dtype=torch.bfloat16, device=device) / 5.0
|
||||
)
|
||||
w1 = (
|
||||
torch.randn(
|
||||
num_experts, 2 * inter_dim, hidden_dim, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
/ 10.0
|
||||
)
|
||||
w2 = (
|
||||
torch.randn(
|
||||
num_experts, hidden_dim, inter_dim, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
/ 10.0
|
||||
)
|
||||
router_logits = torch.randn(bs, num_experts, dtype=torch.float32)
|
||||
|
||||
hidden_states_expanded = (
|
||||
hidden_states.view(bs, -1, hidden_dim)
|
||||
.repeat(1, topk, 1)
|
||||
.reshape(-1, hidden_dim)
|
||||
)
|
||||
hidden_states_3d, masked_m, topk_idx, routing_weights = prepare_inputs(
|
||||
hidden_states_expanded, router_logits, num_experts, topk
|
||||
)
|
||||
|
||||
w1_amax = w1.abs().amax(dim=(1, 2)).to(torch.float32).to(w1.device)
|
||||
w2_amax = w2.abs().amax(dim=(1, 2)).to(torch.float32).to(w2.device)
|
||||
input_global_scale = torch.ones(
|
||||
(num_experts,), dtype=torch.float32, device=hidden_states.device
|
||||
)
|
||||
|
||||
w1_global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w1_amax
|
||||
w2_global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w2_amax
|
||||
a2_global_scale = torch.ones(
|
||||
(num_experts,), dtype=torch.float32, device=hidden_states.device
|
||||
) # assume intermediate scale is 1.0
|
||||
|
||||
w1_fp4, w1_blockscale = scaled_fp4_grouped_quantize(
|
||||
w1,
|
||||
torch.ones(num_experts, dtype=torch.int32, device=w1.device) * 2 * inter_dim,
|
||||
w1_global_scale,
|
||||
)
|
||||
w2_fp4, w2_blockscale = scaled_fp4_grouped_quantize(
|
||||
w2,
|
||||
torch.ones(num_experts, dtype=torch.int32, device=w2.device) * hidden_dim,
|
||||
w2_global_scale,
|
||||
)
|
||||
|
||||
w1_alpha = 1.0 / (input_global_scale * w1_global_scale)
|
||||
w2_alpha = 1.0 / (a2_global_scale * w2_global_scale)
|
||||
|
||||
out = torch.empty_like(hidden_states_3d, device=hidden_states.device)
|
||||
# Note: the 1st dim shouldn't be bs
|
||||
wk = torch.empty(
|
||||
num_experts,
|
||||
hidden_states_3d.shape[1],
|
||||
inter_dim * 2,
|
||||
dtype=hidden_states_3d.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
flashinfer_cutedsl_moe_masked(
|
||||
hidden_states_3d.to(hidden_states.device),
|
||||
input_global_scale,
|
||||
w1_fp4.permute(2, 0, 1),
|
||||
w1_blockscale,
|
||||
w1_alpha,
|
||||
w2_fp4.permute(2, 0, 1),
|
||||
a2_global_scale,
|
||||
w2_blockscale,
|
||||
w2_alpha,
|
||||
masked_m.to(hidden_states.device),
|
||||
wk,
|
||||
out,
|
||||
)
|
||||
|
||||
# reference
|
||||
# input_global_scale is per-expert ([num_experts]); fp4_quantize and
|
||||
# dequantize_nvfp4_to_dtype are non-grouped APIs that expect [1] or
|
||||
# [num_tokens]. Use a single element since all values are uniform here.
|
||||
a_global = input_global_scale[:1].contiguous()
|
||||
a_fp4, a_scale_interleaved = fp4_quantize(hidden_states, a_global)
|
||||
a_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
a_fp4,
|
||||
a_scale_interleaved,
|
||||
a_global,
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
block_size=16,
|
||||
)
|
||||
w1_d = torch.empty(
|
||||
(num_experts, 2 * inter_dim, hidden_dim), device=w1.device, dtype=w1.dtype
|
||||
)
|
||||
w2_d = torch.empty(
|
||||
(num_experts, hidden_dim, inter_dim), device=w2.device, dtype=w2.dtype
|
||||
)
|
||||
|
||||
for idx in range(0, num_experts):
|
||||
w1_fp4_sliced, w1_blockscale_sliced = fp4_quantize(
|
||||
w1[idx], w1_global_scale[idx]
|
||||
)
|
||||
w2_fp4_sliced, w2_blockscale_sliced = fp4_quantize(
|
||||
w2[idx], w2_global_scale[idx]
|
||||
)
|
||||
w1_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w1_fp4_sliced,
|
||||
w1_blockscale_sliced,
|
||||
w1_global_scale[idx],
|
||||
dtype=w1.dtype,
|
||||
device=w1.device,
|
||||
block_size=16,
|
||||
)
|
||||
w2_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w2_fp4_sliced,
|
||||
w2_blockscale_sliced,
|
||||
w2_global_scale[idx],
|
||||
dtype=w2.dtype,
|
||||
device=w2.device,
|
||||
block_size=16,
|
||||
)
|
||||
|
||||
ref_output = torch_moe_nvfp4(
|
||||
a_in_dtype,
|
||||
w1_d,
|
||||
w2_d,
|
||||
topk,
|
||||
routing_weights.to(a_in_dtype.device),
|
||||
topk_idx.to(a_in_dtype.device),
|
||||
)
|
||||
out_weighted = torch.zeros_like(ref_output, device=out.device, dtype=out.dtype)
|
||||
|
||||
positions = torch.nonzero(masked_m[topk_idx], as_tuple=False)
|
||||
rows, cols = positions[:, 0], positions[:, 1]
|
||||
experts = topk_idx[rows, cols]
|
||||
for i in range(num_experts):
|
||||
mask = experts == i
|
||||
if mask.any():
|
||||
idx = torch.nonzero(mask, as_tuple=False).squeeze(-1)
|
||||
r, c = rows[idx], cols[idx]
|
||||
out_weighted[r] += out[i, : len(r), :] * routing_weights[r, c].to(
|
||||
out.device
|
||||
).unsqueeze(-1)
|
||||
torch.testing.assert_close(
|
||||
out_weighted.cpu(), ref_output.cpu(), atol=2e-1, rtol=2e-1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bs, hidden_dim, inter_dim, topk", [(2, 128, 256, 2), (16, 128, 512, 5)]
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_grouped_gemm_nt_masked(
|
||||
bs: int, hidden_dim: int, inter_dim: int, topk: int
|
||||
) -> None:
|
||||
torch.manual_seed(42)
|
||||
B = bs
|
||||
D = hidden_dim
|
||||
N = inter_dim
|
||||
# CuteDSL group gemm has issue when not all experts are active.
|
||||
# i.e. masked = [2, 3, 0, 0, 1] where the 2nd and 3rd experts are inactive
|
||||
# see https://github.com/flashinfer-ai/flashinfer/issues/1856
|
||||
num_experts = bs
|
||||
hidden_states = torch.randn(B, D, dtype=torch.bfloat16, device="cuda")
|
||||
weights = torch.randn(num_experts, N, D, dtype=torch.bfloat16, device="cuda")
|
||||
router_logits = torch.randn(B, num_experts, dtype=torch.float32)
|
||||
|
||||
hidden_states_expanded = (
|
||||
hidden_states.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
)
|
||||
hidden_states_3d, masked_m, topk_idx, _ = prepare_inputs(
|
||||
hidden_states_expanded, router_logits, num_experts, topk
|
||||
)
|
||||
|
||||
a_amax = (
|
||||
hidden_states_3d.abs()
|
||||
.amax(dim=(1, 2))
|
||||
.to(torch.float32)
|
||||
.to(hidden_states.device)
|
||||
)
|
||||
b_amax = weights.abs().amax(dim=(1, 2)).to(torch.float32).to(weights.device)
|
||||
a_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / a_amax
|
||||
b_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / b_amax
|
||||
out_flashinfer = flashinfer_cutedsl_grouped_gemm_nt_masked(
|
||||
hidden_states_3d.to(hidden_states.device), a_gs, weights, b_gs, masked_m
|
||||
)
|
||||
# reference
|
||||
out_ref = grouped_gemm_ref(
|
||||
hidden_states_expanded=hidden_states_expanded,
|
||||
hidden_states_3d=hidden_states_3d,
|
||||
weights=weights,
|
||||
topk_idx=topk_idx,
|
||||
masked_m=masked_m,
|
||||
B=B,
|
||||
topk=topk,
|
||||
num_experts=num_experts,
|
||||
)
|
||||
# Note: just to compare the masked position due to cutedsl may write nan
|
||||
# into unmasked position.
|
||||
for i in range(num_experts):
|
||||
torch.testing.assert_close(
|
||||
out_flashinfer.permute(2, 0, 1)[i, : masked_m[i]],
|
||||
out_ref.to(out_flashinfer.device)[i, : masked_m[i]],
|
||||
atol=1e-1,
|
||||
rtol=1e-1,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_flashinfer_cutedsl_moe_masked(16, 128, 512, 4)
|
||||
test_grouped_gemm_nt_masked(16, 128, 512, 4)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,570 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test deepep dispatch-combine logic."""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
import torch.distributed
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import TritonExperts
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_deep_ep
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
from ...utils import multi_gpu_test
|
||||
from .parallel_utils import ProcessGroupInfo, parallel_launch
|
||||
|
||||
if has_deep_ep():
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import (
|
||||
DeepEPHTPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ll import (
|
||||
DeepEPLLPrepareAndFinalize,
|
||||
)
|
||||
|
||||
from .parallel_utils import DeepEPHTArgs, DeepEPLLArgs, make_deepep_a2a
|
||||
|
||||
requires_deep_ep = pytest.mark.skipif(
|
||||
not has_deep_ep(),
|
||||
reason="Requires deep_ep kernels",
|
||||
)
|
||||
|
||||
MAX_TOKENS_PER_RANK = 64
|
||||
|
||||
|
||||
def make_weights(
|
||||
e, n, k, dtype
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Return weights w1, w2, w1_scale, w2_scale."""
|
||||
if dtype in [torch.float16, torch.bfloat16]:
|
||||
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
|
||||
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
|
||||
return w1, w2, None, None
|
||||
|
||||
# per-out-channel weight quantization
|
||||
assert dtype == current_platform.fp8_dtype()
|
||||
# Keep FP8 inputs finite and bounded. torch.empty made this test depend on
|
||||
# allocator contents, while larger values exceed its fixed W8A8 tolerance.
|
||||
w1 = torch.randn(
|
||||
(e, 2 * n, k), device=current_platform.device_type, dtype=torch.float16
|
||||
).div_(100)
|
||||
w2 = torch.randn(
|
||||
(e, k, n), device=current_platform.device_type, dtype=torch.float16
|
||||
).div_(100)
|
||||
|
||||
n_b_scales = 2 * n
|
||||
k_b_scales = k
|
||||
w1_q = torch.empty_like(w1, dtype=dtype)
|
||||
w2_q = torch.empty_like(w2, dtype=dtype)
|
||||
w1_scale = torch.empty((e, n_b_scales, 1), device="cuda", dtype=torch.float32)
|
||||
w2_scale = torch.empty((e, k_b_scales, 1), device="cuda", dtype=torch.float32)
|
||||
for expert in range(e):
|
||||
w1_q[expert], w1_scale[expert] = ops.scaled_fp8_quant(
|
||||
w1[expert], use_per_token_if_dynamic=True
|
||||
)
|
||||
w2_q[expert], w2_scale[expert] = ops.scaled_fp8_quant(
|
||||
w2[expert], use_per_token_if_dynamic=True
|
||||
)
|
||||
return w1_q, w2_q, w1_scale, w2_scale
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TestConfig:
|
||||
dtype: torch.dtype
|
||||
topk: int
|
||||
m: int
|
||||
k: int
|
||||
n: int
|
||||
num_experts: int
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TestTensors:
|
||||
rank_tokens: torch.Tensor # all ranks make this many tokens
|
||||
rank_token_scales: torch.Tensor | None
|
||||
topk: torch.Tensor
|
||||
topk_weights: torch.Tensor
|
||||
config: TestConfig
|
||||
|
||||
@staticmethod
|
||||
def make(config: TestConfig, low_latency_mode: bool) -> "TestTensors":
|
||||
# TODO (varun) - check that float16 works ?
|
||||
assert config.dtype in [torch.bfloat16, current_platform.fp8_dtype()]
|
||||
token_dtype = (
|
||||
torch.bfloat16
|
||||
if config.dtype == current_platform.fp8_dtype()
|
||||
else config.dtype
|
||||
)
|
||||
rank_tokens = (
|
||||
torch.randn((config.m, config.k), device="cuda", dtype=token_dtype) / 10
|
||||
)
|
||||
rank_token_scales = None
|
||||
|
||||
topk = torch.randint(
|
||||
low=0, high=config.num_experts, size=(config.m, config.topk), device="cuda"
|
||||
).to(dtype=torch.int64)
|
||||
topk_weights = torch.randn(topk.shape, dtype=torch.float32, device="cuda")
|
||||
return TestTensors(
|
||||
rank_tokens=rank_tokens,
|
||||
rank_token_scales=rank_token_scales,
|
||||
topk=topk,
|
||||
topk_weights=topk_weights,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def make_modular_kernel(
|
||||
pg: ProcessGroup,
|
||||
pgi: ProcessGroupInfo,
|
||||
low_latency_mode: bool,
|
||||
hidden_size: int,
|
||||
dp_size: int,
|
||||
num_experts: int,
|
||||
num_local_experts: int,
|
||||
q_dtype: torch.dtype | None,
|
||||
use_fp8_dispatch: bool,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> FusedMoEKernel:
|
||||
ht_args: DeepEPHTArgs | None = None
|
||||
ll_args: DeepEPLLArgs | None = None
|
||||
|
||||
if low_latency_mode:
|
||||
ll_args = DeepEPLLArgs(
|
||||
max_tokens_per_rank=MAX_TOKENS_PER_RANK,
|
||||
hidden_size=hidden_size,
|
||||
num_experts=num_experts,
|
||||
use_fp8_dispatch=use_fp8_dispatch,
|
||||
)
|
||||
else:
|
||||
assert not use_fp8_dispatch, (
|
||||
"FP8 Dispatch is valid only for low-latency kernels"
|
||||
)
|
||||
ht_args = DeepEPHTArgs(num_local_experts=num_local_experts)
|
||||
|
||||
a2a: DeepEPHTPrepareAndFinalize | DeepEPLLPrepareAndFinalize = make_deepep_a2a(
|
||||
pg=pg,
|
||||
pgi=pgi,
|
||||
dp_size=dp_size,
|
||||
q_dtype=q_dtype,
|
||||
block_shape=None,
|
||||
deepep_ht_args=ht_args,
|
||||
deepep_ll_args=ll_args,
|
||||
)
|
||||
|
||||
num_dispatchers = pgi.world_size // dp_size
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
if low_latency_mode:
|
||||
assert not quant_config.per_act_token_quant, "not supported in ll mode"
|
||||
fused_experts = BatchedTritonExperts(
|
||||
max_num_tokens=MAX_TOKENS_PER_RANK,
|
||||
num_dispatchers=num_dispatchers,
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
else:
|
||||
fused_experts = TritonExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
mk = FusedMoEKernel(
|
||||
prepare_finalize=a2a,
|
||||
fused_experts=fused_experts,
|
||||
)
|
||||
return mk
|
||||
|
||||
|
||||
def deep_ep_moe_impl(
|
||||
pg: ProcessGroup,
|
||||
pgi: ProcessGroupInfo,
|
||||
low_latency_mode: bool,
|
||||
dp_size: int,
|
||||
test_tensors: TestTensors,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None,
|
||||
w2_scale: torch.Tensor | None,
|
||||
num_experts: int,
|
||||
use_fp8_dispatch: bool,
|
||||
per_act_token_quant: bool,
|
||||
) -> torch.Tensor:
|
||||
num_local_experts = w1.size(0)
|
||||
|
||||
def build_expert_map():
|
||||
num_local_experts = w1.size(0)
|
||||
expert_map = torch.full((num_experts,), fill_value=-1, dtype=torch.int32)
|
||||
s = pgi.rank * num_local_experts
|
||||
e = s + num_local_experts
|
||||
expert_map[s:e] = torch.tensor(list(range(num_local_experts)))
|
||||
device = torch.accelerator.current_device_index()
|
||||
return expert_map.to(device=device, dtype=torch.int32)
|
||||
|
||||
hidden_size = test_tensors.rank_tokens.size(1)
|
||||
is_quantized = w1.dtype == current_platform.fp8_dtype()
|
||||
q_dtype = None
|
||||
if is_quantized:
|
||||
q_dtype = current_platform.fp8_dtype()
|
||||
|
||||
out_hidden_states = torch.empty_like(test_tensors.rank_tokens)
|
||||
total_num_tokens = test_tensors.rank_tokens.size(0)
|
||||
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
q_dtype,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
a1_scale=test_tensors.rank_token_scales,
|
||||
)
|
||||
|
||||
# Build the kernel (and its DeepEP buffer) once and reuse it across chunks.
|
||||
# Re-creating it per chunk re-inits rocSHMEM, which only allows one
|
||||
# allocation per process on ROCm. The buffer is sized by max_tokens_per_rank
|
||||
# so it is valid for every chunk (mirrors production's cached all2all handle).
|
||||
mk: FusedMoEKernel = make_modular_kernel(
|
||||
pg,
|
||||
pgi,
|
||||
low_latency_mode,
|
||||
hidden_size,
|
||||
dp_size,
|
||||
num_experts,
|
||||
num_local_experts,
|
||||
q_dtype,
|
||||
use_fp8_dispatch,
|
||||
quant_config,
|
||||
)
|
||||
|
||||
def process_chunk(chunk_start, chunk_end, skip_result_store=False):
|
||||
rank_tokens_chunk = test_tensors.rank_tokens[chunk_start:chunk_end]
|
||||
topk_weights_chunk = test_tensors.topk_weights[chunk_start:chunk_end]
|
||||
topk_chunk = test_tensors.topk[chunk_start:chunk_end]
|
||||
|
||||
if low_latency_mode:
|
||||
# Reusing one buffer leaves it dirty; the low-latency kernels need
|
||||
# the zero-initialized regions reset before each dispatch.
|
||||
mk.prepare_finalize.buffer.clean_low_latency_buffer(
|
||||
MAX_TOKENS_PER_RANK,
|
||||
hidden_size,
|
||||
num_experts,
|
||||
)
|
||||
|
||||
out = mk.apply(
|
||||
hidden_states=rank_tokens_chunk,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights_chunk,
|
||||
topk_ids=topk_chunk,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=build_expert_map(),
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
if not skip_result_store:
|
||||
out_hidden_states[chunk_start:chunk_end, :].copy_(out, non_blocking=True)
|
||||
|
||||
max_num_tokens_per_dp = (
|
||||
MAX_TOKENS_PER_RANK if low_latency_mode else total_num_tokens
|
||||
)
|
||||
|
||||
for chunk_start_ in range(0, total_num_tokens, max_num_tokens_per_dp):
|
||||
chunk_start = chunk_start_
|
||||
chunk_end = min(chunk_start + max_num_tokens_per_dp, total_num_tokens)
|
||||
# clamp start and end
|
||||
chunk_start = min(chunk_start, total_num_tokens - 1)
|
||||
chunk_end = min(chunk_end, total_num_tokens)
|
||||
|
||||
process_chunk(
|
||||
chunk_start, chunk_end, skip_result_store=chunk_start_ >= total_num_tokens
|
||||
)
|
||||
|
||||
return out_hidden_states
|
||||
|
||||
|
||||
def torch_moe_impl(
|
||||
test_tensors: TestTensors,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None,
|
||||
w2_scale: torch.Tensor | None,
|
||||
using_fp8_dispatch: bool,
|
||||
per_act_token_quant: bool,
|
||||
):
|
||||
a, topk_ids, topk_weights = (
|
||||
test_tensors.rank_tokens,
|
||||
test_tensors.topk,
|
||||
test_tensors.topk_weights,
|
||||
)
|
||||
if using_fp8_dispatch:
|
||||
# The DeepEP implementation is requested to dispatch using FP8.
|
||||
# For numerical stability for testing, emulate the fp8 dispatch by
|
||||
# blockwise quant and de-quant.
|
||||
assert not per_act_token_quant
|
||||
a = test_tensors.rank_tokens
|
||||
aq, aq_scale = per_token_group_quant_fp8(a, 128, use_ue8m0=False)
|
||||
a = (
|
||||
(aq.view(-1, 128).to(torch.float32) * aq_scale.view(-1, 1))
|
||||
.view(a.shape)
|
||||
.to(a.dtype)
|
||||
)
|
||||
|
||||
is_quantized = w1.dtype == current_platform.fp8_dtype()
|
||||
a_dtype = a.dtype
|
||||
if is_quantized:
|
||||
w1 = w1.to(dtype=torch.float32) * w1_scale
|
||||
w2 = w2.to(dtype=torch.float32) * w2_scale
|
||||
a = a.to(dtype=torch.float32)
|
||||
|
||||
m, _ = a.shape
|
||||
topk = topk_ids.size(1)
|
||||
out = torch.zeros_like(a)
|
||||
|
||||
for i in range(m):
|
||||
a_i = a[i]
|
||||
o_i = out[i]
|
||||
for j in range(topk):
|
||||
e = topk_ids[i][j]
|
||||
e_w = topk_weights[i][j]
|
||||
w1_e = w1[e]
|
||||
w2_e = w2[e]
|
||||
o_i += (
|
||||
SiluAndMul()(a_i @ w1_e.transpose(0, 1)) @ w2_e.transpose(0, 1)
|
||||
) * e_w
|
||||
|
||||
if is_quantized:
|
||||
out = out.to(dtype=a_dtype)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def assert_deepep_close(
|
||||
expected: torch.Tensor,
|
||||
actual: torch.Tensor,
|
||||
k: int,
|
||||
use_fp8_dispatch: bool,
|
||||
) -> None:
|
||||
torch.testing.assert_close(
|
||||
expected,
|
||||
actual,
|
||||
atol=6e-2,
|
||||
rtol=6e-2,
|
||||
)
|
||||
|
||||
|
||||
def _deep_ep_moe(
|
||||
pgi: ProcessGroupInfo,
|
||||
low_latency_mode: bool,
|
||||
dp_size: int,
|
||||
config: TestConfig,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None,
|
||||
w2_scale: torch.Tensor | None,
|
||||
use_fp8_dispatch: bool,
|
||||
per_act_token_quant: bool,
|
||||
):
|
||||
# Set seed in worker process for deterministic tensor generation.
|
||||
set_random_seed(7)
|
||||
|
||||
device = torch.device(f"cuda:{pgi.local_rank}")
|
||||
init_workspace_manager(device)
|
||||
|
||||
if not low_latency_mode:
|
||||
assert not use_fp8_dispatch, (
|
||||
"FP8 dispatch interface is available only in low-latency mode"
|
||||
)
|
||||
|
||||
is_quantized = w1.dtype == current_platform.fp8_dtype()
|
||||
device_idx = torch.accelerator.current_device_index()
|
||||
w1 = w1.to(device=device_idx)
|
||||
w2 = w2.to(device=device_idx)
|
||||
if is_quantized:
|
||||
assert w1_scale is not None and w2_scale is not None
|
||||
w1_scale = w1_scale.to(device=device_idx)
|
||||
w2_scale = w2_scale.to(device=device_idx)
|
||||
|
||||
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
|
||||
pg = torch.distributed.split_group(
|
||||
split_ranks=[list(range(pgi.world_size))],
|
||||
group_desc="deepep_test",
|
||||
)
|
||||
else:
|
||||
pg = torch.distributed.new_group(list(range(pgi.world_size)))
|
||||
test_tensors = TestTensors.make(config, low_latency_mode)
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
# Reference
|
||||
torch_combined = torch_moe_impl(
|
||||
test_tensors,
|
||||
w1,
|
||||
w2,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
use_fp8_dispatch,
|
||||
per_act_token_quant,
|
||||
)
|
||||
|
||||
# Splice experts for this rank.
|
||||
num_local_experts = config.num_experts // pgi.world_size
|
||||
e_start = num_local_experts * pgi.rank
|
||||
e_end = e_start + num_local_experts
|
||||
w1_ep = w1[e_start:e_end]
|
||||
w2_ep = w2[e_start:e_end]
|
||||
|
||||
w1_scale_ep, w2_scale_ep = None, None
|
||||
if is_quantized:
|
||||
w1_scale_ep = w1_scale[e_start:e_end] # type: ignore
|
||||
w2_scale_ep = w2_scale[e_start:e_end] # type: ignore
|
||||
deepep_combined = deep_ep_moe_impl(
|
||||
pg,
|
||||
pgi,
|
||||
low_latency_mode,
|
||||
dp_size,
|
||||
test_tensors,
|
||||
w1_ep,
|
||||
w2_ep,
|
||||
w1_scale_ep,
|
||||
w2_scale_ep,
|
||||
config.num_experts,
|
||||
use_fp8_dispatch,
|
||||
per_act_token_quant,
|
||||
)
|
||||
|
||||
assert_deepep_close(torch_combined, deepep_combined, config.k, use_fp8_dispatch)
|
||||
|
||||
|
||||
MNKs = [
|
||||
(1, 128, 128),
|
||||
(2, 128, 512),
|
||||
(3, 1024, 2048),
|
||||
(32, 128, 1024),
|
||||
(45, 512, 2048),
|
||||
(64, 1024, 1024),
|
||||
(222, 1024, 2048),
|
||||
]
|
||||
|
||||
DTYPES = [torch.bfloat16, current_platform.fp8_dtype()]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("m,n,k", MNKs)
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("topk", [6])
|
||||
@pytest.mark.parametrize("world_dp_size", [(2, 1)])
|
||||
@pytest.mark.parametrize("per_act_token_quant", [False, True])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@requires_deep_ep
|
||||
def test_deep_ep_moe(
|
||||
dtype: torch.dtype,
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
world_dp_size: tuple[int, int],
|
||||
per_act_token_quant: bool,
|
||||
workspace_init,
|
||||
):
|
||||
low_latency_mode = False
|
||||
use_fp8_dispatch = False
|
||||
|
||||
set_random_seed(7)
|
||||
world_size, dp_size = world_dp_size
|
||||
config = TestConfig(dtype=dtype, topk=topk, m=m, k=k, n=n, num_experts=num_experts)
|
||||
|
||||
w1, w2, w1_scale, w2_scale = make_weights(num_experts, n, k, dtype)
|
||||
|
||||
parallel_launch(
|
||||
world_size,
|
||||
_deep_ep_moe,
|
||||
low_latency_mode,
|
||||
dp_size,
|
||||
config,
|
||||
w1,
|
||||
w2,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
use_fp8_dispatch,
|
||||
per_act_token_quant,
|
||||
)
|
||||
|
||||
|
||||
MNKs = [
|
||||
(1, 128, 2560),
|
||||
(2, 128, 2560),
|
||||
(3, 1024, 2560),
|
||||
(32, 128, 2560),
|
||||
(45, 512, 2560),
|
||||
(64, 1024, 2560),
|
||||
(222, 1024, 2560),
|
||||
]
|
||||
DTYPES = [current_platform.fp8_dtype(), torch.bfloat16]
|
||||
USE_FP8_DISPATCH = [True, False]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("m,n,k", MNKs)
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("topk", [6])
|
||||
@pytest.mark.parametrize("world_dp_size", [(2, 1)])
|
||||
@pytest.mark.parametrize("use_fp8_dispatch", USE_FP8_DISPATCH)
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@requires_deep_ep
|
||||
def test_low_latency_deep_ep_moe(
|
||||
dtype: torch.dtype,
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
world_dp_size: tuple[int, int],
|
||||
use_fp8_dispatch: bool,
|
||||
workspace_init,
|
||||
):
|
||||
low_latency_mode = True
|
||||
|
||||
if low_latency_mode and k not in DeepEPLLPrepareAndFinalize.SUPPORTED_HIDDEN_SIZES:
|
||||
pytest.skip(
|
||||
f"Skipping test as hidden size {k} is not in list of supported "
|
||||
f"hidden sizes {DeepEPLLPrepareAndFinalize.SUPPORTED_HIDDEN_SIZES}"
|
||||
)
|
||||
|
||||
set_random_seed(7)
|
||||
world_size, dp_size = world_dp_size
|
||||
config = TestConfig(dtype=dtype, topk=topk, m=m, k=k, n=n, num_experts=num_experts)
|
||||
|
||||
w1, w2, w1_scale, w2_scale = make_weights(num_experts, n, k, dtype)
|
||||
|
||||
parallel_launch(
|
||||
world_size,
|
||||
_deep_ep_moe,
|
||||
low_latency_mode,
|
||||
dp_size,
|
||||
config,
|
||||
w1,
|
||||
w2,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
use_fp8_dispatch,
|
||||
False,
|
||||
)
|
||||
@@ -1,655 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
|
||||
Compares against a pure-PyTorch reference MoE implementation.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
import torch.distributed
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe import TritonExperts
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.import_utils import has_deep_ep_v2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
from ...utils import multi_gpu_test
|
||||
from .parallel_utils import ProcessGroupInfo, parallel_launch
|
||||
|
||||
if has_deep_ep_v2():
|
||||
from .parallel_utils import DeepEPV2Args, make_deepep_v2_a2a
|
||||
|
||||
requires_deep_ep_v2 = pytest.mark.skipif(
|
||||
not has_deep_ep_v2(),
|
||||
reason="Requires DeepEP v2 (ElasticBuffer)",
|
||||
)
|
||||
|
||||
|
||||
def assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
close = torch.isclose(actual, expected, atol=2e-1, rtol=2e-1)
|
||||
close_fraction = close.float().mean().item()
|
||||
assert close_fraction > 0.99, (
|
||||
f"Only {close_fraction:.1%} of FP8 outputs are within tolerance"
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TestConfig:
|
||||
dtype: torch.dtype
|
||||
topk: int
|
||||
m: int
|
||||
k: int
|
||||
n: int
|
||||
num_experts: int
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TestTensors:
|
||||
rank_tokens: torch.Tensor
|
||||
rank_token_scales: torch.Tensor | None
|
||||
intermediate_scales: torch.Tensor | None
|
||||
topk: torch.Tensor
|
||||
topk_weights: torch.Tensor
|
||||
config: TestConfig
|
||||
|
||||
@staticmethod
|
||||
def make(config: TestConfig) -> "TestTensors":
|
||||
assert config.dtype in [torch.bfloat16, torch.float8_e4m3fn]
|
||||
token_dtype = (
|
||||
torch.bfloat16 if config.dtype == torch.float8_e4m3fn else config.dtype
|
||||
)
|
||||
rank_tokens = (
|
||||
torch.randn((config.m, config.k), device="cuda", dtype=token_dtype) / 10
|
||||
)
|
||||
if config.dtype == torch.float8_e4m3fn:
|
||||
rank_token_scales = torch.tensor(1 / 448, device="cuda")
|
||||
intermediate_scales = torch.tensor(8 / 448, device="cuda")
|
||||
else:
|
||||
rank_token_scales = None
|
||||
intermediate_scales = None
|
||||
|
||||
topk = torch.stack(
|
||||
[
|
||||
torch.randperm(config.num_experts, device="cuda")[: config.topk]
|
||||
for _ in range(config.m)
|
||||
]
|
||||
).to(dtype=torch.int64)
|
||||
topk_weights = torch.randn(topk.shape, dtype=torch.float32, device="cuda")
|
||||
return TestTensors(
|
||||
rank_tokens=rank_tokens,
|
||||
rank_token_scales=rank_token_scales,
|
||||
intermediate_scales=intermediate_scales,
|
||||
topk=topk,
|
||||
topk_weights=topk_weights,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def make_modular_kernel(
|
||||
pg: ProcessGroup,
|
||||
pgi: ProcessGroupInfo,
|
||||
dp_size: int,
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
num_local_experts: int,
|
||||
topk: int,
|
||||
q_dtype: torch.dtype | None,
|
||||
use_fp8_dispatch: bool,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
use_cudagraph: bool = False,
|
||||
) -> FusedMoEKernel:
|
||||
v2_args = DeepEPV2Args(
|
||||
num_local_experts=num_local_experts,
|
||||
num_experts=num_experts,
|
||||
num_topk=topk,
|
||||
hidden_size=hidden_size,
|
||||
max_tokens_per_rank=8192,
|
||||
use_fp8_dispatch=use_fp8_dispatch,
|
||||
)
|
||||
|
||||
a2a = make_deepep_v2_a2a(
|
||||
pg=pg,
|
||||
pgi=pgi,
|
||||
dp_size=dp_size,
|
||||
v2_args=v2_args,
|
||||
use_cudagraph=use_cudagraph,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=num_local_experts,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=hidden_size,
|
||||
)
|
||||
|
||||
fused_experts = TritonExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
mk = FusedMoEKernel(
|
||||
prepare_finalize=a2a,
|
||||
fused_experts=fused_experts,
|
||||
)
|
||||
return mk
|
||||
|
||||
|
||||
def deepep_v2_moe_impl(
|
||||
pg: ProcessGroup,
|
||||
pgi: ProcessGroupInfo,
|
||||
dp_size: int,
|
||||
test_tensors: TestTensors,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None,
|
||||
w2_scale: torch.Tensor | None,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
use_fp8_dispatch: bool,
|
||||
per_act_token_quant: bool,
|
||||
) -> torch.Tensor:
|
||||
num_local_experts = w1.size(0)
|
||||
|
||||
def build_expert_map():
|
||||
expert_map = torch.full((num_experts,), fill_value=-1, dtype=torch.int32)
|
||||
s = pgi.rank * num_local_experts
|
||||
e = s + num_local_experts
|
||||
expert_map[s:e] = torch.tensor(list(range(num_local_experts)))
|
||||
device = torch.accelerator.current_device_index()
|
||||
return expert_map.to(device=device, dtype=torch.int32)
|
||||
|
||||
is_quantized = w1.dtype == torch.float8_e4m3fn
|
||||
q_dtype = torch.float8_e4m3fn if is_quantized else None
|
||||
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
q_dtype,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
a1_scale=test_tensors.rank_token_scales,
|
||||
a2_scale=test_tensors.intermediate_scales,
|
||||
)
|
||||
|
||||
hidden_size = test_tensors.rank_tokens.size(1)
|
||||
|
||||
mk: FusedMoEKernel = make_modular_kernel(
|
||||
pg,
|
||||
pgi,
|
||||
dp_size,
|
||||
hidden_size,
|
||||
num_experts,
|
||||
num_local_experts,
|
||||
topk,
|
||||
q_dtype,
|
||||
use_fp8_dispatch,
|
||||
quant_config,
|
||||
)
|
||||
|
||||
out = mk.apply(
|
||||
hidden_states=test_tensors.rank_tokens,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=test_tensors.topk_weights,
|
||||
topk_ids=test_tensors.topk,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=build_expert_map(),
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def _deep_ep_v2_moe(
|
||||
pgi: ProcessGroupInfo,
|
||||
dp_size: int,
|
||||
config: TestConfig,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None,
|
||||
w2_scale: torch.Tensor | None,
|
||||
use_fp8_dispatch: bool,
|
||||
per_act_token_quant: bool,
|
||||
):
|
||||
device = torch.device(f"cuda:{pgi.local_rank}")
|
||||
init_workspace_manager(device)
|
||||
|
||||
is_quantized = w1.dtype == torch.float8_e4m3fn
|
||||
device_idx = torch.accelerator.current_device_index()
|
||||
w1 = w1.to(device=device_idx)
|
||||
w2 = w2.to(device=device_idx)
|
||||
if is_quantized:
|
||||
assert w1_scale is not None and w2_scale is not None
|
||||
w1_scale = w1_scale.to(device=device_idx)
|
||||
w2_scale = w2_scale.to(device=device_idx)
|
||||
|
||||
pg = torch.distributed.new_group(list(range(pgi.world_size)))
|
||||
# The caller's set_random_seed() only seeds the parent; spawn() gives each
|
||||
# worker a fresh unseeded RNG, so seed here too or the inputs below differ
|
||||
# every run. Offset by rank to keep the ranks' data distinct.
|
||||
set_random_seed(7 + pgi.rank)
|
||||
test_tensors = TestTensors.make(config)
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
# Reference
|
||||
q_dtype = torch.float8_e4m3fn if is_quantized else None
|
||||
torch_combined = torch_experts(
|
||||
test_tensors.rank_tokens,
|
||||
w1,
|
||||
w2,
|
||||
test_tensors.topk_weights,
|
||||
test_tensors.topk,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=test_tensors.rank_token_scales,
|
||||
a2_scale=test_tensors.intermediate_scales,
|
||||
quant_dtype=q_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
)
|
||||
|
||||
# Splice experts for this rank
|
||||
num_local_experts = config.num_experts // pgi.world_size
|
||||
e_start = num_local_experts * pgi.rank
|
||||
e_end = e_start + num_local_experts
|
||||
w1_ep = w1[e_start:e_end]
|
||||
w2_ep = w2[e_start:e_end]
|
||||
|
||||
w1_scale_ep, w2_scale_ep = None, None
|
||||
if is_quantized:
|
||||
w1_scale_ep = w1_scale[e_start:e_end] # type: ignore
|
||||
w2_scale_ep = w2_scale[e_start:e_end] # type: ignore
|
||||
|
||||
deepep_combined = deepep_v2_moe_impl(
|
||||
pg,
|
||||
pgi,
|
||||
dp_size,
|
||||
test_tensors,
|
||||
w1_ep,
|
||||
w2_ep,
|
||||
w1_scale_ep,
|
||||
w2_scale_ep,
|
||||
config.num_experts,
|
||||
config.topk,
|
||||
use_fp8_dispatch,
|
||||
per_act_token_quant,
|
||||
)
|
||||
|
||||
if is_quantized:
|
||||
assert_fp8_close(torch_combined, deepep_combined)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
torch_combined,
|
||||
deepep_combined,
|
||||
atol=6e-2,
|
||||
rtol=6e-2,
|
||||
)
|
||||
|
||||
|
||||
MNKs = [
|
||||
(1, 256, 256),
|
||||
(2, 256, 512),
|
||||
(3, 1024, 2048),
|
||||
(32, 256, 1024),
|
||||
(45, 512, 2048),
|
||||
(64, 1024, 1024),
|
||||
(222, 1024, 2048),
|
||||
]
|
||||
|
||||
DTYPES = [torch.bfloat16, torch.float8_e4m3fn]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("m,n,k", MNKs)
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("topk", [6])
|
||||
@pytest.mark.parametrize("world_dp_size", [(2, 1)])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@requires_deep_ep_v2
|
||||
def test_deep_ep_v2_moe(
|
||||
dtype: torch.dtype,
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
world_dp_size: tuple[int, int],
|
||||
workspace_init,
|
||||
):
|
||||
per_act_token_quant = False
|
||||
use_fp8_dispatch = False
|
||||
|
||||
set_random_seed(7)
|
||||
world_size, dp_size = world_dp_size
|
||||
config = TestConfig(dtype=dtype, topk=topk, m=m, k=k, n=n, num_experts=num_experts)
|
||||
|
||||
quant_dtype = dtype if dtype == torch.float8_e4m3fn else None
|
||||
(_, w1, w1_scale, _), (_, w2, w2_scale, _) = make_test_weights(
|
||||
num_experts,
|
||||
n,
|
||||
k,
|
||||
quant_dtype=quant_dtype,
|
||||
per_out_ch_quant=True,
|
||||
)
|
||||
|
||||
parallel_launch(
|
||||
world_size,
|
||||
_deep_ep_v2_moe,
|
||||
dp_size,
|
||||
config,
|
||||
w1,
|
||||
w2,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
use_fp8_dispatch,
|
||||
per_act_token_quant,
|
||||
)
|
||||
|
||||
|
||||
EXPERTS_BACKENDS = [
|
||||
"flashinfer_trtllm",
|
||||
"flashinfer_cutlass",
|
||||
"trtllm_fp8",
|
||||
]
|
||||
|
||||
|
||||
def _make_experts(
|
||||
experts_backend: str,
|
||||
config: TestConfig,
|
||||
moe_config,
|
||||
num_local_experts: int,
|
||||
rank: int,
|
||||
w1_bf16: torch.Tensor,
|
||||
w2_bf16: torch.Tensor,
|
||||
test_tensors: TestTensors,
|
||||
):
|
||||
e_start = num_local_experts * rank
|
||||
e_end = e_start + num_local_experts
|
||||
|
||||
if experts_backend != "trtllm_fp8":
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
backend_to_kernel_cls,
|
||||
convert_to_unquantized_kernel_format,
|
||||
map_unquantized_backend,
|
||||
)
|
||||
|
||||
torch_combined = torch_experts(
|
||||
test_tensors.rank_tokens,
|
||||
w1_bf16,
|
||||
w2_bf16,
|
||||
test_tensors.topk_weights,
|
||||
test_tensors.topk,
|
||||
)
|
||||
backend = map_unquantized_backend(experts_backend)
|
||||
w1_ep, w2_ep = convert_to_unquantized_kernel_format(
|
||||
backend,
|
||||
moe_config,
|
||||
w1_bf16[e_start:e_end],
|
||||
w2_bf16[e_start:e_end],
|
||||
)
|
||||
experts_cls = next(
|
||||
cls
|
||||
for cls in backend_to_kernel_cls(backend)
|
||||
if issubclass(cls, mk.FusedMoEExpertsModular)
|
||||
)
|
||||
fused_experts = experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
)
|
||||
return fused_experts, w1_ep, w2_ep, torch_combined, 1e-1, 2e-1
|
||||
|
||||
from tests.kernels.moe.test_moe_layer import _quantize_fp8_halves
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
|
||||
TrtLlmFp8ExpertsModular,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
)
|
||||
|
||||
w1_ref = w1_bf16.to(torch.float8_e4m3fn).to(torch.bfloat16)
|
||||
w2_ref = w2_bf16.to(torch.float8_e4m3fn).to(torch.bfloat16)
|
||||
|
||||
block_shape = [128, 128]
|
||||
qw = _quantize_fp8_halves(w1_ref, w2_ref, block_shape)
|
||||
assert qw.w13_weight_scale is not None
|
||||
assert qw.w2_weight_scale is not None
|
||||
|
||||
reference_topk_weights = test_tensors.topk_weights.to(torch.bfloat16).to(
|
||||
torch.float32
|
||||
)
|
||||
torch_combined = torch_experts(
|
||||
test_tensors.rank_tokens,
|
||||
qw.w13_weight,
|
||||
qw.w2_weight,
|
||||
reference_topk_weights,
|
||||
test_tensors.topk,
|
||||
w1_scale=qw.w13_weight_scale,
|
||||
w2_scale=qw.w2_weight_scale,
|
||||
quant_dtype=torch.float8_e4m3fn,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
|
||||
class _MockLayer:
|
||||
weight_block_size = block_shape
|
||||
|
||||
class moe_config:
|
||||
is_act_and_mul = True
|
||||
intermediate_size_per_partition = config.n
|
||||
|
||||
class activation:
|
||||
is_gated = True
|
||||
|
||||
w1_ep, w2_ep, w1_scale_ep, w2_scale_ep = convert_to_fp8_moe_kernel_format(
|
||||
fp8_backend=Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
layer=_MockLayer(),
|
||||
w13=qw.w13_weight[e_start:e_end],
|
||||
w2=qw.w2_weight[e_start:e_end],
|
||||
w13_scale=qw.w13_weight_scale[e_start:e_end],
|
||||
w2_scale=qw.w2_weight_scale[e_start:e_end],
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
|
||||
fused_experts = TrtLlmFp8ExpertsModular(
|
||||
moe_config=moe_config,
|
||||
quant_config=FusedMoEQuantConfig.make(
|
||||
torch.float8_e4m3fn,
|
||||
block_shape=block_shape,
|
||||
w1_scale=w1_scale_ep,
|
||||
w2_scale=w2_scale_ep,
|
||||
),
|
||||
)
|
||||
return fused_experts, w1_ep, w2_ep, torch_combined, 6e-2, 6e-2
|
||||
|
||||
|
||||
def _deep_ep_v2_moe_backends(
|
||||
pgi: ProcessGroupInfo,
|
||||
dp_size: int,
|
||||
config: TestConfig,
|
||||
use_cudagraph: bool,
|
||||
experts_backend: str,
|
||||
):
|
||||
import tempfile
|
||||
|
||||
from vllm.config import KernelConfig
|
||||
from vllm.distributed import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
|
||||
device = torch.device(f"cuda:{pgi.local_rank}")
|
||||
init_workspace_manager(device)
|
||||
|
||||
pg = torch.distributed.new_group(list(range(pgi.world_size)))
|
||||
set_random_seed(7 + pgi.rank)
|
||||
test_tensors = TestTensors.make(config)
|
||||
num_local_experts = config.num_experts // pgi.world_size
|
||||
hidden_size = config.k
|
||||
|
||||
# All ranks must use the same global weights before taking their EP slice.
|
||||
w1_bf16 = (
|
||||
torch.randn(
|
||||
(config.num_experts, 2 * config.n, config.k),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
/ 15
|
||||
)
|
||||
w2_bf16 = (
|
||||
torch.randn(
|
||||
(config.num_experts, config.k, config.n),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
/ 15
|
||||
)
|
||||
torch.distributed.broadcast(w1_bf16, src=0, group=pg)
|
||||
torch.distributed.broadcast(w2_bf16, src=0, group=pg)
|
||||
|
||||
vllm_cfg = VllmConfig()
|
||||
vllm_cfg.kernel_config = KernelConfig(moe_backend="flashinfer_trtllm")
|
||||
|
||||
with set_current_vllm_config(vllm_cfg):
|
||||
temp_file = tempfile.mktemp()
|
||||
init_distributed_environment(
|
||||
world_size=pgi.world_size,
|
||||
rank=pgi.rank,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
local_rank=pgi.local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
initialize_model_parallel(tensor_model_parallel_size=1)
|
||||
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=config.num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
experts_per_token=config.topk,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=config.n,
|
||||
)
|
||||
moe_config = dataclasses.replace(
|
||||
moe_config,
|
||||
moe_parallel_config=dataclasses.replace(
|
||||
moe_config.moe_parallel_config,
|
||||
ep_size=pgi.world_size,
|
||||
ep_rank=pgi.rank,
|
||||
use_ep=True,
|
||||
all2all_backend="deepep_v2",
|
||||
),
|
||||
)
|
||||
|
||||
(
|
||||
fused_experts,
|
||||
w1_ep,
|
||||
w2_ep,
|
||||
torch_combined,
|
||||
atol,
|
||||
rtol,
|
||||
) = _make_experts(
|
||||
experts_backend,
|
||||
config,
|
||||
moe_config,
|
||||
num_local_experts,
|
||||
pgi.rank,
|
||||
w1_bf16,
|
||||
w2_bf16,
|
||||
test_tensors,
|
||||
)
|
||||
|
||||
v2_args = DeepEPV2Args(
|
||||
num_local_experts=num_local_experts,
|
||||
num_experts=config.num_experts,
|
||||
num_topk=config.topk,
|
||||
hidden_size=hidden_size,
|
||||
max_tokens_per_rank=8192,
|
||||
use_fp8_dispatch=False,
|
||||
)
|
||||
a2a = make_deepep_v2_a2a(
|
||||
pg=pg,
|
||||
pgi=pgi,
|
||||
dp_size=dp_size,
|
||||
v2_args=v2_args,
|
||||
use_cudagraph=use_cudagraph,
|
||||
)
|
||||
mk_kernel = FusedMoEKernel(
|
||||
prepare_finalize=a2a,
|
||||
fused_experts=fused_experts,
|
||||
)
|
||||
|
||||
with set_forward_context(None, vllm_cfg):
|
||||
for _ in range(3):
|
||||
out = mk_kernel.apply(
|
||||
hidden_states=test_tensors.rank_tokens,
|
||||
w1=w1_ep,
|
||||
w2=w2_ep,
|
||||
topk_weights=test_tensors.topk_weights,
|
||||
topk_ids=test_tensors.topk,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=config.num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(torch_combined, out, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", [(32, 256, 1024)])
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("topk", [6])
|
||||
@pytest.mark.parametrize("world_dp_size", [(2, 1)])
|
||||
@pytest.mark.parametrize("experts_backend", EXPERTS_BACKENDS)
|
||||
@pytest.mark.parametrize("use_cudagraph", [True, False])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@requires_deep_ep_v2
|
||||
@pytest.mark.skipif(
|
||||
not has_flashinfer() or not current_platform.has_device_capability(100),
|
||||
reason="Requires FlashInfer TRTLLM fused MoE (SM100)",
|
||||
)
|
||||
def test_deep_ep_v2_moe_backends(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
world_dp_size: tuple[int, int],
|
||||
experts_backend: str,
|
||||
use_cudagraph: bool,
|
||||
workspace_init,
|
||||
):
|
||||
set_random_seed(7)
|
||||
world_size, dp_size = world_dp_size
|
||||
config = TestConfig(
|
||||
dtype=torch.float8_e4m3fn
|
||||
if experts_backend == "trtllm_fp8"
|
||||
else torch.bfloat16,
|
||||
topk=topk,
|
||||
m=m,
|
||||
k=k,
|
||||
n=n,
|
||||
num_experts=num_experts,
|
||||
)
|
||||
|
||||
parallel_launch(
|
||||
world_size,
|
||||
_deep_ep_v2_moe_backends,
|
||||
dp_size,
|
||||
config,
|
||||
use_cudagraph,
|
||||
experts_backend,
|
||||
)
|
||||
@@ -1,634 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
|
||||
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# vLLM fused-expert reference (Triton fallback + DeepGEMM option)
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
MoEActivation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoEQuantDesc,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
|
||||
deepgemm_moe_permute,
|
||||
ep_gather,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||
_fp8_workspace_shape,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.utils.deep_gemm import (
|
||||
calc_diff,
|
||||
is_deep_gemm_supported,
|
||||
per_block_cast_to_fp8,
|
||||
)
|
||||
|
||||
BLOCK_SIZE = [128, 128]
|
||||
|
||||
|
||||
def test_ep_gather_uses_64_bit_row_offsets():
|
||||
hidden_size = 4096
|
||||
source_row = 1 << 18
|
||||
input_tensor = torch.empty(
|
||||
(source_row + 1, hidden_size),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
input_tensor[source_row].fill_(1)
|
||||
output = torch.empty((1, hidden_size), device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
ep_gather(
|
||||
input_tensor=input_tensor,
|
||||
recv_topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
|
||||
recv_topk_weight=torch.ones((1, 1), device="cuda", dtype=torch.float32),
|
||||
input_index=torch.full((1, 1), source_row, device="cuda", dtype=torch.int32),
|
||||
expert_map=None,
|
||||
output_tensor=output,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_ep_gather_uses_64_bit_output_row_offsets():
|
||||
hidden_size = 6144
|
||||
output_row = (1 << 31) // hidden_size + 1
|
||||
num_rows = output_row + 1
|
||||
output = torch.empty(
|
||||
(num_rows, hidden_size),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
ep_gather(
|
||||
input_tensor=torch.ones((1, hidden_size), device="cuda", dtype=torch.bfloat16),
|
||||
recv_topk_ids=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int64),
|
||||
recv_topk_weight=torch.ones((num_rows, 1), device="cuda", dtype=torch.float32),
|
||||
input_index=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int32),
|
||||
expert_map=None,
|
||||
output_tensor=output,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("num_columns", [2048, 6144, 6145])
|
||||
def test_fp8_workspace_shape(workspace_dtype, num_columns):
|
||||
num_rows = 17
|
||||
shape = _fp8_workspace_shape(num_rows, num_columns, workspace_dtype)
|
||||
|
||||
allocated_bytes = math.prod(shape) * workspace_dtype.itemsize
|
||||
required_bytes = num_rows * num_columns * torch.float8_e4m3fn.itemsize
|
||||
|
||||
assert allocated_bytes >= required_bytes
|
||||
assert allocated_bytes - required_bytes < num_rows * workspace_dtype.itemsize
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init):
|
||||
hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16)
|
||||
activations, scales = per_token_group_quant_fp8(
|
||||
hidden_states,
|
||||
group_size=128,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
topk_ids = torch.tensor([[0], [1]], device="cuda", dtype=torch.int64)
|
||||
|
||||
_, permuted_scales, expert_ids, _, _ = deepgemm_moe_permute(
|
||||
aq=activations,
|
||||
aq_scale=scales,
|
||||
topk_ids=topk_ids,
|
||||
local_num_experts=2,
|
||||
expert_map=None,
|
||||
expert_tokens_meta=None,
|
||||
)
|
||||
|
||||
padding = expert_ids < 0
|
||||
assert padding.any()
|
||||
torch.testing.assert_close(
|
||||
permuted_scales[padding],
|
||||
torch.zeros_like(permuted_scales[padding]),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
def make_block_quant_fp8_weights(
|
||||
e: int,
|
||||
n: int,
|
||||
k: int,
|
||||
block_size: list[int],
|
||||
):
|
||||
"""Generate (w1, w2) expert weights and their per-block scale tensors
|
||||
in FP8 block-quantized format.
|
||||
|
||||
w1 shape: (E, 2N, K)
|
||||
w2 shape: (E, K, N)
|
||||
"""
|
||||
dtype = torch.bfloat16
|
||||
fp8_max, fp8_min = (
|
||||
torch.finfo(torch.float8_e4m3fn).max,
|
||||
torch.finfo(torch.float8_e4m3fn).min,
|
||||
)
|
||||
|
||||
# bf16 reference weights
|
||||
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) / 10
|
||||
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) / 10
|
||||
w1_bf16.clamp_(fp8_min, fp8_max)
|
||||
w2_bf16.clamp_(fp8_min, fp8_max)
|
||||
|
||||
block_n, block_k = block_size
|
||||
n_tiles_w1 = math.ceil((2 * n) / block_n)
|
||||
k_tiles_w1 = math.ceil(k / block_k)
|
||||
n_tiles_w2 = math.ceil(k / block_n)
|
||||
k_tiles_w2 = math.ceil(n / block_k)
|
||||
|
||||
w1 = torch.empty_like(w1_bf16, dtype=torch.float8_e4m3fn)
|
||||
w2 = torch.empty_like(w2_bf16, dtype=torch.float8_e4m3fn)
|
||||
w1_s = torch.empty(e, n_tiles_w1, k_tiles_w1, device="cuda", dtype=torch.float32)
|
||||
w2_s = torch.empty(e, n_tiles_w2, k_tiles_w2, device="cuda", dtype=torch.float32)
|
||||
|
||||
for i in range(e):
|
||||
w1[i], w1_s[i] = per_block_cast_to_fp8(
|
||||
w1_bf16[i], block_size=block_size, use_ue8m0=True
|
||||
)
|
||||
w2[i], w2_s[i] = per_block_cast_to_fp8(
|
||||
w2_bf16[i], block_size=block_size, use_ue8m0=True
|
||||
)
|
||||
|
||||
return w1, w2, w1_s, w2_s
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [16, 129, 512])
|
||||
@pytest.mark.parametrize("with_metadata", [False, True])
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_block_fp8_padding_on_graph_replay(num_tokens, with_metadata, monkeypatch):
|
||||
"""Both FP8 GEMMs consume only live rows as routing changes on replay."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe as impl
|
||||
from vllm.utils.deep_gemm import DeepGemmQuantScaleFMT
|
||||
|
||||
if not (
|
||||
impl.current_platform.is_device_capability_family(90)
|
||||
or (
|
||||
impl.current_platform.is_device_capability_family(100)
|
||||
and DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0
|
||||
)
|
||||
):
|
||||
pytest.skip("Prefix-sum FP8 dispatch requires SM90 or SM10.x")
|
||||
|
||||
torch.manual_seed(0)
|
||||
e, n, k, topk = 8, 1024, 512, 2
|
||||
x = torch.randn(num_tokens, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
|
||||
w1, w2, s1, s2 = make_block_quant_fp8_weights(e, n, k, BLOCK_SIZE)
|
||||
quant = fp8_w8a8_moe_quant_config(w1_scale=s1, w2_scale=s2, block_shape=BLOCK_SIZE)
|
||||
experts = impl.DeepGemmExperts(make_dummy_moe_config(), quant)
|
||||
ids = torch.zeros(num_tokens, topk, device="cuda", dtype=torch.int64)
|
||||
weights = torch.full((num_tokens, topk), 0.5, device="cuda")
|
||||
counts = torch.zeros(e, device="cuda", dtype=torch.int32)
|
||||
metadata = mk.ExpertTokensMetadata(counts, None) if with_metadata else None
|
||||
shapes = experts.workspace_shapes(
|
||||
num_tokens, 2 * n, k, topk, e, e, metadata, MoEActivation.SILU
|
||||
)
|
||||
|
||||
def route(mode):
|
||||
ids[:, 0] = torch.arange(num_tokens, device="cuda") % e
|
||||
ids[:, 1] = (ids[:, 0] + 1) % e
|
||||
if mode == 1:
|
||||
ids.fill_(-1)
|
||||
ids[:1, 0] = e - 1
|
||||
elif mode == 2:
|
||||
ids.fill_(-1)
|
||||
counts.copy_(torch.bincount(ids[ids >= 0], minlength=e).int())
|
||||
|
||||
route(0)
|
||||
variants = []
|
||||
for dense in (True, False):
|
||||
ws1, ws2, output = [
|
||||
torch.empty(shape, device="cuda", dtype=torch.bfloat16) for shape in shapes
|
||||
]
|
||||
|
||||
def invoke(output=output, ws1=ws1, ws2=ws2):
|
||||
experts.apply(
|
||||
output,
|
||||
aq,
|
||||
w1,
|
||||
w2,
|
||||
weights,
|
||||
ids,
|
||||
MoEActivation.SILU,
|
||||
e,
|
||||
None,
|
||||
scales,
|
||||
None,
|
||||
ws1,
|
||||
ws2,
|
||||
metadata,
|
||||
False,
|
||||
)
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
if dense:
|
||||
platform = Mock(wraps=impl.current_platform)
|
||||
platform.is_device_capability_family.return_value = False
|
||||
mp.setattr(impl, "current_platform", platform)
|
||||
gemm = impl.m_grouped_fp8_gemm_nt_contiguous
|
||||
calls: list[int] = []
|
||||
|
||||
def check_layout(*args, dense=dense, calls=calls, gemm=gemm, **kwargs):
|
||||
assert kwargs.get("use_psum_layout", False) == (not dense)
|
||||
calls.append(args[3].numel())
|
||||
return gemm(*args, **kwargs)
|
||||
|
||||
mp.setattr(impl, "m_grouped_fp8_gemm_nt_contiguous", check_layout)
|
||||
invoke()
|
||||
assert len(calls) == 2
|
||||
if not dense:
|
||||
assert calls == [e, e]
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
invoke()
|
||||
variants.append((graph, output, ws1, ws2))
|
||||
|
||||
for mode in (0, 1, 2, 0):
|
||||
route(mode)
|
||||
for graph, _, ws1, ws2 in variants:
|
||||
ws1.fill_(float("nan"))
|
||||
ws2.fill_(float("nan"))
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(variants[0][1], variants[1][1], rtol=0, atol=0)
|
||||
assert torch.isfinite(variants[1][1]).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("n", "k"), [(2048, 512), (512, 1024)])
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_prefix_sum_leaves_unused_capacity_untouched(n, k):
|
||||
"""Both projection shapes skip unused tiles as counts change on replay."""
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
m_grouped_fp8_gemm_nt_contiguous,
|
||||
mk_alignment_scope,
|
||||
)
|
||||
|
||||
if not (
|
||||
current_platform.is_device_capability_family(90)
|
||||
or current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip("Requires SM90 or SM10.x")
|
||||
|
||||
capacity, experts, alignment = 512, 4, 128
|
||||
x = torch.randn(capacity, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
|
||||
w, _, ws, _ = make_block_quant_fp8_weights(experts, n // 2, k, BLOCK_SIZE)
|
||||
ends = torch.tensor([0, 1, 128, 129], device="cuda", dtype=torch.int32)
|
||||
output = torch.empty(capacity, n, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
def invoke():
|
||||
with mk_alignment_scope(alignment):
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(aq, scales), (w, ws), output, ends, use_psum_layout=True
|
||||
)
|
||||
|
||||
invoke()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
invoke()
|
||||
for bounds, allocated_end in (([0, 1, 128, 129], 256), ([0] * 4, 0)):
|
||||
ends.copy_(torch.tensor(bounds, device="cuda", dtype=torch.int32))
|
||||
output.fill_(91)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(output[allocated_end:] == 91)
|
||||
if allocated_end:
|
||||
assert torch.all(output[[0, 128]] != 91)
|
||||
|
||||
|
||||
def run_single_case(m, n, k, topk, num_experts, block_size):
|
||||
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
|
||||
Triton baseline within tolerance.
|
||||
"""
|
||||
tokens_bf16 = (
|
||||
torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
|
||||
.clamp_min_(-1)
|
||||
.clamp_max_(1)
|
||||
)
|
||||
_, a1_scale = per_token_group_quant_fp8(tokens_bf16, block_size[1])
|
||||
|
||||
# expert weight tensors
|
||||
w1, w2, w1_s, w2_s = make_block_quant_fp8_weights(num_experts, n, k, block_size)
|
||||
|
||||
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
|
||||
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
|
||||
|
||||
quant_config = fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
a1_scale=a1_scale,
|
||||
block_shape=block_size,
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
deep_gemm_experts = mk.FusedMoEKernel(
|
||||
prepare_finalize=maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
fused_experts=TritonOrDeepGemmExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
),
|
||||
)
|
||||
|
||||
# triton reference
|
||||
out_triton = fused_experts(
|
||||
hidden_states=tokens_bf16,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
# DeepGemm
|
||||
out_deepgemm = deep_gemm_experts.apply(
|
||||
hidden_states=tokens_bf16,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
diff = calc_diff(out_deepgemm, out_triton)
|
||||
assert diff < 0.001, f"Diff exceeded 1%: {diff}"
|
||||
|
||||
|
||||
# Note: N <= 512 will disable the deepgemm path due to performance issues.
|
||||
MNKs = [
|
||||
(1024, 768, 128),
|
||||
(2048, 768, 512),
|
||||
(512, 1024, 1024),
|
||||
(4096, 4096, 1024),
|
||||
]
|
||||
|
||||
TOPKS = [2, 6]
|
||||
NUM_EXPERTS = [32]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("m", "n", "k"), MNKs)
|
||||
@pytest.mark.parametrize("topk", TOPKS)
|
||||
@pytest.mark.parametrize("num_experts", NUM_EXPERTS)
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_vs_triton(m, n, k, topk, num_experts, monkeypatch, workspace_init):
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
|
||||
|
||||
_DeepGemmExperts = importlib.import_module(
|
||||
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
|
||||
).DeepGemmExperts
|
||||
|
||||
call_counter = {"cnt": 0}
|
||||
|
||||
orig_fn = _DeepGemmExperts.apply
|
||||
|
||||
def _spy_apply(*args, **kwargs):
|
||||
call_counter["cnt"] += 1
|
||||
return orig_fn(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(_DeepGemmExperts, "apply", _spy_apply)
|
||||
if topk > num_experts:
|
||||
pytest.skip(f"topk={topk} > num_experts={num_experts}")
|
||||
|
||||
run_single_case(
|
||||
m=m,
|
||||
n=n,
|
||||
k=k,
|
||||
topk=topk,
|
||||
num_experts=num_experts,
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
# ensure that the DeepGEMM path was indeed taken.
|
||||
assert call_counter["cnt"] == 1, (
|
||||
f"DeepGEMM path was not executed during the test. "
|
||||
f"Call counter: {call_counter['cnt']}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FP4 weight tests (DeepGEMM m_grouped_fp8_fp4_gemm_nt_contiguous)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_mxfp4_weights(
|
||||
e: int,
|
||||
n: int,
|
||||
k: int,
|
||||
):
|
||||
"""Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
|
||||
plus BF16 reference weights for validation.
|
||||
|
||||
w1 shape: (E, 2N, K//2) uint8 — packed FP4
|
||||
w2 shape: (E, K, N//2) uint8 — packed FP4
|
||||
w1_s shape: (E, 2N, K//32) float32 — per-row block-32 scales
|
||||
w2_s shape: (E, K, N//32) float32 — per-row block-32 scales
|
||||
w1_bf16: (E, 2N, K) — original BF16 for reference
|
||||
w2_bf16: (E, K, N) — original BF16 for reference
|
||||
"""
|
||||
from deep_gemm.utils.math import per_token_cast_to_fp4
|
||||
|
||||
dtype = torch.bfloat16
|
||||
gran_k = 32 # MXFP4 block size
|
||||
|
||||
# bf16 reference weights — scale by 1/sqrt(dim) for numerical stability
|
||||
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) * (k**-0.5)
|
||||
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) * (n**-0.5)
|
||||
|
||||
# Quantize per-expert to FP4
|
||||
w1 = torch.empty(e, 2 * n, k // 2, device="cuda", dtype=torch.uint8)
|
||||
w2 = torch.empty(e, k, n // 2, device="cuda", dtype=torch.uint8)
|
||||
w1_s = torch.empty(
|
||||
e, 2 * n, math.ceil(k / gran_k), device="cuda", dtype=torch.float32
|
||||
)
|
||||
w2_s = torch.empty(e, k, math.ceil(n / gran_k), device="cuda", dtype=torch.float32)
|
||||
|
||||
for i in range(e):
|
||||
w1[i], w1_s[i] = per_token_cast_to_fp4(
|
||||
w1_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
|
||||
)
|
||||
w2[i], w2_s[i] = per_token_cast_to_fp4(
|
||||
w2_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
|
||||
)
|
||||
|
||||
return w1, w2, w1_s, w2_s, w1_bf16, w2_bf16
|
||||
|
||||
|
||||
def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
|
||||
"""BF16 token-loop MoE reference for correctness testing."""
|
||||
import torch.nn.functional as F
|
||||
|
||||
num_tokens, hidden_size = x.shape
|
||||
intermediate = w1.shape[1] // 2
|
||||
top_k = topk_ids.shape[1]
|
||||
|
||||
output = torch.zeros(num_tokens, hidden_size, dtype=torch.float32, device=x.device)
|
||||
for t in range(num_tokens):
|
||||
for kk in range(top_k):
|
||||
e = topk_ids[t, kk].item()
|
||||
w = topk_weights[t, kk].item()
|
||||
fc1 = x[t : t + 1].float() @ w1[e].float().T
|
||||
linear = fc1[:, :intermediate]
|
||||
gate = fc1[:, intermediate:]
|
||||
act = F.silu(gate) * linear
|
||||
fc2 = act @ w2[e].float().T
|
||||
output[t] += w * fc2[0]
|
||||
return output.to(torch.bfloat16)
|
||||
|
||||
|
||||
def run_single_fp4_case(m, n, k, topk, num_experts):
|
||||
"""Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
|
||||
DeepGEMM FP4 == BF16 reference within tolerance.
|
||||
"""
|
||||
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
|
||||
|
||||
# FP4 expert weight tensors + BF16 originals for reference
|
||||
w1, w2, w1_s, w2_s, w1_bf16, w2_bf16 = make_mxfp4_weights(num_experts, n, k)
|
||||
|
||||
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
|
||||
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_fp8_dtype = current_platform.fp8_dtype()
|
||||
_block_shape = GroupShape(128, 128)
|
||||
quant_config = FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
|
||||
_a2=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
|
||||
_w1=FusedMoEQuantDesc("mxfp4", None, w1_s, None, None, None),
|
||||
_w2=FusedMoEQuantDesc("mxfp4", None, w2_s, None, None, None),
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||
DeepGemmFP4Experts,
|
||||
)
|
||||
|
||||
deep_gemm_fp4_experts = mk.FusedMoEKernel(
|
||||
prepare_finalize=maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
fused_experts=DeepGemmFP4Experts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
),
|
||||
)
|
||||
|
||||
# DeepGEMM FP4 path
|
||||
out_deepgemm_fp4 = deep_gemm_fp4_experts.apply(
|
||||
hidden_states=tokens_bf16,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
# BF16 reference using the same original weights
|
||||
out_ref = _bf16_moe_reference(tokens_bf16, w1_bf16, w2_bf16, topk_weights, topk_ids)
|
||||
|
||||
# FP4 vs BF16 reference: quantization error from FP4 weights + FP8 activations
|
||||
diff = calc_diff(out_deepgemm_fp4, out_ref)
|
||||
assert diff < 0.05, f"FP4 diff exceeded 5%: {diff}"
|
||||
|
||||
|
||||
# DeepSeek V4 dims: H=4096, I=2048, so N=2*I=4096, K=H=4096.
|
||||
# FP4 quantization with block_k=32 needs large K for good accuracy.
|
||||
FP4_MNKs = [
|
||||
(128, 4096, 4096), # DeepSeek V4 shape
|
||||
(256, 2048, 2048), # Half-size variant
|
||||
(128, 384, 3584), # Kimi-K3 TP8 latent-MoE shape
|
||||
]
|
||||
|
||||
FP4_TOPKS = [2]
|
||||
FP4_NUM_EXPERTS = [8]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("m", "n", "k"), FP4_MNKs)
|
||||
@pytest.mark.parametrize("topk", FP4_TOPKS)
|
||||
@pytest.mark.parametrize("num_experts", FP4_NUM_EXPERTS)
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_fp4_vs_triton(
|
||||
m, n, k, topk, num_experts, monkeypatch, workspace_init
|
||||
):
|
||||
pytest.importorskip("deep_gemm.utils.math")
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
|
||||
|
||||
_DeepGemmFP4Experts = importlib.import_module(
|
||||
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
|
||||
).DeepGemmFP4Experts
|
||||
|
||||
call_counter = {"cnt": 0}
|
||||
|
||||
orig_fn = _DeepGemmFP4Experts.apply
|
||||
|
||||
def _spy_apply(*args, **kwargs):
|
||||
call_counter["cnt"] += 1
|
||||
return orig_fn(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(_DeepGemmFP4Experts, "apply", _spy_apply)
|
||||
if topk > num_experts:
|
||||
pytest.skip(f"topk={topk} > num_experts={num_experts}")
|
||||
|
||||
run_single_fp4_case(
|
||||
m=m,
|
||||
n=n,
|
||||
k=k,
|
||||
topk=topk,
|
||||
num_experts=num_experts,
|
||||
)
|
||||
|
||||
# ensure that the DeepGEMM FP4 path was indeed taken.
|
||||
assert call_counter["cnt"] == 1, (
|
||||
f"DeepGEMM FP4 path was not executed during the test. "
|
||||
f"Call counter: {call_counter['cnt']}"
|
||||
)
|
||||
@@ -1,354 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_device_capability_family(120):
|
||||
pytest.skip(
|
||||
reason="FlashInfer B12x MoE requires SM120 (RTX Pro 6000 / DGX Spark).",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.utils.flashinfer import has_flashinfer_b12x_moe
|
||||
|
||||
if not has_flashinfer_b12x_moe():
|
||||
pytest.skip(
|
||||
reason=(
|
||||
"FlashInfer B12xMoEWrapper not available in installed "
|
||||
"FlashInfer (needs PR #3080)."
|
||||
),
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
# Import fp4_quantize after the skip guard — FlashInfer must be installed.
|
||||
from flashinfer.fp4_quantization import fp4_quantize
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from tests.kernels.utils import torch_moe
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import nvfp4_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_b12x_moe import (
|
||||
FlashInferB12xExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
reorder_w1w3_to_w3w1,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
# Dimensions chosen to satisfy FP4 alignment requirements (k multiple of 256,
|
||||
# n multiple of 128) while keeping tests fast.
|
||||
MNK_FACTORS = [
|
||||
(2, 128, 256),
|
||||
(2, 256, 512),
|
||||
(16, 128, 256),
|
||||
(64, 256, 512),
|
||||
]
|
||||
|
||||
|
||||
def _process_b12x_weights(
|
||||
experts: FlashInferB12xExperts,
|
||||
w1_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w1_scale_2: torch.Tensor,
|
||||
w2_scale_2: torch.Tensor,
|
||||
) -> None:
|
||||
layer = SimpleNamespace(
|
||||
w13_weight_scale=w1_scale,
|
||||
w13_weight_scale_2=w1_scale_2,
|
||||
w2_weight_scale=w2_scale,
|
||||
w2_weight_scale_2=w2_scale_2,
|
||||
)
|
||||
experts.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", [8, 16])
|
||||
@pytest.mark.parametrize("topk", [1, 2, 4])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_b12x_moe(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
e: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
workspace_init,
|
||||
):
|
||||
"""Test FlashInferB12xExperts against a BF16 torch reference.
|
||||
|
||||
The SM12x kernel takes BF16 hidden states directly and fuses token
|
||||
dispatch, W1 GEMM, SwiGLU, and W2 GEMM into one call. We verify
|
||||
correctness against ``torch_moe`` using generous tolerances to account
|
||||
for the internal FP4 quantization of activations and weights.
|
||||
|
||||
Scale convention
|
||||
----------------
|
||||
The SM12x kernel uses ``w1_alpha`` as *both* the activation-quantisation
|
||||
global scale and the weight dequantisation factor. These two roles are
|
||||
conflated into a single parameter in ``launch_sm120_moe``, so they must
|
||||
equal the same value. We use ``global_scale = 1.0`` for
|
||||
``fp4_quantize`` so that ``w1_alpha = ones`` satisfies both roles
|
||||
simultaneously. The alternative — vLLM's convention of baking a large
|
||||
``w_gs`` into block-scale values and compensating with
|
||||
``g1_alphas = 1/w_gs`` — is incompatible with this kernel.
|
||||
"""
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
|
||||
# Generate BF16 reference weights in [gate, up] order.
|
||||
# Shape: w1=(e, 2n, k), w2=(e, k, n).
|
||||
w1_bf16 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 15
|
||||
w2_bf16 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 15
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Quantise weights for the SM12x kernel using FlashInfer's convention:
|
||||
# global_scale = 1.0 → block_scale = max_abs_block / fp4_max
|
||||
# w1_alpha = 1.0 (no extra global factor to compensate)
|
||||
#
|
||||
# The scale factors returned by fp4_quantize(..., is_sf_swizzled_layout=True)
|
||||
# are already in the swizzled 2D layout expected by convert_sf_to_mma_layout.
|
||||
# No additional swizzle_blockscale() call is needed.
|
||||
# ------------------------------------------------------------------ #
|
||||
gs = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
sf_vec_size = 16
|
||||
|
||||
# W1: reorder BF16 from [gate, up] → [up, gate], then quantise.
|
||||
# Note: in reorder_w1w3_to_w3w1, "w1" refers to the gate projection
|
||||
# and "w3" refers to the up projection.
|
||||
# A dummy scale is passed and discarded; real scales come from
|
||||
# fp4_quantize after reordering.
|
||||
w1_reordered, _ = reorder_w1w3_to_w3w1(
|
||||
w1_bf16.clone(),
|
||||
torch.ones((e, 2 * n, 1), device="cuda", dtype=torch.float32),
|
||||
)
|
||||
w1_flat = w1_reordered.reshape(e * 2 * n, k)
|
||||
w1_q_flat, w1_sf_flat = fp4_quantize(
|
||||
w1_flat,
|
||||
global_scale=gs,
|
||||
sf_vec_size=sf_vec_size,
|
||||
is_sf_swizzled_layout=True,
|
||||
)
|
||||
w1_q = w1_q_flat.view(e, 2 * n, k // 2) # uint8, packed FP4
|
||||
w1_blockscale = w1_sf_flat.view(e, 2 * n, w1_sf_flat.shape[1]) # float8
|
||||
|
||||
# W2: no row reordering needed for the down-projection.
|
||||
w2_flat = w2_bf16.reshape(e * k, n)
|
||||
w2_q_flat, w2_sf_flat = fp4_quantize(
|
||||
w2_flat,
|
||||
global_scale=gs,
|
||||
sf_vec_size=sf_vec_size,
|
||||
is_sf_swizzled_layout=True,
|
||||
)
|
||||
w2_q = w2_q_flat.view(e, k, n // 2) # uint8, packed FP4
|
||||
w2_blockscale = w2_sf_flat.view(e, k, w2_sf_flat.shape[1]) # float8
|
||||
|
||||
# All per-expert alphas are 1.0 (global_scale = 1.0, no compensation).
|
||||
ones_e = torch.ones(e, device="cuda", dtype=torch.float32)
|
||||
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=ones_e,
|
||||
g2_alphas=ones_e,
|
||||
a1_gscale=ones_e,
|
||||
a2_gscale=ones_e,
|
||||
w1_scale=w1_blockscale,
|
||||
w2_scale=w2_blockscale,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
in_dtype=dtype,
|
||||
)
|
||||
|
||||
experts = FlashInferB12xExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
_process_b12x_weights(
|
||||
experts,
|
||||
w1_blockscale,
|
||||
w2_blockscale,
|
||||
ones_e,
|
||||
ones_e,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
experts,
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
sm12x_output = kernel.apply(
|
||||
hidden_states=a,
|
||||
w1=w1_q,
|
||||
w2=w2_q,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=e,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
# Reference: BF16 torch MoE using original [gate, up] BF16 weights.
|
||||
# torch_moe's SiluAndMul expects [gate, up] order, matching w1_bf16.
|
||||
torch_output = torch_moe(a, w1_bf16, w2_bf16, score, topk)
|
||||
|
||||
torch.testing.assert_close(sm12x_output, torch_output, atol=2e-1, rtol=2e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", [8, 16])
|
||||
@pytest.mark.parametrize("topk", [1, 2, 4])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_b12x_moe_relu2(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
e: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
workspace_init,
|
||||
):
|
||||
"""Test FlashInferB12xExperts with ReLU2 (non-gated) activation.
|
||||
|
||||
ReLU2 is used by Nemotron-H style models. Unlike the gated SiLU
|
||||
path, w1 has shape [E, N, K] (not [E, 2N, K]) and the activation
|
||||
is relu(x)^2 without a gate/up split.
|
||||
"""
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
|
||||
# Non-gated: w1 shape is (e, n, k), not (e, 2n, k).
|
||||
w1_bf16 = torch.randn((e, n, k), device="cuda", dtype=dtype) / 15
|
||||
w2_bf16 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 15
|
||||
|
||||
gs = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
sf_vec_size = 16
|
||||
|
||||
# W1: no gate/up reordering for non-gated.
|
||||
w1_flat = w1_bf16.reshape(e * n, k)
|
||||
w1_q_flat, w1_sf_flat = fp4_quantize(
|
||||
w1_flat,
|
||||
global_scale=gs,
|
||||
sf_vec_size=sf_vec_size,
|
||||
is_sf_swizzled_layout=True,
|
||||
)
|
||||
w1_q = w1_q_flat.view(e, n, k // 2)
|
||||
w1_blockscale = w1_sf_flat.view(e, n, w1_sf_flat.shape[1])
|
||||
|
||||
w2_flat = w2_bf16.reshape(e * k, n)
|
||||
w2_q_flat, w2_sf_flat = fp4_quantize(
|
||||
w2_flat,
|
||||
global_scale=gs,
|
||||
sf_vec_size=sf_vec_size,
|
||||
is_sf_swizzled_layout=True,
|
||||
)
|
||||
w2_q = w2_q_flat.view(e, k, n // 2)
|
||||
w2_blockscale = w2_sf_flat.view(e, k, w2_sf_flat.shape[1])
|
||||
|
||||
ones_e = torch.ones(e, device="cuda", dtype=torch.float32)
|
||||
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=ones_e,
|
||||
g2_alphas=ones_e,
|
||||
a1_gscale=ones_e,
|
||||
a2_gscale=ones_e,
|
||||
w1_scale=w1_blockscale,
|
||||
w2_scale=w2_blockscale,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
in_dtype=dtype,
|
||||
activation=MoEActivation.RELU2_NO_MUL,
|
||||
)
|
||||
|
||||
experts = FlashInferB12xExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
_process_b12x_weights(
|
||||
experts,
|
||||
w1_blockscale,
|
||||
w2_blockscale,
|
||||
ones_e,
|
||||
ones_e,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
experts,
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
b12x_output = kernel.apply(
|
||||
hidden_states=a,
|
||||
w1=w1_q,
|
||||
w2=w2_q,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=e,
|
||||
activation=MoEActivation.RELU2_NO_MUL,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
torch_output = torch_moe(
|
||||
a,
|
||||
w1_bf16,
|
||||
w2_bf16,
|
||||
score,
|
||||
topk,
|
||||
activation=MoEActivation.RELU2_NO_MUL,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
b12x_output,
|
||||
torch_output,
|
||||
atol=2e-1,
|
||||
rtol=2e-1,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_flashinfer_b12x_moe(16, 128, 256, 8, 2, torch.bfloat16)
|
||||
@@ -1,357 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for FlashInfer CuTeDSL NVFP4 MoE."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
break_fp4_bytes,
|
||||
)
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.activation import (
|
||||
SiluAndMul,
|
||||
SiluAndMulWithClamp,
|
||||
SwigluOAIAndMul,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_moe import (
|
||||
FlashInferCuteDSLExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
make_nvfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
quantize_nvfp4_per_token_input,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutedsl_moe_nvfp4
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not has_flashinfer_cutedsl_moe_nvfp4() or not (
|
||||
current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip(
|
||||
"Requires FlashInfer CuTeDSL NVFP4 MoE on SM100",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
_SWIGLU_ALPHA = 1.702
|
||||
_SWIGLU_BETA = 1.0
|
||||
# The GEMM1 output is O(0.5) at these tensor scales, so the OAI default of 7.0
|
||||
# would never clamp anything. Pick a limit that actually bites.
|
||||
_SWIGLU_LIMIT = 0.3
|
||||
|
||||
_ACT_CASES = [
|
||||
pytest.param(MoEActivation.SILU, None, None, None, id="silu"),
|
||||
pytest.param(MoEActivation.SILU, None, None, _SWIGLU_LIMIT, id="silu-clamp"),
|
||||
pytest.param(MoEActivation.RELU2_NO_MUL, None, None, None, id="relu2_no_mul"),
|
||||
pytest.param(
|
||||
MoEActivation.SWIGLUOAI,
|
||||
_SWIGLU_ALPHA,
|
||||
_SWIGLU_BETA,
|
||||
_SWIGLU_LIMIT,
|
||||
id="swigluoai",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
_SWIGLU_ALPHA,
|
||||
_SWIGLU_BETA,
|
||||
_SWIGLU_LIMIT,
|
||||
id="swigluoai_uninterleave",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _reference_activation(
|
||||
activation: MoEActivation,
|
||||
alpha: float | None,
|
||||
beta: float | None,
|
||||
limit: float | None,
|
||||
):
|
||||
"""VLLM's own op for this activation, so the reference is not re-derived."""
|
||||
if activation == MoEActivation.RELU2_NO_MUL:
|
||||
return lambda x: torch.square(torch.relu(x))
|
||||
if activation == MoEActivation.SWIGLUOAI:
|
||||
# SwigluOAIAndMul hardcodes beta=1 and reads gate/up interleaved.
|
||||
assert beta == 1.0
|
||||
return SwigluOAIAndMul(alpha=alpha, limit=limit)
|
||||
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
return SiluAndMulWithClamp(limit, alpha, beta, compile_native=False)
|
||||
if limit is not None:
|
||||
return SiluAndMulWithClamp(limit, compile_native=False)
|
||||
return SiluAndMul()
|
||||
|
||||
|
||||
def _torch_moe_reference(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
score: torch.Tensor,
|
||||
topk: int,
|
||||
act_fn,
|
||||
) -> torch.Tensor:
|
||||
m = a.shape[0]
|
||||
weights, ids = torch.topk(torch.softmax(score, dim=-1, dtype=torch.float32), topk)
|
||||
x = a.view(m, 1, -1).repeat(1, topk, 1).reshape(m * topk, -1)
|
||||
flat_ids = ids.reshape(-1)
|
||||
out = torch.zeros(m * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
for expert in range(w1.shape[0]):
|
||||
mask = flat_ids == expert
|
||||
if mask.any():
|
||||
acc = act_fn(x[mask] @ w1[expert].transpose(0, 1)).to(a.dtype)
|
||||
out[mask] = acc @ w2[expert].transpose(0, 1)
|
||||
return (
|
||||
(out.view(m, topk, -1).float() * weights.view(m, topk, 1))
|
||||
.sum(dim=1)
|
||||
.to(a.dtype)
|
||||
)
|
||||
|
||||
|
||||
def _quantize_nvfp4_linear(
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
weights_q = []
|
||||
scales = []
|
||||
global_scales = []
|
||||
for expert_weight in weight:
|
||||
global_scale = (
|
||||
FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / expert_weight.abs().max()
|
||||
).to(torch.float32)
|
||||
weight_q, scale = ops.scaled_fp4_quant(
|
||||
expert_weight,
|
||||
global_scale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
weights_q.append(weight_q)
|
||||
scales.append(scale)
|
||||
global_scales.append(global_scale)
|
||||
return torch.stack(weights_q), torch.stack(scales), torch.stack(global_scales)
|
||||
|
||||
|
||||
def _dequantize_nvfp4_linear(
|
||||
tensor_fp4: torch.Tensor,
|
||||
tensor_sf: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
assert tensor_fp4.dtype == torch.uint8
|
||||
m, packed_k = tensor_fp4.shape
|
||||
k = packed_k * 2
|
||||
tensor_f32 = break_fp4_bytes(tensor_fp4, torch.float32)
|
||||
tensor_f32 = tensor_f32.reshape(m, k // 16, 16)
|
||||
tensor_sf = tensor_sf.view(torch.float8_e4m3fn).to(torch.float32)
|
||||
tensor_sf = tensor_sf[:, : k // 16] / global_scale
|
||||
return (tensor_f32 * tensor_sf.unsqueeze(-1)).reshape(m, k).to(dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"m,n,k,e,topk",
|
||||
[(16, 128, 512, 4, 2), (16, 192, 512, 4, 2), (16, 192, 2688, 4, 2)],
|
||||
)
|
||||
@pytest.mark.parametrize("activation,alpha,beta,limit", _ACT_CASES)
|
||||
@pytest.mark.parametrize("per_token_activation", [False, True])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_cutedsl_fp4_moe(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
e: int,
|
||||
topk: int,
|
||||
activation: MoEActivation,
|
||||
alpha: float | None,
|
||||
beta: float | None,
|
||||
limit: float | None,
|
||||
per_token_activation: bool,
|
||||
dtype: torch.dtype,
|
||||
workspace_init,
|
||||
):
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
hidden_states = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
# Keep GEMM1 activation variance stable as K changes, so the fixed
|
||||
# tolerance against the BF16 GEMM2 reference measures comparable FP4
|
||||
# quantization error at each hidden width.
|
||||
hidden_states *= (512 / k) ** 0.5
|
||||
|
||||
w1_rows = 2 * n if activation.is_gated else n
|
||||
w1 = torch.randn((e, w1_rows, k), device="cuda", dtype=dtype) / 15
|
||||
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 15
|
||||
w1_q, w1_scale, w1_global_scale = _quantize_nvfp4_linear(w1)
|
||||
w2_q, w2_scale, w2_global_scale = _quantize_nvfp4_linear(w2)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(
|
||||
hidden_states, score, topk, renormalize=False
|
||||
)
|
||||
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=activation,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=dtype,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
fake_layer = SimpleNamespace(activation=activation, moe_config=moe_config)
|
||||
a1_scale = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
a2_scale = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
(
|
||||
w1_cutedsl,
|
||||
w1_scale_cutedsl,
|
||||
w1_alpha,
|
||||
a1_scale,
|
||||
w2_cutedsl,
|
||||
w2_scale_cutedsl,
|
||||
w2_alpha,
|
||||
a2_scale,
|
||||
) = prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
layer=fake_layer,
|
||||
w13=w1_q,
|
||||
w13_scale=w1_scale,
|
||||
w13_scale_2=(1.0 / w1_global_scale),
|
||||
a13_scale=a1_scale,
|
||||
w2=w2_q,
|
||||
w2_scale=w2_scale,
|
||||
w2_scale_2=(1.0 / w2_global_scale),
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
# I=192 must remain unpadded for gated experts, but become 256 for
|
||||
# non-gated experts. The reference below uses the checkpoint shapes.
|
||||
expected_n = n if activation.is_gated else (n + 127) // 128 * 128
|
||||
assert moe_config.intermediate_size_per_partition == expected_n
|
||||
assert w1_cutedsl.shape[1] == expected_n * (2 if activation.is_gated else 1)
|
||||
assert w2_cutedsl.shape[2] * 2 == expected_n
|
||||
expected_k = (k + 255) // 256 * 256
|
||||
assert moe_config.hidden_dim == expected_k
|
||||
assert moe_config.hidden_dim_unpadded == k
|
||||
assert w1_cutedsl.shape[2] * 2 == expected_k
|
||||
assert w2_cutedsl.shape[1] == expected_k
|
||||
quant_config = make_nvfp4_moe_quant_config(
|
||||
backend=NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
w13_scale_2=w1_alpha,
|
||||
w2_scale_2=w2_alpha,
|
||||
a13_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
w13_scale=w1_scale_cutedsl,
|
||||
w2_scale=w2_scale_cutedsl,
|
||||
# Unset params must be omitted rather than forwarded as None into
|
||||
# the kernel's float-typed SwiGLU arguments.
|
||||
swiglu_alpha=alpha,
|
||||
swiglu_beta=beta,
|
||||
swiglu_limit=limit,
|
||||
)
|
||||
|
||||
cutedsl_experts = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
FlashInferCuteDSLExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
per_token_activation=per_token_activation,
|
||||
),
|
||||
)
|
||||
|
||||
cutedsl_output = cutedsl_experts.apply(
|
||||
# The model runner pads inputs and trims outputs around this kernel.
|
||||
hidden_states=torch.nn.functional.pad(hidden_states, (0, expected_k - k)),
|
||||
w1=w1_cutedsl,
|
||||
w2=w2_cutedsl,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
cutedsl_output = cutedsl_output[:, :k]
|
||||
|
||||
a_global_scale = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
if per_token_activation:
|
||||
a_q, a_scale, a_row_scale = quantize_nvfp4_per_token_input(hidden_states)
|
||||
else:
|
||||
a_q, a_scale = ops.scaled_fp4_quant(
|
||||
hidden_states,
|
||||
a_global_scale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
a_in_dtype = _dequantize_nvfp4_linear(
|
||||
a_q,
|
||||
a_scale,
|
||||
a_global_scale,
|
||||
dtype=dtype,
|
||||
)
|
||||
if per_token_activation:
|
||||
a_in_dtype = (a_in_dtype.float() * a_row_scale.view(-1, 1)).to(dtype)
|
||||
|
||||
w1_d = torch.empty((e, w1_rows, k), device="cuda", dtype=dtype)
|
||||
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
|
||||
for idx in range(e):
|
||||
w1_d[idx] = _dequantize_nvfp4_linear(
|
||||
w1_q[idx],
|
||||
w1_scale[idx],
|
||||
w1_global_scale[idx],
|
||||
dtype=dtype,
|
||||
)
|
||||
w2_d[idx] = _dequantize_nvfp4_linear(
|
||||
w2_q[idx],
|
||||
w2_scale[idx],
|
||||
w2_global_scale[idx],
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
torch_output = _torch_moe_reference(
|
||||
a_in_dtype,
|
||||
w1_d,
|
||||
w2_d,
|
||||
score,
|
||||
topk,
|
||||
_reference_activation(activation, alpha, beta, limit),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
torch_output,
|
||||
cutedsl_output,
|
||||
atol=3e-2,
|
||||
rtol=2e-1,
|
||||
)
|
||||
# Outputs here are O(1e-2) while NVFP4 noise is O(1e-3), so an absolute
|
||||
# tolerance loose enough for the quantization error also accepts a zero
|
||||
# tensor. Compare direction too, which dropped SwiGLU params or a wrong
|
||||
# w13 layout would break.
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
cutedsl_output.flatten().float(), torch_output.flatten().float(), dim=0
|
||||
)
|
||||
assert cosine > 0.99, f"cosine similarity {cosine:.4f} below 0.99"
|
||||
@@ -1,250 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_test_quant_config
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
dequantize_nvfp4_to_dtype,
|
||||
)
|
||||
from tests.kernels.utils import torch_moe
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
is_valid_flashinfer_cutlass_fused_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not has_flashinfer_cutlass_fused_moe() or not current_platform.has_device_capability(
|
||||
100
|
||||
):
|
||||
pytest.skip(
|
||||
"Requires flashinfer_cutlass_fused_moe and nvfp4 support",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
MNK_FACTORS = [
|
||||
(2, 1024, 1024),
|
||||
(2, 3072, 1024),
|
||||
(2, 3072, 1536),
|
||||
(64, 1024, 1536),
|
||||
(64, 3072, 1024),
|
||||
(64, 2048, 1536),
|
||||
(224, 1024, 1024),
|
||||
(224, 1024, 1536),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[MoEActivation.SWIGLUOAI, MoEActivation.SWIGLUOAI_UNINTERLEAVE],
|
||||
)
|
||||
def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch):
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=2,
|
||||
experts_per_token=1,
|
||||
hidden_dim=128,
|
||||
intermediate_size=128,
|
||||
num_local_experts=2,
|
||||
num_logical_experts=2,
|
||||
activation=activation,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
)
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
gemm1_alpha=1.702,
|
||||
gemm1_beta=1.0,
|
||||
gemm1_clamp_limit=7.0,
|
||||
)
|
||||
experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config)
|
||||
|
||||
call_args = {}
|
||||
|
||||
def fake_flashinfer_cutlass_fused_moe(**kwargs):
|
||||
call_args.update(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.experts."
|
||||
"flashinfer_cutlass_moe.flashinfer_cutlass_fused_moe",
|
||||
fake_flashinfer_cutlass_fused_moe,
|
||||
)
|
||||
experts.apply(
|
||||
output=torch.empty((1, 128), device="cuda", dtype=torch.bfloat16),
|
||||
hidden_states=torch.empty((1, 128), device="cuda", dtype=torch.bfloat16),
|
||||
w1=torch.empty((2, 256, 128), device="cuda", dtype=torch.bfloat16),
|
||||
w2=torch.empty((2, 128, 128), device="cuda", dtype=torch.bfloat16),
|
||||
topk_weights=torch.ones((1, 1), device="cuda", dtype=torch.float32),
|
||||
topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
|
||||
activation=activation,
|
||||
global_num_experts=2,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=None,
|
||||
workspace2=None,
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
assert experts._supports_activation(activation)
|
||||
assert call_args["activation_type"] == ActivationType.Swiglu
|
||||
for name, value in (
|
||||
("swiglu_alpha", 1.702),
|
||||
("swiglu_beta", 1.0),
|
||||
("swiglu_limit", 7.0),
|
||||
):
|
||||
torch.testing.assert_close(
|
||||
call_args[name],
|
||||
torch.full((2,), value, device="cuda", dtype=torch.float32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", [40, 64, 256])
|
||||
@pytest.mark.parametrize("topk", [1, 6, 8])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("activation", [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_fp4_moe_no_graph(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
e: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
activation: MoEActivation,
|
||||
workspace_init,
|
||||
):
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
|
||||
quant_blocksize = 16
|
||||
is_gated_act = activation.is_gated
|
||||
|
||||
w1_q, w2_q, quant_config = make_test_quant_config(
|
||||
e,
|
||||
n,
|
||||
k,
|
||||
in_dtype=dtype,
|
||||
quant_dtype="nvfp4",
|
||||
block_shape=None,
|
||||
per_act_token_quant=False,
|
||||
make_gate=is_gated_act,
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
assert is_valid_flashinfer_cutlass_fused_moe(a, w1_q, w2_q)
|
||||
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=activation,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=dtype,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
flashinfer_experts = FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
FlashInferExperts(moe_config=moe_config, quant_config=quant_config),
|
||||
)
|
||||
|
||||
flashinfer_output = flashinfer_experts.apply(
|
||||
hidden_states=a,
|
||||
w1=w1_q,
|
||||
w2=w2_q,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
# Reference check:
|
||||
a_global_scale = (
|
||||
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a.flatten(), dim=-1)
|
||||
).to(torch.float32)
|
||||
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a, a_global_scale)
|
||||
_, m_k = a_fp4.shape
|
||||
a_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
a_fp4,
|
||||
a_scale_interleaved,
|
||||
a_global_scale,
|
||||
dtype=a.dtype,
|
||||
device=a.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
|
||||
w1_d = torch.empty(
|
||||
(e, (2 if is_gated_act else 1) * n, k), device="cuda", dtype=dtype
|
||||
)
|
||||
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
|
||||
|
||||
for idx in range(0, e):
|
||||
w1_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w1_q[idx],
|
||||
quant_config.w1_scale[idx],
|
||||
(1 / quant_config.g1_alphas[idx]),
|
||||
dtype=dtype,
|
||||
device=w1_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
w2_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w2_q[idx],
|
||||
quant_config.w2_scale[idx],
|
||||
(1 / quant_config.g2_alphas[idx]),
|
||||
dtype=dtype,
|
||||
device=w2_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
|
||||
torch_output = torch_moe(
|
||||
a_in_dtype, w1_d, w2_d, score, topk, activation=activation
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
torch_output, flashinfer_output, atol=1e-1, rtol=1e-1
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_flashinfer_fp4_moe_no_graph((2, 1024, 1024), 40, 1, torch.half)
|
||||
@@ -1,369 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end tests for the FlashInfer MoE-EP adapter on real Blackwell GPUs.
|
||||
|
||||
Each test spawns one process per GPU, bootstraps vLLM's expert-parallel
|
||||
groups, builds a ``FlashInferMoeEp`` adapter from bf16 expert weights at the
|
||||
MoE geometry of a real model, warms it up and runs a forward whose inputs are
|
||||
chosen so that every quantization step in the megakernel is exact, for the
|
||||
CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
|
||||
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
|
||||
32). The expected output is then known in closed form and compared bit for
|
||||
bit.
|
||||
|
||||
A test that needs no SM100 also checks the adapter's kwargs against the
|
||||
installed FlashInfer's config types, so config drift fails on any CI runner
|
||||
with FlashInfer installed.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_device
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
|
||||
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
|
||||
FlashInferMoeEpExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
FlashInferMoeEp,
|
||||
FlashInferMoeEpWeights,
|
||||
apply_topk_in_fc1,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Geometry:
|
||||
name: str
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
num_experts: int
|
||||
top_k: int
|
||||
|
||||
|
||||
# Routed-expert geometry of the models the branch was evaluated on. DeepGEMM
|
||||
# keeps one UE8M0 scale byte per 32 elements and requires the per-token scale
|
||||
# rows to be TMA aligned, so hidden and intermediate sizes must be multiples
|
||||
# of 512 (FlashInfer's own validation only checks 128); all four qualify.
|
||||
GEOMETRIES = (
|
||||
Geometry("dsv4-pro", 7168, 3072, 384, 6),
|
||||
Geometry("dsv4-flash", 4096, 2048, 256, 6),
|
||||
Geometry("mistral-small-4", 4096, 2048, 128, 4),
|
||||
Geometry("mistral-large-3", 7168, 4096, 128, 4),
|
||||
)
|
||||
BACKENDS = (FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM)
|
||||
|
||||
NUM_TOKENS_PER_RANK = 4
|
||||
MAX_TOKENS_PER_RANK = 128
|
||||
|
||||
# Every block scale below is either 1 or a power of two, so the FP4 codes
|
||||
# and the FP8 values survive quantization exactly: NVFP4 scales a 16-block by
|
||||
# absmax / 6 in E4M3, DeepGEMM scales a 32-block by absmax / 6 (FP4) or
|
||||
# absmax / 448 (FP8) rounded up to a power of two. A 6.0 in a block pins the
|
||||
# FP4 scale to 1; 2.0, 6.0 and 144 are exact FP8 values at any such scale.
|
||||
#
|
||||
# Token t carries X_SIGNAL at hidden column 16 * t and a 6.0 sentinel next to
|
||||
# it; every weight column the sentinel meets is zero.
|
||||
X_SIGNAL = 2.0
|
||||
X_SENTINEL = 6.0
|
||||
# Gate and up rows are identical, so h = silu(12) * 12 = 144 = 6 * 24
|
||||
# whichever half the kernel treats as the gate, and 144 requantizes exactly.
|
||||
W13_DIAG = 6.0
|
||||
H_SIGNAL = 144.0
|
||||
# Expert e writes E2M1_CODES[e % 7] * H_SIGNAL into output channel e. The 6.0
|
||||
# sentinel sits next to a zero intermediate channel and pins the block scale.
|
||||
W2_SENTINEL = 6.0
|
||||
E2M1_CODES = (0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
|
||||
|
||||
|
||||
def _expert_out(expert: int) -> float:
|
||||
return E2M1_CODES[expert % len(E2M1_CODES)]
|
||||
|
||||
|
||||
def _topk_weights(top_k: int) -> tuple[float, ...]:
|
||||
# The DeepSeek-V4 routing policy applies the router weight before fc2, so
|
||||
# weight * 144 must requantize exactly too: powers of two do.
|
||||
return tuple(2.0 ** -(k + 1) for k in range(top_k))
|
||||
|
||||
|
||||
def _routing_method(backend: str) -> RoutingMethodType:
|
||||
if backend == FLASHINFER_MOE_EP_DEEP_GEMM:
|
||||
return RoutingMethodType.DeepseekV4
|
||||
return RoutingMethodType.Default
|
||||
|
||||
|
||||
def _routing(
|
||||
geometry: Geometry, rank: int, device: torch.device
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""top_k distinct experts per token, strided across ranks so dispatch goes
|
||||
remote."""
|
||||
base = (
|
||||
torch.arange(NUM_TOKENS_PER_RANK, device=device) + rank * NUM_TOKENS_PER_RANK
|
||||
) % geometry.num_experts
|
||||
stride = geometry.num_experts // geometry.top_k
|
||||
offsets = torch.arange(geometry.top_k, device=device) * stride
|
||||
topk_ids = ((base[:, None] + offsets[None, :]) % geometry.num_experts).to(
|
||||
torch.int32
|
||||
)
|
||||
topk_weights = torch.tensor(
|
||||
[_topk_weights(geometry.top_k)] * NUM_TOKENS_PER_RANK,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
return topk_ids, topk_weights
|
||||
|
||||
|
||||
def _hidden_states(geometry: Geometry, device: torch.device) -> torch.Tensor:
|
||||
x = torch.zeros(
|
||||
NUM_TOKENS_PER_RANK, geometry.hidden_size, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
for t in range(NUM_TOKENS_PER_RANK):
|
||||
x[t, 16 * t] = X_SIGNAL
|
||||
x[t, 16 * t + 1] = X_SENTINEL
|
||||
return x
|
||||
|
||||
|
||||
def _local_expert_weights(
|
||||
geometry: Geometry, rank: int, num_local_experts: int, device: torch.device
|
||||
) -> FlashInferMoeEpWeights:
|
||||
w13 = torch.zeros(
|
||||
num_local_experts,
|
||||
2 * geometry.intermediate_size,
|
||||
geometry.hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
w2 = torch.zeros(
|
||||
num_local_experts,
|
||||
geometry.hidden_size,
|
||||
geometry.intermediate_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
token_columns = [16 * t for t in range(NUM_TOKENS_PER_RANK)]
|
||||
for local in range(num_local_experts):
|
||||
expert = rank * num_local_experts + local
|
||||
w13[local, 0, token_columns] = W13_DIAG
|
||||
w13[local, geometry.intermediate_size, token_columns] = W13_DIAG
|
||||
w2[local, expert, 0] = _expert_out(expert)
|
||||
w2[local, expert, 2] = W2_SENTINEL
|
||||
return FlashInferMoeEpWeights(w13=w13, w2=w2)
|
||||
|
||||
|
||||
def _expected_output(
|
||||
geometry: Geometry, topk_ids: torch.Tensor, topk_weights: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
expected = torch.zeros(
|
||||
NUM_TOKENS_PER_RANK,
|
||||
geometry.hidden_size,
|
||||
dtype=torch.float32,
|
||||
device=topk_ids.device,
|
||||
)
|
||||
for t in range(NUM_TOKENS_PER_RANK):
|
||||
for k in range(geometry.top_k):
|
||||
expert = int(topk_ids[t, k])
|
||||
weight = float(topk_weights[t, k])
|
||||
expected[t, expert] += weight * _expert_out(expert) * H_SIGNAL
|
||||
return expected.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _moe_config(
|
||||
vllm_config: VllmConfig,
|
||||
geometry: Geometry,
|
||||
world_size: int,
|
||||
device: torch.device,
|
||||
backend: str,
|
||||
) -> FusedMoEConfig:
|
||||
parallel = FusedMoEParallelConfig.make(
|
||||
tp_size_=1,
|
||||
pcp_size_=1,
|
||||
dp_size_=world_size,
|
||||
sp_size_=1,
|
||||
vllm_parallel_config=vllm_config.parallel_config,
|
||||
)
|
||||
return FusedMoEConfig(
|
||||
num_experts=geometry.num_experts,
|
||||
experts_per_token=geometry.top_k,
|
||||
hidden_dim=geometry.hidden_size,
|
||||
intermediate_size=geometry.intermediate_size,
|
||||
num_local_experts=geometry.num_experts // world_size,
|
||||
num_logical_experts=geometry.num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
device=device,
|
||||
routing_method=_routing_method(backend),
|
||||
moe_parallel_config=parallel,
|
||||
in_dtype=torch.bfloat16,
|
||||
moe_backend=backend,
|
||||
max_num_tokens=MAX_TOKENS_PER_RANK,
|
||||
)
|
||||
|
||||
|
||||
def _check_expert_views_alias_kernel_memory(
|
||||
adapter: FlashInferMoeEp,
|
||||
geometry: Geometry,
|
||||
rank: int,
|
||||
num_local_experts: int,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
"""EPLB moves experts through ``expert_weight_views``; the kernel must see it.
|
||||
|
||||
Swap local experts 0 and 1 in place on every rank, then route through the
|
||||
swapped physical slots: the output must be the one the unswapped logical
|
||||
experts would have produced.
|
||||
"""
|
||||
for view in adapter.expert_weight_views():
|
||||
first = view[0].clone()
|
||||
view[0].copy_(view[1])
|
||||
view[1].copy_(first)
|
||||
torch.distributed.barrier()
|
||||
|
||||
topk_ids, topk_weights = _routing(geometry, rank, device)
|
||||
slot = topk_ids % num_local_experts
|
||||
physical_ids = torch.where(slot == 0, topk_ids + 1, topk_ids)
|
||||
physical_ids = torch.where(slot == 1, topk_ids - 1, physical_ids)
|
||||
output = adapter(_hidden_states(geometry, device), physical_ids, topk_weights)
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(output, _expected_output(geometry, topk_ids, topk_weights))
|
||||
|
||||
|
||||
def _init_warmup_forward(
|
||||
env: dict[str, str], world_size: int, geometry: Geometry, backend: str
|
||||
) -> None:
|
||||
set_env_vars_and_device(env)
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.data_parallel_size = world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
vllm_config.kernel_config.moe_backend = backend
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=1, pipeline_model_parallel_size=1
|
||||
)
|
||||
rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
moe = _moe_config(vllm_config, geometry, world_size, device, backend)
|
||||
supported, reason = FlashInferMoeEpExperts.is_supported_config(
|
||||
FlashInferMoeEpExperts,
|
||||
moe,
|
||||
kMxfp4Static,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported, reason
|
||||
|
||||
adapter = FlashInferMoeEp(
|
||||
moe,
|
||||
_local_expert_weights(geometry, rank, moe.num_local_experts, device),
|
||||
apply_topk_in_fc1=apply_topk_in_fc1(moe),
|
||||
)
|
||||
try:
|
||||
adapter.warmup()
|
||||
|
||||
topk_ids, topk_weights = _routing(geometry, rank, device)
|
||||
output = adapter(_hidden_states(geometry, device), topk_ids, topk_weights)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected = _expected_output(geometry, topk_ids, topk_weights)
|
||||
assert output.dtype == expected.dtype and output.shape == expected.shape
|
||||
assert torch.equal(output, expected), (
|
||||
f"rank {rank}: max abs diff "
|
||||
f"{(output.float() - expected.float()).abs().max().item()}, "
|
||||
f"nonzero output channels per token "
|
||||
f"{[torch.nonzero(row).flatten().tolist() for row in output]}"
|
||||
)
|
||||
|
||||
if backend == FLASHINFER_MOE_EP_CUTEDSL:
|
||||
_check_expert_views_alias_kernel_memory(
|
||||
adapter, geometry, rank, moe.num_local_experts, device
|
||||
)
|
||||
finally:
|
||||
adapter.destroy()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="The FlashInfer MoE-EP megakernels require SM100",
|
||||
)
|
||||
@pytest.mark.parametrize("geometry", GEOMETRIES, ids=[g.name for g in GEOMETRIES])
|
||||
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_init_warmup_forward_is_exact_on_grid_inputs(
|
||||
world_size: int, backend: str, geometry: Geometry
|
||||
):
|
||||
pytest.importorskip("flashinfer.moe_ep")
|
||||
if backend == FLASHINFER_MOE_EP_DEEP_GEMM and not has_deep_gemm():
|
||||
pytest.skip("DeepGEMM is not available")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs")
|
||||
distributed_run(_init_warmup_forward, world_size, geometry, backend)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
|
||||
def test_adapter_kwargs_match_installed_flashinfer(
|
||||
monkeypatch: pytest.MonkeyPatch, backend: str
|
||||
):
|
||||
"""The adapter's kwargs construct the installed FlashInfer's config types.
|
||||
|
||||
Needs no SM100, so config kwarg drift fails in CI rather than at model load.
|
||||
Fakes only the megakernel layer (symmetric memory), the EP group, the device
|
||||
index and the DeepGEMM import.
|
||||
"""
|
||||
moe_ep = pytest.importorskip("flashinfer.moe_ep")
|
||||
layer_args: list[tuple] = []
|
||||
|
||||
class FakeMegaLayer:
|
||||
def __init__(self, *args) -> None:
|
||||
layer_args.append(args)
|
||||
|
||||
def destroy(self) -> None:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(moe_ep, "MoEEpMegaLayer", FakeMegaLayer)
|
||||
monkeypatch.setattr(fi_ep, "_expose_deep_gemm_to_flashinfer", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
fi_ep,
|
||||
"get_ep_group",
|
||||
lambda: SimpleNamespace(world_size=2, rank_in_group=0, device_group=None),
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 0)
|
||||
|
||||
geometry = GEOMETRIES[0]
|
||||
moe = SimpleNamespace(
|
||||
moe_backend=backend,
|
||||
num_experts=geometry.num_experts,
|
||||
experts_per_token=geometry.top_k,
|
||||
hidden_dim=geometry.hidden_size,
|
||||
intermediate_size=geometry.intermediate_size,
|
||||
max_num_tokens=MAX_TOKENS_PER_RANK,
|
||||
swiglu_limit=10.0,
|
||||
routing_method=_routing_method(backend),
|
||||
)
|
||||
weight = torch.zeros(1, dtype=torch.bfloat16)
|
||||
FlashInferMoeEp(
|
||||
moe,
|
||||
FlashInferMoeEpWeights(w13=weight, w2=weight),
|
||||
apply_topk_in_fc1=apply_topk_in_fc1(moe),
|
||||
)
|
||||
|
||||
((_, _, _, mega_config),) = layer_args
|
||||
expected = (
|
||||
moe_ep.Nvfp4CutedslMegaMoeConfig
|
||||
if backend == FLASHINFER_MOE_EP_CUTEDSL
|
||||
else moe_ep.DeepGemmMegaMoeConfig
|
||||
)
|
||||
assert isinstance(mega_config.megakernel, expected)
|
||||
@@ -1,227 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Copyright (c) 2025 FlyDSL Project Contributors
|
||||
|
||||
|
||||
import importlib.util
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
int4_w4a16_moe_quant_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not (current_platform.is_rocm() and on_gfx950()):
|
||||
pytest.skip("This test can only run on ROCm and gfx950.", allow_module_level=True)
|
||||
|
||||
aiter_available = importlib.util.find_spec("aiter") is not None
|
||||
|
||||
if not aiter_available:
|
||||
pytest.skip("These tests require AITER to run.", allow_module_level=True)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.fused_flydsl_moe import ( # noqa: E402
|
||||
fused_flydsl_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E402, E501
|
||||
compressed_tensors_moe_w4a16_flydsl,
|
||||
)
|
||||
|
||||
RoutingBuffers = tuple[
|
||||
torch.Tensor, # sorted_token_ids
|
||||
torch.Tensor, # sorted_weights
|
||||
torch.Tensor, # sorted_expert_ids
|
||||
torch.Tensor, # num_valid_ids (shape [1], i32)
|
||||
int, # sorted_size
|
||||
int, # blocks
|
||||
]
|
||||
|
||||
|
||||
def _assert_flydsl_matches_reference(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
*,
|
||||
atol: float = 0.5,
|
||||
rtol: float = 0.1,
|
||||
max_mismatch_fraction: float = 1e-5,
|
||||
max_error_ratio: float = 3.0,
|
||||
) -> None:
|
||||
"""Compare kernels without making accuracy depend on output size.
|
||||
|
||||
The FlyDSL and reference kernels accumulate BF16 values in different
|
||||
orders. A strict allclose makes one expected rounding outlier fail an
|
||||
otherwise accurate output, which becomes increasingly likely for the
|
||||
largest token counts (up to 117 million output elements here).
|
||||
|
||||
Keep the original elementwise tolerance for 99.999% of values, while also
|
||||
bounding every tolerated outlier to three times its elementwise tolerance.
|
||||
"""
|
||||
assert actual.shape == expected.shape
|
||||
|
||||
mismatch = ~torch.isclose(actual, expected, atol=atol, rtol=rtol)
|
||||
mismatch_count = int(mismatch.sum().item())
|
||||
total = actual.numel()
|
||||
allowed_mismatches = int(total * max_mismatch_fraction)
|
||||
mismatch_msg = (
|
||||
f"FlyDSL/reference mismatch: {mismatch_count}/{total} values "
|
||||
f"({mismatch_count / total:.6%}) exceed atol={atol}, rtol={rtol}; "
|
||||
f"allowed <= {allowed_mismatches}/{total} "
|
||||
f"({max_mismatch_fraction:.6%})"
|
||||
)
|
||||
assert mismatch_count <= allowed_mismatches, mismatch_msg
|
||||
|
||||
if mismatch_count:
|
||||
actual_mismatch = actual[mismatch].float()
|
||||
expected_mismatch = expected[mismatch].float()
|
||||
abs_error = (actual_mismatch - expected_mismatch).abs()
|
||||
tolerance = atol + rtol * expected_mismatch.abs()
|
||||
worst_error_ratio = (abs_error / tolerance).max().item()
|
||||
assert worst_error_ratio <= max_error_ratio, (
|
||||
f"{mismatch_msg}; worst error is {worst_error_ratio:.4f}x its "
|
||||
f"elementwise tolerance (allowed <= {max_error_ratio:.4f}x)"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens", [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384]
|
||||
)
|
||||
@pytest.mark.parametrize("inter_dim", [256, 512])
|
||||
def test_flydsl_moe(num_tokens: int, inter_dim: int):
|
||||
set_random_seed(0)
|
||||
|
||||
device = "cuda"
|
||||
topk = 8
|
||||
num_experts = 384
|
||||
hidden_size = 7168
|
||||
packed_factor = 8
|
||||
w13_num_shards = 2
|
||||
params_dtype = torch.bfloat16
|
||||
group_size = 32
|
||||
w2_scales_size = inter_dim
|
||||
scale_factor = 0.01
|
||||
|
||||
num_groups_w2 = w2_scales_size // group_size
|
||||
num_groups_w13 = hidden_size // group_size
|
||||
|
||||
w13_weight = torch.randint(
|
||||
0,
|
||||
255,
|
||||
(num_experts, hidden_size // packed_factor, w13_num_shards * inter_dim),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
w2_weight = torch.randint(
|
||||
0,
|
||||
255,
|
||||
(num_experts, inter_dim // packed_factor, hidden_size),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
w13_scale = scale_factor * torch.randn(
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
w13_num_shards * inter_dim,
|
||||
dtype=params_dtype,
|
||||
device=device,
|
||||
)
|
||||
w2_scale = scale_factor * torch.randn(
|
||||
num_experts, num_groups_w2, hidden_size, dtype=params_dtype, device=device
|
||||
)
|
||||
|
||||
w13_weight_packed = w13_weight.transpose(1, 2).contiguous().view(torch.uint8)
|
||||
w2_weight_packed = w2_weight.transpose(1, 2).contiguous().view(torch.uint8)
|
||||
w13_weight_scale = w13_scale.transpose(1, 2).contiguous()
|
||||
w2_weight_scale = w2_scale.transpose(1, 2).contiguous()
|
||||
|
||||
moe_quant_config = int4_w4a16_moe_quant_config(
|
||||
w1_scale=w13_weight_scale,
|
||||
w2_scale=w2_weight_scale,
|
||||
w1_zp=None,
|
||||
w2_zp=None,
|
||||
block_shape=[0, group_size],
|
||||
)
|
||||
score = torch.rand((num_tokens, num_experts), device=device, dtype=torch.float32)
|
||||
topk_vals, topk_ids = torch.topk(score, k=topk, dim=1)
|
||||
topk_weights = torch.softmax(topk_vals, dim=1).to(torch.float32)
|
||||
x = torch.randn((num_tokens, hidden_size), dtype=torch.bfloat16, device=device)
|
||||
out_ref = fused_experts(
|
||||
x,
|
||||
w13_weight_packed,
|
||||
w2_weight_packed,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
quant_config=moe_quant_config,
|
||||
)
|
||||
|
||||
w13 = w13_weight
|
||||
w13 = compressed_tensors_moe_w4a16_flydsl._gptq_int32_to_flydsl_packed(w13)
|
||||
w13 = w13.view(-1).contiguous()
|
||||
|
||||
w2 = w2_weight
|
||||
w2 = compressed_tensors_moe_w4a16_flydsl._gptq_int32_to_flydsl_packed(w2)
|
||||
w2 = w2.view(-1).contiguous()
|
||||
|
||||
w13_scale_flydsl = w13_scale
|
||||
w2_scale_flydsl = w2_scale
|
||||
|
||||
if group_size > 0 and w13_scale.dim() == 3 and w13_scale.shape[1] > 1:
|
||||
E, G, N = w13_scale.shape
|
||||
w13_scale_flydsl = (
|
||||
w13_scale_flydsl.view(E, G // 2, 2, N)
|
||||
.permute(0, 1, 3, 2)
|
||||
.contiguous()
|
||||
.view(-1)
|
||||
.contiguous()
|
||||
)
|
||||
elif w13_scale.dim() == 3 and w13_scale.shape[1] == 1:
|
||||
w13_scale_flydsl = w13_scale_flydsl.squeeze(1)
|
||||
|
||||
if group_size > 0 and w2_scale.dim() == 3 and w2_scale.shape[1] > 1:
|
||||
E, G, N = w2_scale.shape
|
||||
w2_scale_flydsl = (
|
||||
w2_scale_flydsl.view(E, G // 2, 2, N)
|
||||
.permute(0, 1, 3, 2)
|
||||
.contiguous()
|
||||
.view(-1)
|
||||
.contiguous()
|
||||
)
|
||||
elif w2_scale.dim() == 3 and w2_scale.shape[1] == 1:
|
||||
w2_scale_flydsl = w2_scale_flydsl.squeeze(1)
|
||||
|
||||
w13_scale_flydsl = w13_scale_flydsl.contiguous()
|
||||
w2_scale_flydsl = w2_scale_flydsl.contiguous()
|
||||
|
||||
w13.is_shuffled = True
|
||||
w2.is_shuffled = True
|
||||
|
||||
out = fused_flydsl_moe(
|
||||
x,
|
||||
w13,
|
||||
w2,
|
||||
num_experts,
|
||||
inter_dim,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
w1_scale=w13_scale_flydsl,
|
||||
w2_scale=w2_scale_flydsl,
|
||||
topk=topk_weights.shape[-1],
|
||||
group_size=group_size,
|
||||
doweight_stage1=False,
|
||||
scale_is_bf16=True,
|
||||
)
|
||||
|
||||
_assert_flydsl_matches_reference(out, out_ref)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_flydsl_moe(512, 256)
|
||||
@@ -1,311 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.distributed as distributed
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
unquantized_fused_moe_method as unquantized_module,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization import fp8 as fp8_module
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
|
||||
|
||||
def _make_fp8_tp_experts(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
tp_rank,
|
||||
backend="flashinfer_trtllm",
|
||||
*,
|
||||
mock_backend=True,
|
||||
num_experts=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=640,
|
||||
quant_config=None,
|
||||
config_overrides=None,
|
||||
):
|
||||
# Exercise allocation and the public weight loader without selecting a GPU kernel.
|
||||
if mock_backend:
|
||||
monkeypatch.setattr(
|
||||
fp8_module, "select_fp8_moe_backend", lambda **kwargs: (None, None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
unquantized_module,
|
||||
"select_unquantized_moe_backend",
|
||||
lambda **kwargs: (None, None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
fp8_module, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
# The block-shape validator imports this function from vllm.distributed.
|
||||
monkeypatch.setattr(
|
||||
distributed, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
config = make_dummy_moe_config(
|
||||
num_experts=num_experts,
|
||||
hidden_dim=hidden_dim,
|
||||
intermediate_size=intermediate_size,
|
||||
experts_per_token=min(num_experts, 10),
|
||||
)
|
||||
config.routing_method = RoutingMethodType.RenormalizeNaive
|
||||
config.moe_parallel_config.tp_size = tp_size
|
||||
config.moe_parallel_config.tp_rank = tp_rank
|
||||
config.intermediate_size_per_partition = intermediate_size // tp_size
|
||||
config.intermediate_size_per_partition_unpadded = intermediate_size // tp_size
|
||||
config.moe_backend = backend
|
||||
for field, value in (config_overrides or {}).items():
|
||||
setattr(config, field, value)
|
||||
return RoutedExperts(
|
||||
"model.layers.0.mlp.experts",
|
||||
torch.bfloat16,
|
||||
config,
|
||||
quant_config
|
||||
or Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128]),
|
||||
expert_map_manager=SimpleNamespace(
|
||||
local_num_experts=num_experts,
|
||||
placement_strategy="linear",
|
||||
expert_map=None,
|
||||
expert_mask=None,
|
||||
routing_tables=None,
|
||||
map_global_to_local=lambda index: index,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
@pytest.mark.parametrize("batched", [False, True])
|
||||
@pytest.mark.parametrize("intermediate_size", [640, 896])
|
||||
def test_fp8_block_aligned_tp_preserves_checkpoint(
|
||||
monkeypatch, tp_size, batched, intermediate_size
|
||||
):
|
||||
"""All ranks reconstruct the original dequantized projections, including
|
||||
padding-only ranks. Reloading must clear stale weights and scales.
|
||||
"""
|
||||
generator = torch.Generator().manual_seed(42)
|
||||
weights = {
|
||||
"w1": torch.randn(2, intermediate_size, 256, generator=generator).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
"w3": torch.randn(2, intermediate_size, 256, generator=generator).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
"w2": torch.randn(2, 256, intermediate_size, generator=generator).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
}
|
||||
scales = {
|
||||
name: torch.rand(2, w.shape[1] // 128, w.shape[2] // 128, generator=generator)
|
||||
+ 0.1
|
||||
for name, w in weights.items()
|
||||
}
|
||||
reconstructed: dict[str, list[torch.Tensor]] = {name: [] for name in weights}
|
||||
num_blocks = intermediate_size // 128
|
||||
for rank in range(tp_size):
|
||||
layer = _make_fp8_tp_experts(
|
||||
monkeypatch, tp_size, rank, intermediate_size=intermediate_size
|
||||
)
|
||||
assert layer.quant_method.weight_scale_refine is None
|
||||
assert layer.quant_method.moe_block_shape == [128, 128]
|
||||
width = layer.moe_config.intermediate_size_per_partition
|
||||
assert width == ((num_blocks + tp_size - 1) // tp_size) * 128
|
||||
for name in weights:
|
||||
prefix = "w2" if name == "w2" else "w13"
|
||||
for suffix, checkpoint in (
|
||||
("weight", weights[name]),
|
||||
("weight_scale_inv", scales[name]),
|
||||
):
|
||||
param_name = f"{prefix}_{suffix}"
|
||||
param = getattr(layer, param_name)
|
||||
# Also reload after dirtying the destination, to catch padding leaks.
|
||||
for _ in range(2):
|
||||
if name != "w3":
|
||||
param.data.fill_(7)
|
||||
if batched:
|
||||
assert param.weight_loader(
|
||||
param, checkpoint, param_name, name, 0, return_success=True
|
||||
)
|
||||
else:
|
||||
for expert in range(2):
|
||||
assert param.weight_loader(
|
||||
param,
|
||||
checkpoint[expert],
|
||||
param_name,
|
||||
name,
|
||||
expert,
|
||||
return_success=True,
|
||||
)
|
||||
w = getattr(layer, f"{prefix}_weight").float()
|
||||
s = getattr(layer, f"{prefix}_weight_scale_inv")
|
||||
if name in ("w1", "w3"):
|
||||
half = 0 if name == "w1" else 1
|
||||
w = w.chunk(2, dim=1)[half]
|
||||
s = s.chunk(2, dim=1)[half]
|
||||
dim = 2 if name == "w2" else 1
|
||||
valid = max(0, min(width, intermediate_size - rank * width))
|
||||
assert torch.count_nonzero(w.narrow(dim, valid, width - valid)) == 0
|
||||
assert torch.all(s.narrow(dim, valid // 128, (width - valid) // 128) == 1)
|
||||
dequant = w * s.repeat_interleave(128, dim=1).repeat_interleave(128, dim=2)
|
||||
reconstructed[name].append(dequant.narrow(dim, 0, valid))
|
||||
for name, parts in reconstructed.items():
|
||||
expected = weights[name].float() * scales[name].repeat_interleave(
|
||||
128, dim=1
|
||||
).repeat_interleave(128, dim=2)
|
||||
assert torch.equal(torch.cat(parts, dim=2 if name == "w2" else 1), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["auto", "triton"])
|
||||
@pytest.mark.parametrize("tp_size,block", [(2, 64), (4, 32)])
|
||||
def test_fp8_tp_default_keeps_refined_layout(monkeypatch, backend, tp_size, block):
|
||||
layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0, backend)
|
||||
assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size
|
||||
assert layer.quant_method.moe_block_shape == [block, block]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("hidden_dim", 192),
|
||||
("intermediate_size", 672),
|
||||
("is_lora_enabled", True),
|
||||
("has_bias", True),
|
||||
],
|
||||
)
|
||||
def test_fp8_block_aligned_tp_rejects_unsupported_layout(monkeypatch, field, value):
|
||||
with pytest.raises(ValueError, match="Block-aligned FP8 TP sharding requires"):
|
||||
_make_fp8_tp_experts(monkeypatch, 4, 0, config_overrides={field: value})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 5])
|
||||
def test_fp8_aligned_tp_keeps_original_layout(monkeypatch, tp_size):
|
||||
layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0)
|
||||
assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size
|
||||
assert not layer.moe_config.tp_shard_with_padding
|
||||
assert layer.quant_method.weight_scale_refine is None
|
||||
|
||||
|
||||
def test_fp8_skipped_layer_keeps_original_tp_layout(monkeypatch, default_vllm_config):
|
||||
layer = _make_fp8_tp_experts(
|
||||
monkeypatch,
|
||||
4,
|
||||
0,
|
||||
quant_config=Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
weight_block_size=[128, 128],
|
||||
ignored_layers=["model.layers.0.mlp.experts"],
|
||||
),
|
||||
)
|
||||
assert isinstance(layer.quant_method, unquantized_module.UnquantizedFusedMoEMethod)
|
||||
assert not layer.moe_config.tp_shard_with_padding
|
||||
assert layer.moe_config.intermediate_size_per_partition == 160
|
||||
assert layer.w13_weight.shape == (2, 320, 256)
|
||||
assert layer.w2_weight.shape == (2, 256, 160)
|
||||
|
||||
|
||||
def test_fp8_block_aligned_tp_rejects_presharded_weights(monkeypatch):
|
||||
layer = _make_fp8_tp_experts(monkeypatch, 4, 0)
|
||||
with pytest.raises(ValueError, match="unsharded checkpoint"):
|
||||
layer.w13_weight.weight_loader(
|
||||
layer.w13_weight,
|
||||
torch.zeros(160, 256, dtype=torch.float8_e4m3fn),
|
||||
"w13_weight",
|
||||
"w1",
|
||||
0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17])
|
||||
@torch.inference_mode()
|
||||
def test_fp8_block_aligned_tp_flashinfer_matches_unsharded(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
num_tokens,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
):
|
||||
"""Compare sharded FI with unsharded FI, reporting the cross-backend
|
||||
difference separately because activation quantization can differ.
|
||||
"""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Requires datacenter Blackwell and FlashInfer TRTLLM")
|
||||
torch.manual_seed(42)
|
||||
e, h, n, topk = 16, 2560, 640, 10
|
||||
with torch.device("cuda"), set_current_vllm_config(VllmConfig()):
|
||||
x = torch.randn(num_tokens, h, dtype=torch.bfloat16) / 10
|
||||
logits = torch.randn(num_tokens, e, dtype=torch.float32)
|
||||
probabilities = logits.softmax(dim=-1)
|
||||
topk_weights, topk_ids = probabilities.topk(topk, dim=-1)
|
||||
topk_weights /= topk_weights.sum(dim=-1, keepdim=True)
|
||||
weights = {
|
||||
"w1": torch.randn(e, n, h).to(torch.float8_e4m3fn),
|
||||
"w3": torch.randn(e, n, h).to(torch.float8_e4m3fn),
|
||||
"w2": torch.randn(e, h, n).to(torch.float8_e4m3fn),
|
||||
}
|
||||
scales = {
|
||||
name: (torch.rand(e, w.shape[1] // 128, w.shape[2] // 128) + 0.5) / 32
|
||||
for name, w in weights.items()
|
||||
}
|
||||
ref = fused_experts(
|
||||
x,
|
||||
torch.cat([weights["w1"], weights["w3"]], dim=1),
|
||||
weights["w2"],
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_config=fp8_w8a8_moe_quant_config(
|
||||
w1_scale=torch.cat([scales["w1"], scales["w3"]], dim=1),
|
||||
w2_scale=scales["w2"],
|
||||
block_shape=[128, 128],
|
||||
),
|
||||
)
|
||||
total = torch.zeros_like(ref, dtype=torch.float32)
|
||||
fi_reference = None
|
||||
for rank in range(-1, tp_size):
|
||||
layer = _make_fp8_tp_experts(
|
||||
monkeypatch,
|
||||
1 if rank == -1 else tp_size,
|
||||
max(rank, 0),
|
||||
mock_backend=False,
|
||||
num_experts=e,
|
||||
hidden_dim=h,
|
||||
)
|
||||
for name in weights:
|
||||
prefix = "w2" if name == "w2" else "w13"
|
||||
for suffix, checkpoint in (
|
||||
("weight", weights[name]),
|
||||
("weight_scale_inv", scales[name]),
|
||||
):
|
||||
param_name = f"{prefix}_{suffix}"
|
||||
param = getattr(layer, param_name)
|
||||
param.weight_loader(param, checkpoint, param_name, name, 0)
|
||||
method = layer.quant_method
|
||||
method.process_weights_after_loading(layer)
|
||||
assert method.is_monolithic
|
||||
out = method.apply_monolithic(layer, x, logits)
|
||||
assert torch.isfinite(out).all()
|
||||
if rank == -1:
|
||||
fi_reference = out.float().clone()
|
||||
else:
|
||||
total += out.float()
|
||||
# Different FP8 GEMM reduction orders need not be bitwise identical.
|
||||
assert fi_reference is not None
|
||||
relative_l2 = (total - fi_reference).norm() / fi_reference.norm().clamp_min(
|
||||
1e-8
|
||||
)
|
||||
cross_backend_l2 = (
|
||||
fi_reference - ref.float()
|
||||
).norm() / ref.float().norm().clamp_min(1e-8)
|
||||
print(
|
||||
f"TP{tp_size} M={num_tokens}: shard_l2={relative_l2.item():.6f}, "
|
||||
f"unsharded_fi_vs_triton_l2={cross_backend_l2.item():.6f}"
|
||||
)
|
||||
assert relative_l2 < 0.01, relative_l2.item()
|
||||
@@ -1,229 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness of the fused globalize + align + sort decode metadata kernel.
|
||||
|
||||
The fused op replaces the DeepEP-v2 INDEXED cudagraph-decode chain
|
||||
(_globalize_recv_topk_idx + moe_align_block_size + count_and_sort). It is
|
||||
checked against that exact reference chain over a grid of shapes / EP ranks /
|
||||
fill distributions.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.fused_globalize_align_block_size import (
|
||||
fused_globalize_align_block_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_deep_ep_v2
|
||||
|
||||
if current_platform.is_cuda() and has_deep_ep_v2():
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
|
||||
_globalize_recv_topk_idx,
|
||||
)
|
||||
else:
|
||||
pytest.skip("Requires CUDA and DeepEP v2", allow_module_level=True)
|
||||
|
||||
|
||||
def _expert_map(global_num_experts, rank_expert_offset, local_num_experts, device):
|
||||
expert_map = torch.full((global_num_experts,), -1, device=device, dtype=torch.int32)
|
||||
rng = torch.arange(local_num_experts, device=device, dtype=torch.int32)
|
||||
expert_map[rank_expert_offset : rank_expert_offset + local_num_experts] = rng
|
||||
return expert_map
|
||||
|
||||
|
||||
def _make_case(n, topk, ep_size, local_e, block_size, seed, frac_recv, rank, fill):
|
||||
torch.manual_seed(seed)
|
||||
device = "cuda"
|
||||
global_num_experts = local_e * ep_size
|
||||
rank_idx = {"first": 0, "mid": ep_size // 2, "last": ep_size - 1}[rank]
|
||||
rank_expert_offset = rank_idx * local_e
|
||||
|
||||
if fill == "all_nonlocal":
|
||||
local_idx = torch.full((n, topk), -1, device=device, dtype=torch.int64)
|
||||
elif fill == "single":
|
||||
local_idx = torch.zeros((n, topk), device=device, dtype=torch.int64)
|
||||
elif fill == "all_local":
|
||||
local_idx = torch.randint(
|
||||
0, local_e, (n, topk), device=device, dtype=torch.int64
|
||||
)
|
||||
else:
|
||||
local_idx = torch.randint(
|
||||
-1, local_e, (n, topk), device=device, dtype=torch.int64
|
||||
)
|
||||
|
||||
num_recv = max(0, min(n, int(round(n * frac_recv))))
|
||||
if num_recv < n:
|
||||
local_idx[num_recv:] = torch.randint(
|
||||
-1, local_e, (n - num_recv, topk), device=device, dtype=torch.int64
|
||||
)
|
||||
psum = torch.zeros(4, device=device, dtype=torch.int32)
|
||||
psum[-1] = num_recv
|
||||
return local_idx, psum, rank_expert_offset, global_num_experts
|
||||
|
||||
|
||||
def _group_by_expert(sorted_ids, expert_ids, block_size, valid_len, total):
|
||||
groups: dict[int, list[int]] = {}
|
||||
for b in range(valid_len // block_size):
|
||||
eid = int(expert_ids[b].item())
|
||||
blk = sorted_ids[b * block_size : (b + 1) * block_size]
|
||||
groups.setdefault(eid, []).extend(blk[blk < total].tolist())
|
||||
return groups
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1, 7, 64, 257])
|
||||
# topk=16 with local_num_experts=28 is the Kimi K3 decode config. topk=6 and
|
||||
# block_size=16, 128 are outside the old template dispatch table: they guard
|
||||
# against a regression to fixed (topk, block_size) specialization.
|
||||
@pytest.mark.parametrize("topk", [1, 6, 8, 16])
|
||||
@pytest.mark.parametrize("block_size", [16, 32, 64, 128])
|
||||
@pytest.mark.parametrize("local_e", [28, 32])
|
||||
@pytest.mark.parametrize("rank", ["first", "mid", "last"])
|
||||
@pytest.mark.parametrize("fill", ["rand", "all_local", "all_nonlocal", "single"])
|
||||
def test_fused_matches_reference(n, topk, block_size, local_e, rank, fill):
|
||||
ep_size = 4
|
||||
local_idx, psum, reo, gne = _make_case(
|
||||
n,
|
||||
topk,
|
||||
ep_size,
|
||||
local_e,
|
||||
block_size,
|
||||
seed=n * 131 + topk,
|
||||
frac_recv=0.75,
|
||||
rank=rank,
|
||||
fill=fill,
|
||||
)
|
||||
|
||||
ref_global = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
|
||||
ref_sorted, ref_expert_ids, ref_num = moe_align_block_size(
|
||||
topk_ids=ref_global,
|
||||
block_size=block_size,
|
||||
num_experts=gne,
|
||||
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
|
||||
ignore_invalid_experts=True,
|
||||
)
|
||||
|
||||
fused_global, fused_sorted, fused_expert_ids, fused_num = (
|
||||
fused_globalize_align_block_size(
|
||||
local_idx.clone(), psum, reo, gne, local_e, block_size
|
||||
)
|
||||
)
|
||||
|
||||
total = n * topk
|
||||
assert torch.equal(fused_global, ref_global)
|
||||
assert torch.equal(fused_num, ref_num)
|
||||
|
||||
valid = int(fused_num.item())
|
||||
nb = valid // block_size
|
||||
assert torch.equal(fused_expert_ids[:nb], ref_expert_ids[:nb])
|
||||
assert bool((fused_expert_ids[nb:] == -1).all())
|
||||
assert bool((fused_sorted[valid:] == total).all())
|
||||
|
||||
ref_groups = _group_by_expert(
|
||||
ref_sorted, ref_expert_ids, block_size, int(ref_num.item()), total
|
||||
)
|
||||
fused_groups = _group_by_expert(
|
||||
fused_sorted, fused_expert_ids, block_size, valid, total
|
||||
)
|
||||
assert set(ref_groups) == set(fused_groups)
|
||||
for eid in ref_groups:
|
||||
assert sorted(ref_groups[eid]) == sorted(fused_groups[eid])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [7, 257])
|
||||
def test_fused_scatter_idx_matches_reference(n):
|
||||
"""The optional scatter output masks invalid and padded routes."""
|
||||
local_e = 28
|
||||
local_idx, psum, reo, gne = _make_case(
|
||||
n, 16, 4, local_e, 64, seed=n, frac_recv=0.75, rank="mid", fill="rand"
|
||||
)
|
||||
ref_global = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
|
||||
*_, ref_scatter = moe_align_block_size(
|
||||
topk_ids=ref_global,
|
||||
block_size=64,
|
||||
num_experts=gne,
|
||||
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
|
||||
ignore_invalid_experts=True,
|
||||
return_scatter_idx=True,
|
||||
)
|
||||
|
||||
fused_global, *_, fused_scatter = fused_globalize_align_block_size(
|
||||
local_idx.clone(), psum, reo, gne, local_e, 64, return_scatter_idx=True
|
||||
)
|
||||
assert torch.equal(fused_global, ref_global)
|
||||
assert torch.equal(fused_scatter, ref_scatter)
|
||||
|
||||
|
||||
# These have effective_work 1504 (single block) and 6068 (cooperative).
|
||||
@pytest.mark.parametrize("n", [1, 64])
|
||||
def test_fused_cuda_graph_replay(n):
|
||||
"""Both launch paths remain capturable and consume replay-time inputs."""
|
||||
topk = 16
|
||||
local_e = 28
|
||||
block_size = 64
|
||||
ep_size = 4
|
||||
|
||||
local_idx, psum, reo, gne = _make_case(
|
||||
n,
|
||||
topk,
|
||||
ep_size,
|
||||
local_e,
|
||||
block_size,
|
||||
seed=n,
|
||||
frac_recv=0.75,
|
||||
rank="mid",
|
||||
fill="rand",
|
||||
)
|
||||
graph_idx = torch.empty_like(local_idx)
|
||||
graph_psum = torch.empty_like(psum)
|
||||
|
||||
# Initialize lazy CUDA state before capture.
|
||||
fused_globalize_align_block_size(
|
||||
local_idx.clone(), psum, reo, gne, local_e, block_size
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_idx.copy_(local_idx)
|
||||
graph_psum.copy_(psum)
|
||||
graph_outputs = fused_globalize_align_block_size(
|
||||
graph_idx, graph_psum, reo, gne, local_e, block_size
|
||||
)
|
||||
|
||||
for frac_recv in (0.75, 0.5):
|
||||
psum[-1] = int(round(n * frac_recv))
|
||||
ref_idx = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
|
||||
ref_sorted, ref_expert_ids, ref_num = moe_align_block_size(
|
||||
topk_ids=ref_idx,
|
||||
block_size=block_size,
|
||||
num_experts=gne,
|
||||
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
|
||||
ignore_invalid_experts=True,
|
||||
)
|
||||
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
fused_idx, fused_sorted, fused_expert_ids, fused_num = graph_outputs
|
||||
|
||||
assert torch.equal(fused_idx, ref_idx)
|
||||
assert torch.equal(fused_num, ref_num)
|
||||
valid = int(fused_num.item())
|
||||
num_blocks = valid // block_size
|
||||
assert torch.equal(fused_expert_ids[:num_blocks], ref_expert_ids[:num_blocks])
|
||||
fused_groups = _group_by_expert(
|
||||
fused_sorted, fused_expert_ids, block_size, valid, n * topk
|
||||
)
|
||||
ref_groups = _group_by_expert(
|
||||
ref_sorted,
|
||||
ref_expert_ids,
|
||||
block_size,
|
||||
int(ref_num.item()),
|
||||
n * topk,
|
||||
)
|
||||
assert set(fused_groups) == set(ref_groups)
|
||||
for eid in ref_groups:
|
||||
assert sorted(fused_groups[eid]) == sorted(ref_groups[eid])
|
||||
@@ -1,277 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MoE fused topk kernel.
|
||||
|
||||
Run `pytest tests/kernels/moe/test_fused_topk.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
fused_topk_bias,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def torch_topk(
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
e_score_correction_bias: torch.Tensor = None,
|
||||
scoring_func: str = "softmax",
|
||||
):
|
||||
if scoring_func == "softmax":
|
||||
scores = torch.softmax(gating_output.float(), dim=-1)
|
||||
else:
|
||||
assert scoring_func == "sigmoid"
|
||||
scores = torch.sigmoid(gating_output.float())
|
||||
|
||||
if e_score_correction_bias is not None:
|
||||
num_experts = gating_output.shape[-1]
|
||||
scores_for_choice = scores.view(
|
||||
-1, num_experts
|
||||
) + e_score_correction_bias.unsqueeze(0)
|
||||
_, topk_ids = torch.topk(scores_for_choice, k=topk, dim=-1)
|
||||
topk_weights = scores.gather(1, topk_ids)
|
||||
else:
|
||||
topk_weights, topk_ids = torch.topk(scores, k=topk, dim=-1)
|
||||
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 33, 56])
|
||||
@pytest.mark.parametrize("hidden_size", [1024, 2048])
|
||||
@pytest.mark.parametrize("num_experts", [6, 16])
|
||||
@pytest.mark.parametrize("topk", [3, 4])
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
|
||||
def test_fused_topk(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
scoring_func: str,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
|
||||
topk_weights_ref, topk_ids_ref = torch_topk(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids, _ = fused_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
topk_weights_ref.to(torch.float32), topk_weights, atol=1e-2, rtol=1e-2
|
||||
)
|
||||
torch.testing.assert_close(topk_ids_ref.to(torch.int32), topk_ids, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 33, 56])
|
||||
@pytest.mark.parametrize("hidden_size", [1024, 2048])
|
||||
@pytest.mark.parametrize("num_experts", [6, 16])
|
||||
@pytest.mark.parametrize("topk", [3, 4])
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
|
||||
def test_fused_topk_bias(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
scoring_func: str,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
e_score_correction_bias = torch.randn(
|
||||
(num_experts,), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
|
||||
topk_weights_ref, topk_ids_ref = torch_topk(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
scoring_func=scoring_func,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
topk_weights_ref.to(torch.float32), topk_weights, atol=1e-2, rtol=1e-2
|
||||
)
|
||||
torch.testing.assert_close(topk_ids_ref.to(torch.int32), topk_ids, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [6, 8, 16])
|
||||
@pytest.mark.parametrize("topk", [3, 4])
|
||||
@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"])
|
||||
@pytest.mark.parametrize("bad_value", [float("nan"), float("inf")])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
|
||||
def test_fused_topk_nan_inf_clamp(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
scoring_func: str,
|
||||
bad_value: float,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
"""Regression test for the NaN/Inf clamp in topk_softmax_kernels.cu.
|
||||
|
||||
Degenerate hidden states (e.g., from CUDA graph padding) can produce
|
||||
NaN/Inf gating logits. Without the clamp, softmax/sigmoid outputs are
|
||||
NaN and the argmax loop picks expert 0 for every top-k slot (since
|
||||
"NaN > NaN" is false per IEEE 754), yielding duplicate expert IDs that
|
||||
crash downstream MoE sort kernels. The fix clamps NaN/Inf to 0 before
|
||||
argmax so index tie-breaking selects unique experts [0, 1, ..., k-1].
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 4
|
||||
hidden_size = 1024
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
|
||||
# Row 0: all normal. Rows 1-3: fully poisoned with NaN or Inf.
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
gating_output[1:, :] = bad_value
|
||||
|
||||
topk_weights, topk_ids, _ = fused_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=False,
|
||||
scoring_func=scoring_func,
|
||||
)
|
||||
|
||||
# Normal row must still match the torch reference.
|
||||
ref_weights, ref_ids = torch_topk(
|
||||
gating_output=gating_output[:1],
|
||||
topk=topk,
|
||||
renormalize=False,
|
||||
scoring_func=scoring_func,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ref_weights.to(torch.float32), topk_weights[:1], atol=1e-2, rtol=1e-2
|
||||
)
|
||||
torch.testing.assert_close(ref_ids.to(torch.int32), topk_ids[:1], atol=0, rtol=0)
|
||||
|
||||
# Poisoned rows: IDs must be unique (no duplicates) and weights must be
|
||||
# finite (no NaN/Inf propagation into downstream MoE kernels).
|
||||
for row in range(1, num_tokens):
|
||||
row_ids = topk_ids[row]
|
||||
assert row_ids.unique().numel() == topk, (
|
||||
f"Row {row} has duplicate expert IDs {row_ids.tolist()} "
|
||||
f"(bad_value={bad_value}, scoring_func={scoring_func})"
|
||||
)
|
||||
assert torch.isfinite(topk_weights[row]).all(), (
|
||||
f"Row {row} has non-finite weights {topk_weights[row].tolist()} "
|
||||
f"(bad_value={bad_value}, scoring_func={scoring_func})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [6, 8, 16])
|
||||
@pytest.mark.parametrize("topk", [3, 4])
|
||||
@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"])
|
||||
@pytest.mark.parametrize("bad_value", [float("nan"), float("inf")])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
|
||||
def test_fused_topk_bias_nan_inf_clamp(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
scoring_func: str,
|
||||
bad_value: float,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
"""Regression test: NaN/Inf in gating logits must not produce duplicate
|
||||
expert IDs or non-finite weights when e_score_correction_bias is present.
|
||||
|
||||
Same scenario as test_fused_topk_nan_inf_clamp but exercising the bias
|
||||
path (fused_topk_bias) so the fix in topk_softmax_kernels.cu is covered
|
||||
for that entry point as well.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 4
|
||||
hidden_size = 1024
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
e_score_correction_bias = torch.randn(
|
||||
(num_experts,), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
gating_output[1:, :] = bad_value
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
topk=topk,
|
||||
renormalize=False,
|
||||
scoring_func=scoring_func,
|
||||
)
|
||||
|
||||
# Normal row must still match the torch reference.
|
||||
ref_weights, ref_ids = torch_topk(
|
||||
gating_output=gating_output[:1],
|
||||
topk=topk,
|
||||
renormalize=False,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
scoring_func=scoring_func,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ref_weights.to(torch.float32), topk_weights[:1], atol=1e-2, rtol=1e-2
|
||||
)
|
||||
torch.testing.assert_close(ref_ids.to(torch.int32), topk_ids[:1], atol=0, rtol=0)
|
||||
|
||||
# Poisoned rows: IDs must be unique (no duplicates) and weights must be
|
||||
# finite (no NaN/Inf propagation into downstream MoE kernels).
|
||||
for row in range(1, num_tokens):
|
||||
row_ids = topk_ids[row]
|
||||
assert row_ids.unique().numel() == topk, (
|
||||
f"Row {row} has duplicate expert IDs {row_ids.tolist()} "
|
||||
f"(bad_value={bad_value}, scoring_func={scoring_func})"
|
||||
)
|
||||
assert torch.isfinite(topk_weights[row]).all(), (
|
||||
f"Row {row} has non-finite weights {topk_weights[row].tolist()} "
|
||||
f"(bad_value={bad_value}, scoring_func={scoring_func})"
|
||||
)
|
||||
@@ -1,513 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, fields, is_dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import get_triton_kernels_version, has_triton_kernels
|
||||
|
||||
if not has_triton_kernels():
|
||||
pytest.skip(
|
||||
"triton_kernels not found, skipping all related tests",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
import triton_kernels.swiglu
|
||||
|
||||
if get_triton_kernels_version() == "3.8":
|
||||
# 3.8: matmul_ogs -> matmul, matmul_ogs_details -> matmul_details
|
||||
import triton_kernels.matmul_details.opt_flags as opt_flags
|
||||
from triton_kernels.matmul import FlexCtx, PrecisionConfig
|
||||
else:
|
||||
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
|
||||
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
|
||||
from triton_kernels.numerics import InFlexData
|
||||
from triton_kernels.numerics_details.mxfp import downcast_to_mxfp, upcast_from_mxfp
|
||||
from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor
|
||||
from triton_kernels.tensor_details import layout
|
||||
from triton_kernels.testing import assert_close
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import (
|
||||
_sm12x_block_m,
|
||||
triton_kernel_moe_forward,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
from .utils import mxfp4_w_layouts, shuffle_weight
|
||||
|
||||
|
||||
def deshuffle(w: torch.Tensor):
|
||||
first = w[..., ::2]
|
||||
second = w[..., 1::2]
|
||||
|
||||
deshuffled = torch.concat((first, second), dim=-1)
|
||||
return deshuffled
|
||||
|
||||
|
||||
def init_compute_data(M, K, N, E, a_dtype: str, w_dtype: str, num_warps: int):
|
||||
randbits = [torch.randperm(E) for _ in range(M)]
|
||||
x_list = [
|
||||
(-1) ** i
|
||||
* ((16384 + ((i * 512) % 4096) + bits).to(torch.int16).view(torch.bfloat16))
|
||||
for i, bits in enumerate(randbits)
|
||||
]
|
||||
exp_data = torch.stack(x_list).to(device="cuda") # simulating gate_output (M, E)
|
||||
|
||||
# create input tensor
|
||||
x = torch.randn((M, K), dtype=torch.bfloat16, device="cuda")
|
||||
w1 = torch.randn((E, 2 * N, K), dtype=torch.bfloat16, device="cuda")
|
||||
w1_bias = torch.randn((E, 2 * N), dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
w2 = torch.randn((E, K, N), dtype=torch.bfloat16, device="cuda")
|
||||
w2_bias = torch.randn((E, K), dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
exp_data_tri = exp_data.clone()
|
||||
x_tri = x.clone()
|
||||
w1_tri = w1.clone()
|
||||
w2_tri = w2.clone()
|
||||
|
||||
w1_bias_tri = w1_bias.clone()
|
||||
w2_bias_tri = w2_bias.clone()
|
||||
w1_bias_tri = w1_bias_tri.to(torch.float32)
|
||||
w2_bias_tri = w2_bias_tri.to(torch.float32)
|
||||
|
||||
dtype_dict = {
|
||||
"bf16": torch.bfloat16,
|
||||
"fp8_e4m3": torch.float8_e4m3fn,
|
||||
"fp8_e5m2": torch.float8_e5m2,
|
||||
}
|
||||
|
||||
x = x.to(dtype_dict[a_dtype]).to(torch.bfloat16)
|
||||
if w_dtype != "mx4":
|
||||
# simulate quantization support on reference impl
|
||||
w1 = w1.to(dtype_dict[w_dtype]).to(torch.bfloat16)
|
||||
w2 = w2.to(dtype_dict[w_dtype]).to(torch.bfloat16)
|
||||
|
||||
# triton moe kernel use transposed shape for matmul
|
||||
w1_tri = w1_tri.transpose(-2, -1)
|
||||
w2_tri = w2_tri.transpose(-2, -1)
|
||||
|
||||
# shuffle weights
|
||||
w1_tri = shuffle_weight(w1_tri)
|
||||
w1_bias_tri = shuffle_weight(w1_bias_tri)
|
||||
|
||||
# quant triton_weights
|
||||
x_tri = x.to(dtype_dict[a_dtype])
|
||||
if w_dtype != "mx4":
|
||||
pytest.skip("NYI")
|
||||
else: # quantize to mx4
|
||||
# Padding alignment depends on the platform. On CDNA4 the scale
|
||||
# swizzle requires SCALE_K % 8 == 0 (K % 256) and
|
||||
# SCALE_N % 32 == 0 (2*N % 512), matching the production
|
||||
# alignment in mxfp4_round_up_hidden_size_and_intermediate_size.
|
||||
# On CUDA (Hopper) the scale layout pads internally, so the
|
||||
# original 64/128 alignment is sufficient.
|
||||
if current_platform.is_rocm():
|
||||
k_align, n2_align = 256, 512
|
||||
else:
|
||||
k_align, n2_align = 64, 128
|
||||
w1_bottom_pad = round_up(w1_tri.shape[1], k_align) - w1_tri.shape[1]
|
||||
w1_right_pad = round_up(w1_tri.shape[2], n2_align) - w1_tri.shape[2]
|
||||
|
||||
w2_bottom_pad = w1_right_pad // 2
|
||||
w2_right_pad = w1_bottom_pad
|
||||
|
||||
x_pad = w1_bottom_pad
|
||||
|
||||
w1_tri = F.pad(
|
||||
w1_tri,
|
||||
(0, w1_right_pad, 0, w1_bottom_pad, 0, 0),
|
||||
mode="constant",
|
||||
value=0,
|
||||
)
|
||||
w2_tri = F.pad(
|
||||
w2_tri,
|
||||
(0, w2_right_pad, 0, w2_bottom_pad, 0, 0),
|
||||
mode="constant",
|
||||
value=0,
|
||||
)
|
||||
|
||||
w1_bias_tri = F.pad(
|
||||
w1_bias_tri, (0, w1_right_pad, 0, 0), mode="constant", value=0
|
||||
)
|
||||
w2_bias_tri = F.pad(
|
||||
w2_bias_tri, (0, w2_right_pad, 0, 0), mode="constant", value=0
|
||||
)
|
||||
|
||||
x_tri = F.pad(x_tri, (0, x_pad, 0, 0), mode="constant", value=0)
|
||||
|
||||
(
|
||||
w_layout,
|
||||
w_layout_opts,
|
||||
w_scale_layout,
|
||||
w_scale_layout_opts,
|
||||
) = mxfp4_w_layouts(mx_axis=1, num_warps=num_warps)
|
||||
|
||||
w1_tri, w1_scale_tri = downcast_to_mxfp(w1_tri, torch.uint8, axis=1)
|
||||
w1 = upcast_from_mxfp(w1_tri, w1_scale_tri, torch.bfloat16, axis=1)
|
||||
|
||||
w2_tri, w2_scale_tri = downcast_to_mxfp(w2_tri, torch.uint8, axis=1)
|
||||
w2 = upcast_from_mxfp(w2_tri, w2_scale_tri, torch.bfloat16, axis=1)
|
||||
|
||||
w1_tri = convert_layout(
|
||||
wrap_torch_tensor(w1_tri, FP4), w_layout, **w_layout_opts
|
||||
)
|
||||
w1_scale_tri = convert_layout(
|
||||
wrap_torch_tensor(w1_scale_tri),
|
||||
w_scale_layout,
|
||||
**w_scale_layout_opts,
|
||||
)
|
||||
|
||||
w2_tri = convert_layout(
|
||||
wrap_torch_tensor(w2_tri, FP4), w_layout, **w_layout_opts
|
||||
)
|
||||
w2_scale_tri = convert_layout(
|
||||
wrap_torch_tensor(w2_scale_tri),
|
||||
w_scale_layout,
|
||||
**w_scale_layout_opts,
|
||||
)
|
||||
|
||||
pc1 = PrecisionConfig(
|
||||
**mx_scale_kwargs(w1_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData())
|
||||
)
|
||||
pc2 = PrecisionConfig(
|
||||
**mx_scale_kwargs(w2_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData())
|
||||
)
|
||||
|
||||
# tucuate so the rest can run properly
|
||||
w1 = w1[..., :K, : 2 * N]
|
||||
w2 = w2[..., :N, :K]
|
||||
|
||||
w1 = deshuffle(w1)
|
||||
|
||||
w1 = w1.transpose(-1, -2).contiguous()
|
||||
w2 = w2.transpose(-1, -2).contiguous()
|
||||
|
||||
return (
|
||||
x,
|
||||
w1,
|
||||
w1_bias,
|
||||
w2,
|
||||
w2_bias,
|
||||
exp_data,
|
||||
x_tri,
|
||||
w1_tri,
|
||||
w2_tri,
|
||||
exp_data_tri,
|
||||
w1_bias_tri,
|
||||
w2_bias_tri,
|
||||
pc1,
|
||||
pc2,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelConfig:
|
||||
num_hidden_layers: int = 36
|
||||
num_experts: int = 128
|
||||
experts_per_token: int = 4
|
||||
vocab_size: int = 201088
|
||||
hidden_size: int = 2880
|
||||
intermediate_size: int = 2880
|
||||
head_dim: int = 64
|
||||
num_attention_heads: int = 64
|
||||
num_key_value_heads: int = 8
|
||||
sliding_window: int = 128
|
||||
initial_context_length: int = 4096
|
||||
rope_theta: float = 150000.0
|
||||
rope_parameters_factor: float = 32.0
|
||||
rope_ntk_alpha: float = 1.0
|
||||
rope_ntk_beta: float = 32.0
|
||||
|
||||
|
||||
def swiglu(x, alpha: float = 1.702, limit: float = 1.0):
|
||||
# Note we add an extra bias of 1 to the linear layer
|
||||
x_glu, x_linear = torch.chunk(x, 2, dim=-1)
|
||||
if limit is not None:
|
||||
x_glu = x_glu.clamp(max=limit)
|
||||
out_glu = x_glu * torch.sigmoid(alpha * x_glu)
|
||||
if limit is not None:
|
||||
x_linear = x_linear.clamp(min=-limit, max=limit)
|
||||
return out_glu * (x_linear + 1)
|
||||
|
||||
|
||||
def oai_moe_forward(
|
||||
hidden_states: torch.Tensor, # (M, K)
|
||||
w1: torch.Tensor, # (E, 2N)
|
||||
w1_bias: torch.Tensor, # (E, 2N, K)
|
||||
w2: torch.Tensor, # (E, K, N)
|
||||
w2_bias: torch.Tensor, # (E, N)
|
||||
gating_output: torch.Tensor, # (M, E)
|
||||
topk: int,
|
||||
):
|
||||
# model.py 309:330, assuming gating and norm
|
||||
t = hidden_states
|
||||
experts = torch.topk(gating_output, k=topk, dim=-1, sorted=True)
|
||||
expert_weights = torch.nn.functional.softmax(experts.values, dim=1)
|
||||
expert_indices = experts.indices
|
||||
|
||||
# MLP #1
|
||||
mlp1_weight = w1[expert_indices, ...]
|
||||
mlp1_bias = w1_bias[expert_indices, ...]
|
||||
t = torch.einsum("beck,bk->bec", mlp1_weight, t) + mlp1_bias
|
||||
t = swiglu(t, limit=7)
|
||||
|
||||
# MLP #2
|
||||
mlp2_weight = w2[expert_indices, ...]
|
||||
mlp2_bias = w2_bias[expert_indices, ...]
|
||||
t = torch.einsum("beck,bek->bec", mlp2_weight, t)
|
||||
t += mlp2_bias
|
||||
|
||||
# Weighted sum of experts
|
||||
t = torch.einsum("bec,be->bc", t, expert_weights)
|
||||
|
||||
return t
|
||||
|
||||
|
||||
@dataclass
|
||||
class Case:
|
||||
a_dtype: str
|
||||
w_dtype: str
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
", ".join(f.name for f in fields(Case)),
|
||||
[
|
||||
tuple(getattr(case, f.name) for f in fields(Case))
|
||||
for case in [
|
||||
# Case(a_dtype="bf16", w_dtype="bf16"),
|
||||
# Case(a_dtype="fp8_e4m3", w_dtype="fp8_e5m2"),
|
||||
Case(a_dtype="bf16", w_dtype="mx4")
|
||||
]
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_token", [2])
|
||||
@pytest.mark.parametrize("tp", [1, 2, 4, 8])
|
||||
def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init):
|
||||
if not hasattr(layout, "make_default_matmul_mxfp4_w_layout"):
|
||||
pytest.skip("make_default_matmul_mxfp4_w_layout not available")
|
||||
|
||||
M = num_token
|
||||
E = ModelConfig.num_experts
|
||||
K = ModelConfig.hidden_size
|
||||
N = ModelConfig.intermediate_size // tp
|
||||
topk = ModelConfig.experts_per_token
|
||||
|
||||
(
|
||||
x,
|
||||
w1,
|
||||
w1_bias,
|
||||
w2,
|
||||
w2_bias,
|
||||
exp_data,
|
||||
x_tri,
|
||||
w1_tri,
|
||||
w2_tri,
|
||||
exp_data_tri,
|
||||
w1_bias_tri,
|
||||
w2_bias_tri,
|
||||
pc1,
|
||||
pc2,
|
||||
) = init_compute_data(M, K, N, E, a_dtype, w_dtype, num_warps=8)
|
||||
|
||||
if current_platform.is_device_capability_family(
|
||||
100
|
||||
) or current_platform.is_device_capability_family(120):
|
||||
# Mirrors `_swizzle_mxfp4`; SM12x also needs epilogue_subtile=1.
|
||||
constraints = {
|
||||
"is_persistent": True,
|
||||
"epilogue_subtile": 1,
|
||||
}
|
||||
opt_flags.update_opt_flags_constraints(constraints)
|
||||
|
||||
if a_dtype == "bf16" and w_dtype == "mx4":
|
||||
quant_config = mxfp4_w4a16_moe_quant_config(
|
||||
w1_scale=pc1,
|
||||
w2_scale=pc2,
|
||||
w1_bias=w1_bias_tri,
|
||||
w2_bias=w2_bias_tri,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Quantization configuration for activation={a_dtype} and weight={w_dtype} "
|
||||
f"has not been implemented."
|
||||
)
|
||||
|
||||
out_triton_monolithic = triton_kernel_moe_forward(
|
||||
hidden_states=x_tri,
|
||||
w1=w1_tri,
|
||||
w2=w2_tri,
|
||||
gating_output=exp_data_tri,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
out_triton_monolithic = out_triton_monolithic[..., :K]
|
||||
|
||||
out_ref = oai_moe_forward(
|
||||
hidden_states=x,
|
||||
w1=w1,
|
||||
w1_bias=w1_bias,
|
||||
w2=w2,
|
||||
w2_bias=w2_bias,
|
||||
gating_output=exp_data,
|
||||
topk=topk,
|
||||
)
|
||||
assert_close(ref=out_ref, tri=out_triton_monolithic, maxtol=0.025, rmstol=0.005)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "num_experts", "expected"),
|
||||
[
|
||||
(4, 32, 16), # bs=1 decode: below the 16-row floor
|
||||
(32 * 20, 32, 32), # 20 tokens/expert rounds up to 32
|
||||
(32 * 33, 32, 32), # heuristic would pick 64; capped for 99KB smem
|
||||
(12000, 32, 32), # prefill: heuristic would pick 128
|
||||
],
|
||||
)
|
||||
def test_sm12x_block_m(num_rows, num_experts, expected):
|
||||
assert _sm12x_block_m(num_rows, num_experts) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sm12x", [True, False])
|
||||
def test_gpt_oss_priority_prefers_marlin_on_sm12x(
|
||||
monkeypatch: pytest.MonkeyPatch, sm12x: bool
|
||||
):
|
||||
"""SM12x runs the OAI Triton kernels at small tiles only, so MARLIN must
|
||||
stay the default there while TRITON remains selectable explicitly."""
|
||||
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
mxfp4_oracle.current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda family: sm12x and family == 120,
|
||||
)
|
||||
order = mxfp4_oracle._get_priority_backends_for_gpt_oss()
|
||||
triton_first = order.index(Mxfp4MoeBackend.TRITON) < order.index(
|
||||
Mxfp4MoeBackend.MARLIN
|
||||
)
|
||||
assert triton_first is not sm12x
|
||||
|
||||
|
||||
def test_unit_shuffle():
|
||||
N = ModelConfig.intermediate_size
|
||||
K = ModelConfig.hidden_size
|
||||
m = torch.randn((K, 2 * N), dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
x = torch.randn(K, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
m_shuffled = shuffle_weight(m)
|
||||
|
||||
out_ref = x @ m
|
||||
out_ref = swiglu(out_ref, limit=1.0)
|
||||
|
||||
out = x @ m_shuffled
|
||||
out = triton_kernels.swiglu.swiglu_torch(
|
||||
out,
|
||||
alpha=1.702,
|
||||
precision_config=triton_kernels.swiglu.PrecisionConfig(limit=1.0),
|
||||
)
|
||||
|
||||
assert_close(ref=out_ref, tri=out)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_tokens", [1, 33, 512])
|
||||
@pytest.mark.parametrize("n_experts,topk", [(32, 4), (128, 4)])
|
||||
def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
|
||||
"""routing_data_from_sparse_topk must produce routing structures
|
||||
identical to make_routing_data for the same topk result."""
|
||||
from vllm.model_executor.layers.fused_moe.experts import (
|
||||
gpt_oss_triton_kernels_moe as gptoss_moe,
|
||||
)
|
||||
|
||||
make_routing_data = gptoss_moe.make_routing_data
|
||||
routing_data_from_sparse_topk = gptoss_moe.routing_data_from_sparse_topk
|
||||
|
||||
if gptoss_moe.triton_kernels_version == "3.5.1":
|
||||
pytest.skip("SparseMatrix path requires triton_kernels v3.6.0+")
|
||||
|
||||
from triton_kernels.topk import topk as topk_fn
|
||||
|
||||
torch.manual_seed(0)
|
||||
logits = torch.randn(n_tokens, n_experts, dtype=torch.bfloat16, device="cuda")
|
||||
sparse_topk = topk_fn(logits, topk, apply_softmax=True)
|
||||
assert not isinstance(sparse_topk, tuple)
|
||||
|
||||
rd_new, gather_new, scatter_new = routing_data_from_sparse_topk(
|
||||
sparse_topk, n_experts
|
||||
)
|
||||
rd_ref, gather_ref, scatter_ref = make_routing_data(
|
||||
sparse_topk.indx.to(torch.long), sparse_topk.vals, n_experts
|
||||
)
|
||||
|
||||
def assert_equivalent(new, ref, path="RoutingData"):
|
||||
"""Recursively compare tensor-valued leaves of routing structures."""
|
||||
if isinstance(ref, torch.Tensor):
|
||||
torch.testing.assert_close(new, ref, msg=lambda m: f"{path}: {m}")
|
||||
elif isinstance(ref, (int, float, bool, str)) or ref is None:
|
||||
assert new == ref, f"{path}: {new!r} != {ref!r}"
|
||||
elif is_dataclass(ref):
|
||||
for f in fields(ref):
|
||||
assert_equivalent(
|
||||
getattr(new, f.name), getattr(ref, f.name), f"{path}.{f.name}"
|
||||
)
|
||||
elif callable(ref):
|
||||
assert callable(new), f"{path}: callable vs {type(new)}"
|
||||
elif hasattr(ref, "__dict__"):
|
||||
for k, v in vars(ref).items():
|
||||
assert_equivalent(getattr(new, k), v, f"{path}.{k}")
|
||||
elif hasattr(type(ref), "__slots__"):
|
||||
for k in type(ref).__slots__:
|
||||
assert_equivalent(getattr(new, k), getattr(ref, k), f"{path}.{k}")
|
||||
else:
|
||||
assert type(new) is type(ref), f"{path}: {type(new)} vs {type(ref)}"
|
||||
|
||||
assert_equivalent(rd_new, rd_ref)
|
||||
if isinstance(gather_ref, torch.Tensor):
|
||||
torch.testing.assert_close(gather_new, gather_ref)
|
||||
torch.testing.assert_close(scatter_new, scatter_ref)
|
||||
return
|
||||
torch.testing.assert_close(gather_new.src_indx, gather_ref.src_indx)
|
||||
torch.testing.assert_close(gather_new.dst_indx, gather_ref.dst_indx)
|
||||
torch.testing.assert_close(scatter_new.src_indx, scatter_ref.src_indx)
|
||||
torch.testing.assert_close(scatter_new.dst_indx, scatter_ref.dst_indx)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_tokens", [1, 33, 4099])
|
||||
@pytest.mark.parametrize("n_experts,topk", [(128, 4), (256, 6)])
|
||||
def test_make_routing_data_tk38_matches_reference(n_tokens, n_experts, topk):
|
||||
from vllm.model_executor.layers.fused_moe.experts import (
|
||||
gpt_oss_triton_kernels_moe as gptoss_moe,
|
||||
)
|
||||
|
||||
if gptoss_moe.triton_kernels_version != "3.8":
|
||||
pytest.skip("requires triton_kernels 3.8")
|
||||
|
||||
torch.manual_seed(0)
|
||||
topk_ids = torch.stack(
|
||||
[torch.randperm(n_experts, device="cuda")[:topk] for _ in range(n_tokens)]
|
||||
).to(torch.int32)
|
||||
topk_ids[torch.rand(topk_ids.shape, device="cuda") < 0.5] = -1
|
||||
topk_weights = torch.rand(topk_ids.shape, device="cuda")
|
||||
|
||||
routing, gather_tok, scatter_tok = gptoss_moe.make_routing_data(
|
||||
topk_ids, topk_weights, n_experts
|
||||
)
|
||||
|
||||
flat = topk_ids.reshape(-1).to(torch.int64)
|
||||
order = torch.argsort(flat.clamp(min=0), stable=True)
|
||||
counts = torch.bincount(flat.clamp(min=0), minlength=n_experts).to(torch.int32)
|
||||
weights = topk_weights.reshape(-1) * (flat >= 0).to(torch.float32)
|
||||
|
||||
assert torch.equal(routing.ragged.slice_sizes, counts)
|
||||
assert torch.equal(gather_tok, (order // topk).to(torch.int32))
|
||||
assert torch.equal(scatter_tok, gather_tok)
|
||||
assert torch.equal(routing.gate_scal, weights[order])
|
||||
assert routing.n_expts_act == topk
|
||||
assert routing.n_valid == flat.numel()
|
||||
@@ -1,401 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MoE grouped topk kernel.
|
||||
|
||||
Run `pytest tests/kernels/moe/test_grouped_topk.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
VllmConfig,
|
||||
get_cached_compilation_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
GroupedTopk,
|
||||
GroupedTopKRouter,
|
||||
fused_grouped_topk,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
def _run_single_group_topk(
|
||||
logits: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
topk: int,
|
||||
*,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return fused_grouped_topk(
|
||||
hidden_states=torch.empty(
|
||||
(logits.shape[0], 0), dtype=logits.dtype, device=logits.device
|
||||
),
|
||||
gating_output=logits,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
e_score_correction_bias=bias,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
|
||||
def _single_group_reference(
|
||||
logits: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
topk: int,
|
||||
*,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if scoring_func == "sigmoid":
|
||||
scores = 0.5 * torch.tanh(0.5 * logits.float()) + 0.5
|
||||
else:
|
||||
scores = torch.softmax(logits, dim=-1).float()
|
||||
indices = torch.argsort(
|
||||
scores + bias.float(), dim=-1, descending=True, stable=True
|
||||
)[:, :topk]
|
||||
values = scores.gather(1, indices)
|
||||
if renormalize:
|
||||
values /= values.sum(dim=-1, keepdim=True) + 1e-20
|
||||
values *= routed_scaling_factor
|
||||
return values, indices.to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("skip_padding", [False, True])
|
||||
def test_kimi_grouped_topk_masks_padding_when_enabled(skip_padding: bool):
|
||||
hidden_states = torch.randn((2, 16), device="cuda")
|
||||
gating_output = torch.randn((2, 896), device="cuda")
|
||||
is_padding = torch.tensor([False, True], device="cuda")
|
||||
router = GroupedTopKRouter(
|
||||
top_k=16,
|
||||
global_num_experts=896,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
skip_padding=skip_padding,
|
||||
)
|
||||
|
||||
with set_forward_context(None, VllmConfig(), is_padding=is_padding):
|
||||
topk_weights, topk_ids = router._compute_routing(
|
||||
hidden_states, gating_output, torch.int32
|
||||
)
|
||||
|
||||
assert torch.all(topk_ids[0] >= 0)
|
||||
if skip_padding:
|
||||
assert torch.all(topk_ids[1] == -1)
|
||||
assert torch.all(topk_weights[1] == 0)
|
||||
else:
|
||||
assert torch.all(topk_ids[1] >= 0)
|
||||
assert torch.all(topk_weights[1] > 0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("n_token", [1, 33, 64])
|
||||
@pytest.mark.parametrize("n_hidden", [1024, 2048])
|
||||
@pytest.mark.parametrize(
|
||||
"n_expert,topk,num_expert_group,topk_group",
|
||||
[
|
||||
(16, 2, 8, 2),
|
||||
(128, 2, 8, 2),
|
||||
(256, 8, 8, 4),
|
||||
(384, 8, 1, 1),
|
||||
(512, 22, 1, 1),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"])
|
||||
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 2.5])
|
||||
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("bias_dtype", [torch.float32])
|
||||
def test_grouped_topk(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
n_token: int,
|
||||
n_hidden: int,
|
||||
n_expert: int,
|
||||
topk: int,
|
||||
num_expert_group: int,
|
||||
topk_group: int,
|
||||
renormalize: bool,
|
||||
scoring_func: str,
|
||||
routed_scaling_factor: float,
|
||||
input_dtype: torch.dtype,
|
||||
bias_dtype: torch.dtype,
|
||||
):
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=["all", "+grouped_topk"])
|
||||
)
|
||||
get_cached_compilation_config.cache_clear()
|
||||
|
||||
set_random_seed(0)
|
||||
hidden_states = torch.randn((n_token, n_hidden), dtype=input_dtype, device="cuda")
|
||||
gating_output = torch.randn((n_token, n_expert), dtype=input_dtype, device="cuda")
|
||||
e_score_correction_bias = torch.randn((n_expert,), dtype=bias_dtype, device="cuda")
|
||||
|
||||
with set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
m.setenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "0")
|
||||
m.setattr(envs, "VLLM_BATCH_INVARIANT", True)
|
||||
grouped_topk = GroupedTopk(
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
assert grouped_topk._forward_method.__name__ == "forward_cuda"
|
||||
baseline_topk_weights, baseline_topk_ids = grouped_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
test_topk_weights, test_topk_ids = fused_grouped_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
baseline_topk_weights, test_topk_weights, atol=2e-2, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(baseline_topk_ids, test_topk_ids, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
def test_grouped_topk_single_group_large_batch():
|
||||
set_random_seed(0)
|
||||
logits = torch.randn((1536, 896), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.randn((896,), dtype=torch.float32, device="cuda")
|
||||
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits, bias, 16, scoring_func="sigmoid", renormalize=True
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits, bias, 16, scoring_func="sigmoid", renormalize=True
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts,topk,input_dtype,bias_dtype",
|
||||
[
|
||||
(512, 9, torch.bfloat16, torch.float32),
|
||||
(512, 16, torch.float16, torch.float16),
|
||||
(513, 9, torch.float32, torch.bfloat16),
|
||||
(513, 16, torch.bfloat16, torch.float32),
|
||||
(895, 9, torch.float16, torch.bfloat16),
|
||||
(896, 16, torch.float32, torch.float16),
|
||||
(897, 9, torch.bfloat16, torch.bfloat16),
|
||||
(897, 16, torch.float16, torch.float32),
|
||||
(1024, 9, torch.float32, torch.bfloat16),
|
||||
(1024, 16, torch.bfloat16, torch.float16),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"scoring_func,renormalize,routed_scaling_factor",
|
||||
[
|
||||
("sigmoid", True, 1.0),
|
||||
("sigmoid", False, 2.5),
|
||||
("softmax", True, 2.5),
|
||||
("softmax", False, 1.0),
|
||||
],
|
||||
)
|
||||
def test_grouped_topk_single_group_tiers(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
input_dtype: torch.dtype,
|
||||
bias_dtype: torch.dtype,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float,
|
||||
):
|
||||
set_random_seed(7)
|
||||
logits = torch.randn((17, num_experts), dtype=input_dtype, device="cuda")
|
||||
bias = torch.randn((num_experts,), dtype=bias_dtype, device="cuda")
|
||||
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts,topk,scoring_func",
|
||||
[
|
||||
(128, 8, "sigmoid"),
|
||||
(129, 8, "sigmoid"),
|
||||
(257, 8, "sigmoid"),
|
||||
(385, 8, "sigmoid"),
|
||||
(512, 9, "sigmoid"),
|
||||
(513, 9, "sigmoid"),
|
||||
(769, 9, "sigmoid"),
|
||||
(897, 16, "sigmoid"),
|
||||
(1024, 16, "sigmoid"),
|
||||
(128, 4, "softmax"),
|
||||
(128, 5, "softmax"),
|
||||
(129, 8, "softmax"),
|
||||
(161, 8, "softmax"),
|
||||
(256, 9, "softmax"),
|
||||
(257, 8, "softmax"),
|
||||
(512, 9, "softmax"),
|
||||
(512, 17, "softmax"),
|
||||
(512, 23, "softmax"),
|
||||
(513, 8, "softmax"),
|
||||
(577, 9, "softmax"),
|
||||
(769, 9, "softmax"),
|
||||
(897, 9, "softmax"),
|
||||
(1024, 16, "softmax"),
|
||||
],
|
||||
)
|
||||
def test_grouped_topk_single_group_capacity_tiers(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
scoring_func: str,
|
||||
):
|
||||
set_random_seed(11)
|
||||
logits = torch.randn((3, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
|
||||
def test_grouped_topk_single_group_stable_ties(num_experts: int):
|
||||
logits = torch.zeros((1, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
|
||||
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
16,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
|
||||
expected_values = torch.full((1, 16), 2.5 / 16, dtype=torch.float32, device="cuda")
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
|
||||
@pytest.mark.parametrize("num_finite", [0, 15])
|
||||
@pytest.mark.parametrize("renormalize", [False, True])
|
||||
def test_grouped_topk_single_group_nonfinite_scores(
|
||||
num_experts: int, num_finite: int, renormalize: bool
|
||||
):
|
||||
logits = torch.full(
|
||||
(1, num_experts), float("nan"), dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
if num_finite:
|
||||
logits[0, :num_finite] = torch.arange(
|
||||
num_finite, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
logits[0, num_finite] = torch.inf
|
||||
logits[0, num_finite + 1] = -torch.inf
|
||||
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
|
||||
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
16,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
if num_finite == 0:
|
||||
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
|
||||
if renormalize:
|
||||
expected_values = torch.full(
|
||||
(1, 16), 1 / 16, dtype=torch.float32, device="cuda"
|
||||
)
|
||||
else:
|
||||
expected_values = torch.zeros((1, 16), dtype=torch.float32, device="cuda")
|
||||
else:
|
||||
expected_ids = torch.cat(
|
||||
(
|
||||
torch.arange(num_finite - 1, -1, -1, dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([num_finite], dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
)[None]
|
||||
finite_values = logits[0, :num_finite].float().sigmoid().flip(0)
|
||||
if renormalize:
|
||||
finite_values /= finite_values.sum()
|
||||
finite_values *= 2.5
|
||||
expected_values = torch.cat(
|
||||
(finite_values, torch.zeros(1, dtype=torch.float32, device="cuda"))
|
||||
)[None]
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
@@ -1,373 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test comparing Marlin INT4 MoE vs FlashInfer TRT-LLM MXINT4 MoE."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
fused_marlin_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
|
||||
TrtLlmMxint4ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
grouped_topk,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import (
|
||||
prepare_static_weights_for_trtllm_mxint4_moe,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
def mxint4_quantize(
|
||||
x: torch.Tensor, sf_vec_size: int = 32
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize BF16 tensor to MXINT4 with block scaling (group_size=sf_vec_size).
|
||||
|
||||
Returns:
|
||||
- uint8 packed (2 INT4/byte): [..., k//2] - stores SIGNED INT4 [-8, 7]
|
||||
- scales in BF16: [..., k//sf_vec_size]
|
||||
|
||||
"""
|
||||
x_reshaped = x.reshape(-1, sf_vec_size)
|
||||
x_max = x_reshaped.max(dim=-1, keepdim=True)[0].to(torch.float32)
|
||||
x_min = x_reshaped.min(dim=-1, keepdim=True)[0].to(torch.float32)
|
||||
x_max = x_max * 8.0 / 7.0
|
||||
amax = torch.where(x_max > -x_min, x_max, -x_min)
|
||||
scales = amax / 8.0
|
||||
x_scaled = x_reshaped * scales.reciprocal()
|
||||
x_int8 = (
|
||||
x_scaled.round().clamp(-8, 7).to(torch.int8).reshape(-1, sf_vec_size // 2, 2)
|
||||
)
|
||||
x_int4 = (x_int8[..., 0] & 0x0F) | ((x_int8[..., 1] & 0x0F) << 4)
|
||||
return (
|
||||
x_int4.to(torch.uint8).reshape(*x.shape[:-1], x.shape[-1] // 2),
|
||||
scales.to(x.dtype).reshape(*x.shape[:-1], x.shape[-1] // sf_vec_size),
|
||||
)
|
||||
|
||||
|
||||
def mxint4_quantize_moe_weights(
|
||||
weights_bf16: torch.Tensor, group_size: int = 32
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize MoE weights [e, n, k] to MxInt4 format.
|
||||
|
||||
Args:
|
||||
weights_bf16: BF16 weights of shape [num_experts, out_features, in_features]
|
||||
group_size: Quantization group size (default: 32)
|
||||
|
||||
Returns:
|
||||
- weights_mxint4: Quantized weights [e, n, k//2] uint8
|
||||
- scales_mxint4: Quantization scales [e, n, k//group_size] bf16
|
||||
|
||||
"""
|
||||
e = weights_bf16.shape[0]
|
||||
weight_list = []
|
||||
scale_list = []
|
||||
|
||||
for i in range(e):
|
||||
w_q, w_s = mxint4_quantize(weights_bf16[i], sf_vec_size=group_size)
|
||||
weight_list.append(w_q)
|
||||
scale_list.append(w_s)
|
||||
|
||||
return torch.stack(weight_list), torch.stack(scale_list)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"mxint4_quantize",
|
||||
"mxint4_quantize_moe_weights",
|
||||
"marlin_quantize_moe_weights",
|
||||
]
|
||||
|
||||
|
||||
def test_trtllm_mxint4_activation_supports_vllm_gated_silu():
|
||||
assert TrtLlmMxint4ExpertsMonolithic._supports_activation(MoEActivation.SILU)
|
||||
assert TrtLlmMxint4ExpertsMonolithic._supports_activation(MoEActivation.SWIGLUOAI)
|
||||
assert not TrtLlmMxint4ExpertsMonolithic._supports_activation(
|
||||
MoEActivation.RELU2_NO_MUL
|
||||
)
|
||||
|
||||
|
||||
def marlin_quantize_moe_weights(
|
||||
weights_bf16: torch.Tensor, group_size: int = 32
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize MoE weights [e, n, k] to Marlin INT4 format.
|
||||
|
||||
Args:
|
||||
weights_bf16: BF16 weights of shape [num_experts, out_features, in_features]
|
||||
group_size: Quantization group size (default: 32)
|
||||
|
||||
Returns:
|
||||
- weights_marlin: Marlin quantized weights [e, k//8, n] int32
|
||||
- scales_marlin: Marlin quantization scales [e, k//group_size, n] bf16
|
||||
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
|
||||
marlin_quantize,
|
||||
)
|
||||
|
||||
e, n, k = weights_bf16.shape
|
||||
weight_list = []
|
||||
scale_list = []
|
||||
|
||||
for i in range(e):
|
||||
# Transpose for Marlin: [n, k] → [k, n]
|
||||
w_t = weights_bf16[i].T.contiguous()
|
||||
_, w_q, w_s = marlin_quantize(w_t, scalar_types.uint4b8, group_size)
|
||||
weight_list.append(w_q)
|
||||
scale_list.append(w_s)
|
||||
|
||||
# Stack to get [e, ...] shape
|
||||
weights_marlin = torch.stack(weight_list) # [e, k // 8, n]
|
||||
scales_marlin = torch.stack(scale_list) # [e, k // group_size, n]
|
||||
|
||||
return weights_marlin, scales_marlin
|
||||
|
||||
|
||||
TRTLLM_GEN_AVAILABLE = (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TRTLLM_GEN_AVAILABLE, reason="Skip for non SM100")
|
||||
@pytest.mark.parametrize("m", [1, 33])
|
||||
@pytest.mark.parametrize("n", [7168])
|
||||
@pytest.mark.parametrize("k", [512])
|
||||
@pytest.mark.parametrize("e", [384])
|
||||
@pytest.mark.parametrize("topk", [8])
|
||||
@pytest.mark.parametrize("group_size", [32])
|
||||
def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group_size):
|
||||
"""Compare Marlin INT4 MoE vs FlashInfer TRT-LLM MXINT4 MoE.
|
||||
|
||||
Uses mxint4_quantize() to generate common INT4 weights + BF16 scales,
|
||||
then runs both Marlin and TRT-LLM kernels and compares outputs.
|
||||
"""
|
||||
pytest.importorskip("flashinfer")
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_INT4", "1")
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
dtype = torch.bfloat16
|
||||
|
||||
# DeepSeekV3 routing config (from Kimi-K2-Thinking config.json)
|
||||
n_group = 1 # n_group from model config
|
||||
topk_group = 1 # topk_group from model config
|
||||
routed_scaling = 2.827 # routed_scaling_factor from model config
|
||||
|
||||
# Input - realistic activation range for LLM (after LayerNorm: mean~0, std~1)
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) * 0.5
|
||||
|
||||
# Generate routing logits and bias (DeepSeekV3 expects float logits)
|
||||
# Realistic ranges: logits typically [-3, 3], bias [-2, 2]
|
||||
routing_logits = torch.randn((m, e), device="cuda", dtype=torch.float32) * 1.5
|
||||
routing_bias = torch.randn(e, device="cuda", dtype=torch.float32) * 0.8
|
||||
|
||||
# 1. Generate BF16 weights (SHARED between both paths)
|
||||
# Realistic weight initialization: Xavier/Glorot uniform scaling
|
||||
# std = sqrt(2 / (fan_in + fan_out))
|
||||
std_w1 = (2.0 / (k + 2 * n)) ** 0.5
|
||||
std_w2 = (2.0 / (n + k)) ** 0.5
|
||||
w1_bf16 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) * std_w1
|
||||
w2_bf16 = torch.randn((e, k, n), device="cuda", dtype=dtype) * std_w2
|
||||
|
||||
# === Path 1: TRT-LLM FlashInfer MXINT4 MoE ===
|
||||
# Similar to: if self.use_flashinfer_mxint4_moe
|
||||
# Quantize using MXINT4 method (signed INT4)
|
||||
w1_int4, w1_scales = mxint4_quantize_moe_weights(w1_bf16, group_size)
|
||||
w2_int4, w2_scales = mxint4_quantize_moe_weights(w2_bf16, group_size)
|
||||
|
||||
trtllm_weights = prepare_static_weights_for_trtllm_mxint4_moe(
|
||||
gemm1_weights=w1_int4,
|
||||
gemm1_scales=w1_scales,
|
||||
gemm2_weights=w2_int4,
|
||||
gemm2_scales=w2_scales,
|
||||
)
|
||||
|
||||
from flashinfer import RoutingMethodType
|
||||
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
|
||||
|
||||
# Routing handled internally by trtllm_mxint4_block_scale_moe
|
||||
trtllm_output = trtllm_mxint4_block_scale_moe(
|
||||
routing_logits=routing_logits,
|
||||
routing_bias=routing_bias.to(torch.bfloat16),
|
||||
hidden_states=a,
|
||||
gemm1_weights=trtllm_weights["gemm1_weights"],
|
||||
gemm1_weights_scale=trtllm_weights["gemm1_scales"],
|
||||
gemm1_alpha=None,
|
||||
gemm1_beta=None,
|
||||
gemm1_clamp_limit=None,
|
||||
gemm2_weights=trtllm_weights["gemm2_weights"],
|
||||
gemm2_weights_scale=trtllm_weights["gemm2_scales"],
|
||||
num_experts=e,
|
||||
top_k=topk,
|
||||
n_group=n_group,
|
||||
topk_group=topk_group,
|
||||
intermediate_size=n,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=e,
|
||||
routed_scaling_factor=routed_scaling,
|
||||
routing_method_type=RoutingMethodType.DeepSeekV3,
|
||||
enable_pdl=None,
|
||||
output=None,
|
||||
tune_max_num_tokens=8192,
|
||||
).to(dtype)
|
||||
|
||||
# === Path 2: Marlin INT4 MoE ===
|
||||
# Similar to: else (non-flashinfer path)
|
||||
# Quantize using Marlin's method (UINT4b8)
|
||||
w1_marlin, w1_scales_marlin = marlin_quantize_moe_weights(w1_bf16, group_size)
|
||||
w2_marlin, w2_scales_marlin = marlin_quantize_moe_weights(w2_bf16, group_size)
|
||||
|
||||
# Use production routing kernel (same as router.select_experts internally uses)
|
||||
topk_weights, topk_ids = grouped_topk(
|
||||
hidden_states=a,
|
||||
gating_output=routing_logits,
|
||||
topk=topk,
|
||||
renormalize=False, # DeepSeekV3 doesn't renormalize
|
||||
num_expert_group=n_group,
|
||||
topk_group=topk_group,
|
||||
scoring_func="sigmoid", # DeepSeekV3 uses sigmoid
|
||||
routed_scaling_factor=routed_scaling,
|
||||
e_score_correction_bias=routing_bias,
|
||||
)
|
||||
|
||||
marlin_output = fused_marlin_moe(
|
||||
hidden_states=a,
|
||||
w1=w1_marlin,
|
||||
w2=w2_marlin,
|
||||
bias1=None,
|
||||
bias2=None,
|
||||
w1_scale=w1_scales_marlin,
|
||||
w2_scale=w2_scales_marlin,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
quant_type_id=scalar_types.uint4b8.id,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
global_scale1=None,
|
||||
global_scale2=None,
|
||||
input_global_scale1=None,
|
||||
input_global_scale2=None,
|
||||
w1_zeros=None,
|
||||
w2_zeros=None,
|
||||
input_dtype=dtype,
|
||||
)
|
||||
|
||||
# Sanity check: manually compute BF16 reference for comparison
|
||||
# Use same routing as Marlin path for consistency
|
||||
bf16_output = torch.zeros((m, k), device="cuda", dtype=dtype)
|
||||
for token_idx in range(m):
|
||||
for expert_rank in range(topk):
|
||||
expert_id = topk_ids[token_idx, expert_rank].item()
|
||||
weight = topk_weights[token_idx, expert_rank].item()
|
||||
# w1: [2*n, k] @ [k] -> [2*n]
|
||||
up_gate = a[token_idx] @ w1_bf16[expert_id].T # [2*n]
|
||||
gate, up = up_gate.chunk(2, dim=0)
|
||||
intermediate = torch.nn.functional.silu(gate) * up # [n]
|
||||
# w2: [k, n] @ [n] -> [k]
|
||||
expert_out = intermediate @ w2_bf16[expert_id].T # [k]
|
||||
bf16_output[token_idx] += weight * expert_out
|
||||
# Compare against BF16 reference.
|
||||
torch.testing.assert_close(marlin_output, bf16_output, atol=0.3, rtol=1.0)
|
||||
torch.testing.assert_close(trtllm_output, bf16_output, atol=0.3, rtol=1.0)
|
||||
|
||||
# Compare against each other for sanity.
|
||||
# Note: Different quantization schemes (UINT4b8 vs signed MXINT4) cause
|
||||
# some differences
|
||||
torch.testing.assert_close(marlin_output, trtllm_output, atol=0.3, rtol=6.0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TRTLLM_GEN_AVAILABLE, reason="Skip for non SM100")
|
||||
@pytest.mark.parametrize("m", [1, 33])
|
||||
@pytest.mark.parametrize("n", [7168])
|
||||
@pytest.mark.parametrize("k", [512])
|
||||
@pytest.mark.parametrize("e", [384])
|
||||
@pytest.mark.parametrize("topk", [8])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_trtllm_mxint4_moe_wrapper(m, n, k, e, topk):
|
||||
"""Test that the flashinfer_trtllm_mxint4_moe wrapper matches the raw
|
||||
trtllm_mxint4_block_scale_moe kernel call."""
|
||||
pytest.importorskip("flashinfer")
|
||||
from flashinfer import RoutingMethodType
|
||||
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import (
|
||||
flashinfer_trtllm_mxint4_moe,
|
||||
)
|
||||
|
||||
set_random_seed(0)
|
||||
dtype = torch.bfloat16
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) * 0.5
|
||||
router_logits = torch.randn((m, e), device="cuda", dtype=torch.float32) * 1.5
|
||||
routing_bias = torch.randn(e, device="cuda", dtype=torch.float32) * 0.8
|
||||
|
||||
std_w1 = (2.0 / (k + 2 * n)) ** 0.5
|
||||
std_w2 = (2.0 / (n + k)) ** 0.5
|
||||
w1_bf16 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) * std_w1
|
||||
w2_bf16 = torch.randn((e, k, n), device="cuda", dtype=dtype) * std_w2
|
||||
|
||||
w1_int4, w1_scales = mxint4_quantize_moe_weights(w1_bf16)
|
||||
w2_int4, w2_scales = mxint4_quantize_moe_weights(w2_bf16)
|
||||
|
||||
prepared = prepare_static_weights_for_trtllm_mxint4_moe(
|
||||
gemm1_weights=w1_int4,
|
||||
gemm1_scales=w1_scales,
|
||||
gemm2_weights=w2_int4,
|
||||
gemm2_scales=w2_scales,
|
||||
)
|
||||
|
||||
# Raw kernel call (reference)
|
||||
raw_out = trtllm_mxint4_block_scale_moe(
|
||||
routing_logits=router_logits.to(torch.float32),
|
||||
routing_bias=routing_bias.to(torch.bfloat16),
|
||||
hidden_states=a,
|
||||
gemm1_weights=prepared["gemm1_weights"].data,
|
||||
gemm1_weights_scale=prepared["gemm1_scales"].data,
|
||||
gemm1_alpha=None,
|
||||
gemm1_beta=None,
|
||||
gemm1_clamp_limit=None,
|
||||
gemm2_weights=prepared["gemm2_weights"].data,
|
||||
gemm2_weights_scale=prepared["gemm2_scales"].data,
|
||||
num_experts=e,
|
||||
top_k=topk,
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
intermediate_size=n,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=e,
|
||||
routed_scaling_factor=None,
|
||||
routing_method_type=RoutingMethodType.DeepSeekV3,
|
||||
enable_pdl=None,
|
||||
output=None,
|
||||
tune_max_num_tokens=8192,
|
||||
)
|
||||
if not isinstance(raw_out, torch.Tensor):
|
||||
raw_out = raw_out[0]
|
||||
raw_out = raw_out.to(dtype)
|
||||
|
||||
# Wrapper call
|
||||
wrapper_out = flashinfer_trtllm_mxint4_moe(
|
||||
x=a,
|
||||
router_logits=router_logits,
|
||||
w13_weight_packed=prepared["gemm1_weights"],
|
||||
w13_weight_scale=prepared["gemm1_scales"],
|
||||
w2_weight_packed=prepared["gemm2_weights"],
|
||||
w2_weight_scale=prepared["gemm2_scales"],
|
||||
global_num_experts=e,
|
||||
top_k=topk,
|
||||
intermediate_size_per_partition=n,
|
||||
local_num_experts=e,
|
||||
ep_rank=0,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
e_score_correction_bias=routing_bias,
|
||||
routing_method_type=RoutingMethodType.DeepSeekV3,
|
||||
)
|
||||
|
||||
assert wrapper_out.shape == (m, k)
|
||||
assert wrapper_out.dtype == dtype
|
||||
torch.testing.assert_close(wrapper_out, raw_out, atol=0.0, rtol=0.0)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,414 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test modular OAI Triton MoE."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.utils import wait_for_gpu_memory_to_clear
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.utils.import_utils import get_triton_kernels_version, has_triton_kernels
|
||||
|
||||
if not has_triton_kernels():
|
||||
pytest.skip(
|
||||
"triton_kernels not found, skipping all related tests",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
if get_triton_kernels_version() == "3.8":
|
||||
# 3.8: matmul_ogs -> matmul
|
||||
from triton_kernels.matmul import FlexCtx, PrecisionConfig
|
||||
else:
|
||||
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
|
||||
from triton_kernels.numerics import InFlexData
|
||||
from triton_kernels.numerics_details.mxfp import downcast_to_mxfp, upcast_from_mxfp
|
||||
from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor
|
||||
from triton_kernels.testing import assert_close
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import (
|
||||
OAITritonExperts,
|
||||
UnfusedOAITritonExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
from .utils import make_dummy_moe_config, mxfp4_w_layouts, shuffle_weight
|
||||
|
||||
MNK = [
|
||||
(1, 512, 384),
|
||||
(1, 2880, 2880),
|
||||
(2, 512, 384),
|
||||
(2, 2880, 2880),
|
||||
(16, 2880, 2880),
|
||||
]
|
||||
|
||||
|
||||
def deepseek_v4_flash_moe_topology():
|
||||
"""MoE sizes for the DeepSeek-V4-Flash`.
|
||||
Default weights from: ``deepseek-ai/DeepSeek-V4-Flash`.
|
||||
``moe_intermediate_size``, ``n_routed_experts``, and ``num_experts_per_tok``.
|
||||
"""
|
||||
defaults = {
|
||||
"hidden_size": 4096,
|
||||
"moe_intermediate_size": 2048,
|
||||
"n_routed_experts": 256,
|
||||
"num_experts_per_tok": 6,
|
||||
}
|
||||
|
||||
return defaults
|
||||
|
||||
|
||||
def scaled_deepseek_v4_flash_problem(
|
||||
*,
|
||||
dim_scale: int = 8,
|
||||
expert_scale: int = 8,
|
||||
):
|
||||
"""Smaller K/N/E for kernel tests; keeps production top_k and K:N ratio (~2:1)."""
|
||||
t = deepseek_v4_flash_moe_topology()
|
||||
k = max(128, t["hidden_size"] // dim_scale)
|
||||
n = max(64, t["moe_intermediate_size"] // dim_scale)
|
||||
num_experts = max(t["num_experts_per_tok"], t["n_routed_experts"] // expert_scale)
|
||||
topk = t["num_experts_per_tok"]
|
||||
return k, n, num_experts, topk
|
||||
|
||||
|
||||
def unshuffle_weight(w: torch.Tensor):
|
||||
first = w[..., ::2]
|
||||
second = w[..., 1::2]
|
||||
return torch.concat((first, second), dim=-1)
|
||||
|
||||
|
||||
def make_weights(dtype, k, n, e):
|
||||
w1 = torch.randn((e, k, 2 * n), dtype=dtype, device="cuda")
|
||||
w1_bias = torch.randn((e, 2 * n), dtype=dtype, device="cuda")
|
||||
|
||||
w2 = torch.randn((e, n, k), dtype=dtype, device="cuda")
|
||||
w2_bias = torch.randn((e, k), dtype=dtype, device="cuda")
|
||||
|
||||
w1_tri = w1.clone()
|
||||
w2_tri = w2.clone()
|
||||
|
||||
w1_bias_tri = w1_bias.clone()
|
||||
w2_bias_tri = w2_bias.clone()
|
||||
w1_bias_tri = w1_bias_tri.to(torch.float32)
|
||||
w2_bias_tri = w2_bias_tri.to(torch.float32)
|
||||
|
||||
# shuffle weights
|
||||
w1_tri = shuffle_weight(w1_tri)
|
||||
w1_bias_tri = shuffle_weight(w1_bias_tri)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
k_align, n2_align = 256, 512
|
||||
else:
|
||||
k_align, n2_align = 64, 128
|
||||
|
||||
w1_bottom_pad = round_up(w1_tri.shape[1], k_align) - w1_tri.shape[1]
|
||||
w1_right_pad = round_up(w1_tri.shape[2], n2_align) - w1_tri.shape[2]
|
||||
w2_bottom_pad = w1_right_pad // 2
|
||||
w2_right_pad = w1_bottom_pad
|
||||
|
||||
w1_tri = F.pad(w1_tri, (0, w1_right_pad, 0, w1_bottom_pad, 0, 0))
|
||||
w2_tri = F.pad(w2_tri, (0, w2_right_pad, 0, w2_bottom_pad, 0, 0))
|
||||
w1_bias_tri = F.pad(w1_bias_tri, (0, w1_right_pad, 0, 0))
|
||||
w2_bias_tri = F.pad(w2_bias_tri, (0, w2_right_pad, 0, 0))
|
||||
|
||||
# quant triton_weights
|
||||
w1_tri, w1_scale_tri = downcast_to_mxfp(w1_tri, torch.uint8, axis=1)
|
||||
w1 = upcast_from_mxfp(w1_tri, w1_scale_tri, dtype, axis=1)
|
||||
w1 = w1[..., :k, : 2 * n]
|
||||
w1 = unshuffle_weight(w1)
|
||||
|
||||
w2_tri, w2_scale_tri = downcast_to_mxfp(w2_tri, torch.uint8, axis=1)
|
||||
w2 = upcast_from_mxfp(w2_tri, w2_scale_tri, dtype, axis=1)
|
||||
w2 = w2[..., :n, :k]
|
||||
|
||||
num_warps = 8
|
||||
(
|
||||
w_layout,
|
||||
w_layout_opts,
|
||||
w_scale_layout,
|
||||
w_scale_layout_opts,
|
||||
) = mxfp4_w_layouts(mx_axis=1, num_warps=num_warps)
|
||||
|
||||
w1_tri = convert_layout(wrap_torch_tensor(w1_tri, FP4), w_layout, **w_layout_opts)
|
||||
w1_scale_tri = convert_layout(
|
||||
wrap_torch_tensor(w1_scale_tri),
|
||||
w_scale_layout,
|
||||
**w_scale_layout_opts,
|
||||
)
|
||||
|
||||
w2_tri = convert_layout(wrap_torch_tensor(w2_tri, FP4), w_layout, **w_layout_opts)
|
||||
w2_scale_tri = convert_layout(
|
||||
wrap_torch_tensor(w2_scale_tri),
|
||||
w_scale_layout,
|
||||
**w_scale_layout_opts,
|
||||
)
|
||||
|
||||
w1_precision_config = PrecisionConfig(
|
||||
**mx_scale_kwargs(w1_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData())
|
||||
)
|
||||
w2_precision_config = PrecisionConfig(
|
||||
**mx_scale_kwargs(w2_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData())
|
||||
)
|
||||
|
||||
return (
|
||||
w1,
|
||||
w2,
|
||||
w1_bias,
|
||||
w2_bias,
|
||||
w1_tri,
|
||||
w2_tri,
|
||||
w1_bias_tri,
|
||||
w2_bias_tri,
|
||||
w1_precision_config,
|
||||
w2_precision_config,
|
||||
w1_bottom_pad,
|
||||
)
|
||||
|
||||
|
||||
def swiglu(x, alpha: float = 1.702, limit: float = 1.0):
|
||||
# Note we add an extra bias of 1 to the linear layer
|
||||
x_glu, x_linear = torch.chunk(x, 2, dim=-1)
|
||||
if limit is not None:
|
||||
x_glu = x_glu.clamp(max=limit)
|
||||
out_glu = x_glu * torch.sigmoid(alpha * x_glu)
|
||||
if limit is not None:
|
||||
x_linear = x_linear.clamp(min=-limit, max=limit)
|
||||
return out_glu * (x_linear + 1)
|
||||
|
||||
|
||||
def torch_moe_impl(
|
||||
hidden_states: torch.Tensor, # (M, K)
|
||||
w1: torch.Tensor, # (E, K, 2N)
|
||||
w2: torch.Tensor, # (E, N, K)
|
||||
w1_bias: torch.Tensor, # (E, 2N)
|
||||
w2_bias: torch.Tensor, # (E, K)
|
||||
topk_weights: torch.Tensor, # (M, topk)
|
||||
topk_ids: torch.Tensor, # (M, topk)
|
||||
):
|
||||
w1 = w1[topk_ids, ...]
|
||||
w1_bias = w1_bias[topk_ids, ...]
|
||||
hidden_states = torch.einsum("bekc,bk->bec", w1, hidden_states) + w1_bias
|
||||
hidden_states = swiglu(hidden_states, limit=7)
|
||||
|
||||
w2 = w2[topk_ids, ...]
|
||||
w2_bias = w2_bias[topk_ids, ...]
|
||||
hidden_states = torch.einsum("bekc,bek->bec", w2, hidden_states) + w2_bias
|
||||
|
||||
# Weighted sum of experts
|
||||
hidden_states = torch.einsum("bec,be->bc", hidden_states, topk_weights)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def oai_triton_moe_impl(
|
||||
x: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_scale: PrecisionConfig,
|
||||
w2_scale: PrecisionConfig,
|
||||
w1_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
num_experts: int,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
unfused: bool = False,
|
||||
) -> torch.Tensor:
|
||||
quant_config = mxfp4_w4a16_moe_quant_config(
|
||||
w1_bias=w1_bias,
|
||||
w2_bias=w2_bias,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
if unfused:
|
||||
fused_experts = UnfusedOAITritonExperts(moe_config, quant_config)
|
||||
else:
|
||||
fused_experts = OAITritonExperts(moe_config, quant_config)
|
||||
|
||||
mk = FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
fused_experts,
|
||||
)
|
||||
|
||||
return mk.apply(
|
||||
hidden_states=x,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SWIGLUOAI,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not OAITritonExperts._supports_current_device(),
|
||||
reason="OAI Triton MoE is not supported on this device.",
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("m,n,k", MNK)
|
||||
@pytest.mark.parametrize("num_experts", [32, 128])
|
||||
@pytest.mark.parametrize("topk", [4])
|
||||
@pytest.mark.parametrize("unfused", [True, False])
|
||||
def test_oai_triton_moe(
|
||||
dtype: torch.dtype,
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
unfused: bool,
|
||||
workspace_init,
|
||||
):
|
||||
wait_for_gpu_memory_to_clear(devices=[0], threshold_ratio=0.1)
|
||||
set_random_seed(0)
|
||||
|
||||
(
|
||||
w1,
|
||||
w2,
|
||||
w1_bias,
|
||||
w2_bias,
|
||||
w1_tri,
|
||||
w2_tri,
|
||||
w1_bias_tri,
|
||||
w2_bias_tri,
|
||||
w1_precision_config,
|
||||
w2_precision_config,
|
||||
x_pad,
|
||||
) = make_weights(dtype, k, n, num_experts)
|
||||
|
||||
x = torch.randn((m, k), dtype=dtype, device="cuda")
|
||||
x_tri = F.pad(x, (0, x_pad, 0, 0))
|
||||
router_logits = torch.randn(m, num_experts, device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1, sorted=True)
|
||||
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
out_ref = torch_moe_impl(x, w1, w2, w1_bias, w2_bias, topk_weights, topk_ids)
|
||||
|
||||
out = oai_triton_moe_impl(
|
||||
x_tri,
|
||||
w1_tri,
|
||||
w2_tri,
|
||||
w1_precision_config,
|
||||
w2_precision_config,
|
||||
w1_bias_tri,
|
||||
w2_bias_tri,
|
||||
num_experts,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
unfused,
|
||||
)
|
||||
out = out[..., :k]
|
||||
|
||||
assert_close(ref=out_ref, tri=out, maxtol=0.025, rmstol=0.005)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not UnfusedOAITritonExperts._supports_current_device(),
|
||||
reason="Unfused OAI Triton MoE is not supported on this device.",
|
||||
)
|
||||
def test_unfused_oai_triton_experts_apply_direct_deepseek_v4_topology(workspace_init):
|
||||
"""Exercise ``UnfusedOAITritonExperts.apply`` with explicit workspaces.
|
||||
|
||||
Same MoE topology as ``launch_dsv4.sh`` / DeepSeek-V4-Flash ``config.json``,
|
||||
with linear dimensions and expert count scaled down for test GPU memory.
|
||||
"""
|
||||
wait_for_gpu_memory_to_clear(devices=[0], threshold_ratio=0.1)
|
||||
set_random_seed(0)
|
||||
|
||||
k, n, num_experts, topk = scaled_deepseek_v4_flash_problem()
|
||||
m = 7
|
||||
dtype = torch.bfloat16
|
||||
|
||||
(
|
||||
w1,
|
||||
w2,
|
||||
w1_bias,
|
||||
w2_bias,
|
||||
w1_tri,
|
||||
w2_tri,
|
||||
w1_bias_tri,
|
||||
w2_bias_tri,
|
||||
w1_precision_config,
|
||||
w2_precision_config,
|
||||
x_pad,
|
||||
) = make_weights(dtype, k, n, num_experts)
|
||||
|
||||
x = torch.randn((m, k), dtype=dtype, device="cuda")
|
||||
x_tri = F.pad(x, (0, x_pad, 0, 0))
|
||||
router_logits = torch.randn(m, num_experts, device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1, sorted=True)
|
||||
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
|
||||
|
||||
quant_config = mxfp4_w4a16_moe_quant_config(
|
||||
w1_bias=w1_bias_tri,
|
||||
w2_bias=w2_bias_tri,
|
||||
w1_scale=w1_precision_config,
|
||||
w2_scale=w2_precision_config,
|
||||
)
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
)
|
||||
experts = UnfusedOAITritonExperts(moe_config, quant_config)
|
||||
|
||||
_, _, N, K, top_k = experts.moe_problem_size(x_tri, w1_tri, w2_tri, topk_ids)
|
||||
assert top_k == topk
|
||||
ws13_shape, ws2_shape, out_shape = experts.workspace_shapes(
|
||||
m,
|
||||
N,
|
||||
K,
|
||||
topk,
|
||||
num_experts,
|
||||
num_experts,
|
||||
None,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
)
|
||||
workspace13 = torch.empty(ws13_shape, dtype=dtype, device="cuda")
|
||||
workspace2 = torch.empty(ws2_shape, dtype=dtype, device="cuda")
|
||||
output = torch.empty(out_shape, dtype=dtype, device="cuda")
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
out_ref = torch_moe_impl(x, w1, w2, w1_bias, w2_bias, topk_weights, topk_ids)
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=x_tri,
|
||||
w1=w1_tri,
|
||||
w2=w2_tri,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SWIGLUOAI,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=workspace13,
|
||||
workspace2=workspace2,
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
output = output[..., :k]
|
||||
|
||||
assert_close(ref=out_ref, tri=output, maxtol=0.025, rmstol=0.005)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,512 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MOE permute/unpermute kernel.
|
||||
|
||||
Run `pytest tests/kernels/test_moe_permute_unpermute.py`.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.expert_map_manager import (
|
||||
determine_expert_map,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
MoEPermuteScratch,
|
||||
get_moe_permute_scratch,
|
||||
moe_permute,
|
||||
moe_permute_unpermute_supported,
|
||||
moe_prepare_scatter,
|
||||
moe_unpermute,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
NUM_EXPERTS = [16, 64, 256]
|
||||
TOP_KS = [2, 6, 8]
|
||||
EP_SIZE = [1, 4, 16]
|
||||
set_random_seed(0)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"moe_permute_unpermute_supported is not defined for ROCm",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def torch_permute(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
# token_expert_indices: torch.Tensor,
|
||||
topk: int,
|
||||
n_expert: int,
|
||||
n_local_expert: int,
|
||||
start_expert: int,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
n_token = hidden_states.shape[0]
|
||||
if expert_map is not None:
|
||||
is_local_expert = expert_map[topk_ids] != -1
|
||||
not_local_expert = expert_map[topk_ids] == -1
|
||||
topk_ids = is_local_expert * (topk_ids - start_expert) + not_local_expert * (
|
||||
topk_ids + n_expert
|
||||
)
|
||||
token_expert_indices = torch.arange(
|
||||
0, n_token * topk, dtype=torch.int32, device=hidden_states.device
|
||||
).reshape((n_token, topk))
|
||||
|
||||
sorted_topk_ids, sorted_indices = torch.sort(topk_ids.flatten(), stable=True)
|
||||
dst_row_id2src_row_id_map = token_expert_indices.flatten()[sorted_indices]
|
||||
|
||||
expert_first_token_offset = torch.zeros(
|
||||
n_local_expert + 1, dtype=torch.int64, device="cuda"
|
||||
)
|
||||
idx = 0
|
||||
for i in range(0, n_local_expert):
|
||||
cnt = 0
|
||||
while idx < sorted_topk_ids.numel() and sorted_topk_ids[idx] == i:
|
||||
cnt += 1
|
||||
idx += 1
|
||||
expert_first_token_offset[i + 1] = expert_first_token_offset[i] + cnt
|
||||
|
||||
_, src2dst_idx = torch.sort(dst_row_id2src_row_id_map)
|
||||
valid_row_idx = []
|
||||
permuted_hidden_states = hidden_states[dst_row_id2src_row_id_map // topk, ...]
|
||||
src_row_id2dst_row_id_map = torch.arange(
|
||||
0, n_token * topk, device="cuda", dtype=torch.int32
|
||||
)[src2dst_idx].reshape((n_token, topk))
|
||||
valid_row_idx += [i for i in range(expert_first_token_offset[-1])]
|
||||
dst_row_id2src_row_id_map[expert_first_token_offset[-1] :] = n_token * topk
|
||||
return [
|
||||
permuted_hidden_states,
|
||||
expert_first_token_offset,
|
||||
src_row_id2dst_row_id_map,
|
||||
dst_row_id2src_row_id_map,
|
||||
valid_row_idx,
|
||||
]
|
||||
|
||||
|
||||
def torch_unpermute(
|
||||
permuted_hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
token_expert_indices: torch.Tensor,
|
||||
src_row_id2dst_row_id_map: torch.Tensor,
|
||||
valid_row_idx: torch.Tensor,
|
||||
topk: int,
|
||||
n_expert: int,
|
||||
) -> torch.Tensor:
|
||||
# ignore invalid row
|
||||
n_hidden = permuted_hidden_states.shape[1]
|
||||
mask = torch.zeros(permuted_hidden_states.shape[0], dtype=bool, device="cuda")
|
||||
mask[valid_row_idx] = True
|
||||
permuted_hidden_states[~mask] = 0
|
||||
|
||||
permuted_hidden_states = permuted_hidden_states[
|
||||
src_row_id2dst_row_id_map.flatten(), ...
|
||||
]
|
||||
permuted_hidden_states = permuted_hidden_states.view(-1, topk, n_hidden)
|
||||
output = (
|
||||
(permuted_hidden_states * topk_weights.unsqueeze(2))
|
||||
.sum(1)
|
||||
.to(permuted_hidden_states.dtype)
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_token", [1, 33, 1024, 5000])
|
||||
@pytest.mark.parametrize("n_hidden", [2048, 7168])
|
||||
@pytest.mark.parametrize("n_expert", NUM_EXPERTS)
|
||||
@pytest.mark.parametrize("topk", TOP_KS)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("ep_size", EP_SIZE)
|
||||
@pytest.mark.parametrize("use_scratch", [False, True])
|
||||
def test_moe_permute_unpermute(
|
||||
n_token: int,
|
||||
n_hidden: int,
|
||||
topk: int,
|
||||
n_expert: int,
|
||||
ep_size: int,
|
||||
dtype: torch.dtype,
|
||||
use_scratch: bool,
|
||||
workspace_init,
|
||||
):
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
ep_rank = np.random.randint(0, ep_size)
|
||||
expert_map = None
|
||||
n_local_expert = n_expert
|
||||
if ep_size != 1:
|
||||
n_local_expert, expert_map, _ = determine_expert_map(ep_size, ep_rank, n_expert)
|
||||
expert_map = expert_map.cuda()
|
||||
start_expert = n_local_expert * ep_rank
|
||||
set_random_seed(0)
|
||||
hidden_states = torch.randn((n_token, n_hidden), device="cuda").to(dtype)
|
||||
gating_output = torch.randn((n_token, n_expert), device="cuda").to(dtype)
|
||||
topk_weights, topk_ids, token_expert_indices = fused_topk(
|
||||
hidden_states, gating_output, topk, False
|
||||
)
|
||||
(
|
||||
gold_permuted_hidden_states,
|
||||
gold_expert_first_token_offset,
|
||||
gold_inv_permuted_idx,
|
||||
gold_permuted_idx,
|
||||
valid_row_idx,
|
||||
) = torch_permute(
|
||||
hidden_states,
|
||||
topk_ids,
|
||||
# token_expert_indices,
|
||||
topk,
|
||||
n_expert,
|
||||
n_local_expert,
|
||||
start_expert,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
|
||||
scratch = None
|
||||
if use_scratch:
|
||||
scratch = get_moe_permute_scratch(
|
||||
num_experts=n_expert,
|
||||
num_local_experts=n_local_expert,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
(
|
||||
permuted_hidden_states,
|
||||
_,
|
||||
expert_first_token_offset,
|
||||
inv_permuted_idx,
|
||||
_,
|
||||
) = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
n_expert=n_expert,
|
||||
n_local_expert=n_local_expert,
|
||||
expert_map=expert_map,
|
||||
scratch=scratch,
|
||||
)
|
||||
|
||||
# check expert_first_token_offset
|
||||
torch.testing.assert_close(
|
||||
gold_expert_first_token_offset, expert_first_token_offset, atol=0, rtol=0
|
||||
)
|
||||
# check src_row_id2dst_row_id_map
|
||||
torch.testing.assert_close(
|
||||
gold_inv_permuted_idx.flatten(), inv_permuted_idx, atol=0, rtol=0
|
||||
)
|
||||
|
||||
# check permuted_hidden_states, only valid token
|
||||
torch.testing.assert_close(
|
||||
gold_permuted_hidden_states[valid_row_idx],
|
||||
permuted_hidden_states[valid_row_idx],
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
# add a random tensor to simulate group gemm
|
||||
result0 = 0.5 * permuted_hidden_states + torch.randn_like(permuted_hidden_states)
|
||||
result4 = torch.empty_like(hidden_states)
|
||||
moe_unpermute(
|
||||
result4, result0, topk_weights, inv_permuted_idx, expert_first_token_offset
|
||||
)
|
||||
|
||||
gold4 = torch_unpermute(
|
||||
result0,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
inv_permuted_idx,
|
||||
valid_row_idx,
|
||||
topk,
|
||||
n_local_expert,
|
||||
)
|
||||
# check unpermuted hidden
|
||||
torch.testing.assert_close(result4, gold4, atol=2e-2, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("n_token", [1, 33, 128])
|
||||
@pytest.mark.parametrize("topk", [1, 6, 8])
|
||||
def test_moe_permute_reuses_scratch_buffers(
|
||||
dtype: torch.dtype, n_token: int, topk: int, workspace_init
|
||||
):
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
n_hidden = 2048
|
||||
n_expert = 16
|
||||
hidden_states = torch.randn((n_token, n_hidden), device="cuda").to(dtype)
|
||||
gating_output = torch.randn((n_token, n_expert), device="cuda").to(dtype)
|
||||
_, topk_ids, _ = fused_topk(hidden_states, gating_output, topk, False)
|
||||
|
||||
scratch_config = dict(
|
||||
num_experts=n_expert,
|
||||
num_local_experts=n_expert,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
scratch = get_moe_permute_scratch(**scratch_config)
|
||||
|
||||
first = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
n_expert=n_expert,
|
||||
scratch=scratch,
|
||||
)
|
||||
current_workspace_manager().lock()
|
||||
assert get_moe_permute_scratch(**scratch_config) is scratch
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
second = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
n_expert=n_expert,
|
||||
scratch=get_moe_permute_scratch(**scratch_config),
|
||||
)
|
||||
|
||||
for _ in range(2):
|
||||
hidden_states.add_(1)
|
||||
topk_ids.copy_(topk_ids.roll(1, 0))
|
||||
graph.replay()
|
||||
expected = moe_permute(hidden_states, None, topk_ids, n_expert)
|
||||
for actual, reference in zip(second, expected):
|
||||
if actual is not None:
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
(
|
||||
permuted_hidden_states_1,
|
||||
_,
|
||||
expert_first_token_offset_1,
|
||||
inv_permuted_idx_1,
|
||||
permuted_idx_1,
|
||||
) = first
|
||||
(
|
||||
permuted_hidden_states_2,
|
||||
_,
|
||||
expert_first_token_offset_2,
|
||||
inv_permuted_idx_2,
|
||||
permuted_idx_2,
|
||||
) = second
|
||||
|
||||
torch.testing.assert_close(expert_first_token_offset_1, expert_first_token_offset_2)
|
||||
torch.testing.assert_close(inv_permuted_idx_1, inv_permuted_idx_2)
|
||||
torch.testing.assert_close(permuted_idx_1, permuted_idx_2)
|
||||
|
||||
assert (
|
||||
expert_first_token_offset_1.untyped_storage().data_ptr()
|
||||
== expert_first_token_offset_2.untyped_storage().data_ptr()
|
||||
)
|
||||
assert (
|
||||
inv_permuted_idx_1.untyped_storage().data_ptr()
|
||||
== scratch.inv_permuted_idx.untyped_storage().data_ptr()
|
||||
)
|
||||
assert (
|
||||
permuted_idx_1.untyped_storage().data_ptr()
|
||||
== scratch.permuted_idx.untyped_storage().data_ptr()
|
||||
)
|
||||
|
||||
|
||||
def test_moe_permute_scratch_reused_across_graph_sizes(workspace_init) -> None:
|
||||
"""Switching graph sizes must not expose metadata left by a larger batch."""
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
scratch = get_moe_permute_scratch(
|
||||
num_experts=16,
|
||||
num_local_experts=16,
|
||||
device=device,
|
||||
)
|
||||
scratch.reserve(128 * 6)
|
||||
current_workspace_manager().lock()
|
||||
runs = []
|
||||
for n_token in (128, 1, 33):
|
||||
hidden = torch.randn((n_token, 2048), dtype=torch.bfloat16, device=device)
|
||||
topk_ids = torch.randint(16, (n_token, 6), device=device)
|
||||
moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
|
||||
runs.append((graph, hidden, topk_ids, output))
|
||||
|
||||
for graph, hidden, topk_ids, output in runs + runs[::-1]:
|
||||
hidden.add_(1)
|
||||
topk_ids.copy_((topk_ids + 1) % 16)
|
||||
graph.replay()
|
||||
expected = moe_permute(hidden, None, topk_ids, 16)
|
||||
for actual, reference in zip(output, expected):
|
||||
if actual is not None:
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
|
||||
def test_moe_permute_scratch_isolated_across_execution_slots(monkeypatch) -> None:
|
||||
"""Concurrent graph replays in different ubatches/lanes cannot share scratch."""
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
from vllm.v1.worker import workspace
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
manager = workspace.WorkspaceManager(device, num_ubatches=2, num_lanes=2)
|
||||
monkeypatch.setattr(workspace, "_manager", manager)
|
||||
ubatch = 0
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: ubatch)
|
||||
scratch_config = dict(
|
||||
num_experts=16,
|
||||
num_local_experts=16,
|
||||
device=device,
|
||||
)
|
||||
runs = []
|
||||
for i, n_token in enumerate((1, 7, 33, 64)):
|
||||
ubatch = i // 2
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with workspace.use_workspace_lane(i % 2), torch.cuda.stream(stream):
|
||||
hidden = torch.randn((n_token, 2048), dtype=torch.bfloat16, device=device)
|
||||
topk_ids = torch.randint(16, (n_token, 6), device=device)
|
||||
scratch = get_moe_permute_scratch(**scratch_config)
|
||||
moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
output = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
|
||||
runs.append((stream, graph, hidden, topk_ids, output, scratch))
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
|
||||
assert len({run[-1].permuted_idx.data_ptr() for run in runs}) == 4
|
||||
manager.lock()
|
||||
for _ in range(3):
|
||||
for i, (stream, graph, hidden, topk_ids, _, scratch) in enumerate(runs):
|
||||
ubatch = i // 2
|
||||
with workspace.use_workspace_lane(i % 2), torch.cuda.stream(stream):
|
||||
assert get_moe_permute_scratch(**scratch_config) is scratch
|
||||
hidden.add_(1)
|
||||
topk_ids.copy_((topk_ids + 1) % 16)
|
||||
graph.replay()
|
||||
for stream, _, hidden, topk_ids, output, _ in runs:
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
expected = moe_permute(hidden, None, topk_ids, 16)
|
||||
for actual, reference in zip(output, expected):
|
||||
if actual is not None:
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
|
||||
def test_moe_permute_scratch_without_manager(monkeypatch) -> None:
|
||||
"""Standalone calls get independent scratch with initialized row indices."""
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
from vllm.v1.worker import workspace
|
||||
|
||||
monkeypatch.setattr(workspace, "_manager", None)
|
||||
config = dict(
|
||||
num_experts=4,
|
||||
num_local_experts=4,
|
||||
device=torch.device("cuda"),
|
||||
)
|
||||
first = get_moe_permute_scratch(**config)
|
||||
second = get_moe_permute_scratch(**config)
|
||||
|
||||
first.reserve(8)
|
||||
second.reserve(8)
|
||||
assert (
|
||||
first.token_expert_indices.data_ptr() != second.token_expert_indices.data_ptr()
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
first.token_expert_indices, torch.arange(8, dtype=torch.int32, device="cuda")
|
||||
)
|
||||
|
||||
|
||||
def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
hidden_states = torch.arange(5 * 16, dtype=torch.bfloat16, device="cuda").view(
|
||||
5, 16
|
||||
)
|
||||
topk_ids = torch.tensor([[0], [-1], [1], [4], [2]], device="cuda")
|
||||
expert_map = torch.tensor([0, 1, -1, -1], dtype=torch.int32, device="cuda")
|
||||
scratch = MoEPermuteScratch(
|
||||
num_experts=4,
|
||||
num_local_experts=2,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
permuted, _, expert_offsets, inverse, _ = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
n_expert=4,
|
||||
n_local_expert=2,
|
||||
expert_map=expert_map,
|
||||
scratch=scratch,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
expert_offsets,
|
||||
torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"),
|
||||
)
|
||||
torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]])
|
||||
assert torch.all(inverse[[1, 3, 4]] >= expert_offsets[-1])
|
||||
expected = torch.zeros_like(hidden_states)
|
||||
expected[[0, 2]] = hidden_states[[0, 2]]
|
||||
output = torch.empty_like(hidden_states)
|
||||
moe_unpermute(
|
||||
output, permuted, torch.ones(5, 1, device="cuda"), inverse, expert_offsets
|
||||
)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
expected_inverse = inverse.clone()
|
||||
expected_offsets = expert_offsets.clone()
|
||||
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
|
||||
torch.testing.assert_close(indices.flatten(), expected_inverse)
|
||||
torch.testing.assert_close(expert_offsets, expected_offsets)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
|
||||
topk_ids.zero_()
|
||||
graph.replay()
|
||||
|
||||
expected_inverse = torch.arange(5, dtype=torch.int32, device="cuda")
|
||||
expected_offsets = torch.tensor([0, 5, 5], dtype=torch.int64, device="cuda")
|
||||
torch.testing.assert_close(indices.flatten(), expected_inverse)
|
||||
torch.testing.assert_close(expert_offsets, expected_offsets)
|
||||
|
||||
|
||||
def test_moe_permute_scratch_grows_until_workspace_lock(workspace_init) -> None:
|
||||
"""Capacity follows the inputs seen in warmup, whatever their topk or size."""
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
scratch = get_moe_permute_scratch(
|
||||
num_experts=16, num_local_experts=16, device=device
|
||||
)
|
||||
|
||||
def run(n_token: int, topk: int, dtype: torch.dtype) -> None:
|
||||
hidden = torch.randn((n_token, 64), device=device).to(dtype)
|
||||
topk_ids = torch.randint(16, (n_token, topk), device=device)
|
||||
actual = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
|
||||
expected = moe_permute(hidden, None, topk_ids, 16)
|
||||
for a, e in zip(actual, expected):
|
||||
if a is not None:
|
||||
torch.testing.assert_close(a.float(), e.float())
|
||||
|
||||
run(8, 6, torch.bfloat16)
|
||||
run(64, 6, torch.bfloat16)
|
||||
run(384, 1, torch.float8_e4m3fn)
|
||||
assert scratch.max_expanded_rows == 384
|
||||
|
||||
current_workspace_manager().lock()
|
||||
run(64, 6, torch.bfloat16)
|
||||
with pytest.raises(AssertionError, match="Scratch growth is not allowed"):
|
||||
run(65, 6, torch.bfloat16)
|
||||
|
||||
# Experts rebuilt after the lock (e.g. a weight reload) keep the capacity.
|
||||
rebuilt = get_moe_permute_scratch(
|
||||
num_experts=16, num_local_experts=16, device=device
|
||||
)
|
||||
assert rebuilt.max_expanded_rows == 384
|
||||
@@ -1,247 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Two-rank MoRI/AITER numerical counterpart to the DeepEP v2 MoE matrix."""
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.modular_kernel_tools.parallel_utils import (
|
||||
parallel_launch_with_config,
|
||||
)
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
|
||||
from tests.kernels.utils import torch_experts
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import ParallelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_mori
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
MNKS = [
|
||||
(1, 256, 256),
|
||||
(2, 256, 512),
|
||||
(3, 1024, 2048),
|
||||
(32, 256, 1024),
|
||||
(45, 512, 2048),
|
||||
(64, 1024, 1024),
|
||||
(222, 1024, 2048),
|
||||
]
|
||||
|
||||
|
||||
def _mxfp4_aiter_weights(w1, w2, start, end):
|
||||
"""MXFP4-quantize experts [start, end) into AITER's W4A4 kernel layout."""
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_quant_config,
|
||||
)
|
||||
|
||||
def quantize(w):
|
||||
w = w[start:end]
|
||||
q, s = dynamic_mxfp4_quant(w.flatten(0, 1))
|
||||
return q.view(*w.shape[:2], -1), s.view(*w.shape[:2], -1)
|
||||
|
||||
backend = Mxfp4MoeBackend.AITER_MXFP4_MXFP4
|
||||
(q1, s1), (q2, s2) = quantize(w1), quantize(w2)
|
||||
q1, q2, s1, s2, _, _ = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
backend, None, q1, q2, s1, s2
|
||||
)
|
||||
return q1, q2, make_mxfp4_moe_quant_config(backend, s1, s2)
|
||||
|
||||
|
||||
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterFp8ChannelExperts,
|
||||
AiterMxfp4Experts,
|
||||
AiterUnquantizedExperts,
|
||||
rocm_aiter_fused_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
|
||||
MoriPrepareAndFinalize,
|
||||
)
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
init_workspace_manager(pgi.device)
|
||||
set_random_seed(47)
|
||||
fp8 = scheme == "fp8_token_channel"
|
||||
mxfp4 = scheme == "mxfp4"
|
||||
dtype = current_platform.fp8_dtype() if fp8 else None
|
||||
(_, w1, s1, _), (_, w2, s2, _) = make_test_weights(
|
||||
32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8
|
||||
)
|
||||
# Share global expert weights, but give each rank different token inputs.
|
||||
set_random_seed(101 + pgi.rank)
|
||||
x = torch.randn(m, k, device=pgi.device, dtype=torch.bfloat16) / 10
|
||||
routing = torch.randn(m, 32, device=pgi.device)
|
||||
# Guarantee a route to both ranks, including the one-token case.
|
||||
routing[:, 0] = routing[:, 16] = float("inf")
|
||||
ids = routing.topk(6, dim=-1).indices.to(torch.int32)
|
||||
weights = torch.randn(m, 6, device=pgi.device, dtype=torch.float32)
|
||||
parallel = replace(
|
||||
FusedMoEParallelConfig.make_no_parallel(),
|
||||
dp_size=2,
|
||||
dp_rank=pgi.rank,
|
||||
ep_size=2,
|
||||
ep_rank=pgi.rank,
|
||||
use_ep=True,
|
||||
all2all_backend="mori_high_throughput",
|
||||
)
|
||||
config = replace(
|
||||
make_dummy_moe_config(
|
||||
num_experts=32,
|
||||
num_local_experts=16,
|
||||
experts_per_token=6,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
max_num_tokens=256,
|
||||
),
|
||||
moe_parallel_config=parallel,
|
||||
)
|
||||
start, end = pgi.rank * 16, (pgi.rank + 1) * 16
|
||||
if mxfp4:
|
||||
local_w1, local_w2, quant = _mxfp4_aiter_weights(w1, w2, start, end)
|
||||
ref_w1, ref_w2, ref_quant = _mxfp4_aiter_weights(w1, w2, 0, 32)
|
||||
else:
|
||||
quant = FusedMoEQuantConfig.make(
|
||||
dtype,
|
||||
per_act_token_quant=fp8,
|
||||
per_out_ch_quant=fp8,
|
||||
w1_scale=None if s1 is None else s1[start:end].contiguous(),
|
||||
w2_scale=None if s2 is None else s2[start:end].contiguous(),
|
||||
)
|
||||
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
|
||||
w1[start:end].contiguous(), w2[start:end].contiguous()
|
||||
)
|
||||
local_w1.is_shuffled = local_w2.is_shuffled = True
|
||||
prepare = maybe_make_prepare_finalize(config, quant)
|
||||
assert isinstance(prepare, MoriPrepareAndFinalize)
|
||||
assert prepare.use_fp8_dispatch == fp8
|
||||
assert prepare.use_fp4_dispatch == mxfp4
|
||||
assert prepare.num_dispatchers() == 2
|
||||
experts_cls = {
|
||||
"bf16": AiterUnquantizedExperts,
|
||||
"fp8_token_channel": AiterFp8ChannelExperts,
|
||||
"mxfp4": AiterMxfp4Experts,
|
||||
}[scheme]
|
||||
experts = experts_cls(config, quant)
|
||||
assert not experts.expects_unquantized_inputs
|
||||
kernel = FusedMoEKernel(prepare, experts)
|
||||
expert_mask = torch.zeros(32, dtype=torch.int32, device=pgi.device)
|
||||
expert_mask[start:end] = 1
|
||||
|
||||
def reference(route_weights):
|
||||
if mxfp4:
|
||||
# AITER's non-EP MXFP4 path quantizes the BF16 input the same way
|
||||
# as FP4 dispatch, so the BF16 tolerances below still apply.
|
||||
return rocm_aiter_fused_experts(
|
||||
x, ref_w1, ref_w2, route_weights, ids, config, quant_config=ref_quant
|
||||
)
|
||||
return torch_experts(
|
||||
x,
|
||||
w1,
|
||||
w2,
|
||||
route_weights,
|
||||
ids,
|
||||
w1_scale=s1,
|
||||
w2_scale=s2,
|
||||
quant_dtype=dtype,
|
||||
per_act_token_quant=fp8,
|
||||
)
|
||||
|
||||
def run():
|
||||
with set_forward_context(None, vllm_config):
|
||||
return kernel.apply(
|
||||
x,
|
||||
local_w1,
|
||||
local_w2,
|
||||
weights,
|
||||
ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=32,
|
||||
expert_map=expert_mask,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
def check(actual):
|
||||
expected = reference(weights)
|
||||
assert actual.shape == expected.shape
|
||||
assert actual.dtype == expected.dtype
|
||||
local_weights = weights * ((ids >= start) & (ids < end))
|
||||
missing_remote = reference(local_weights)
|
||||
norm = expected.float().norm()
|
||||
assert norm > 0
|
||||
norm_budget = 0.2 if fp8 else 0.06
|
||||
assert (actual.float() - expected.float()).norm() / norm < norm_budget
|
||||
# The norm check also rejects low-signal zero or rank-local results.
|
||||
for incorrect in (torch.zeros_like(expected), missing_remote):
|
||||
assert (incorrect.float() - expected.float()).norm() / norm > norm_budget
|
||||
if fp8:
|
||||
assert (
|
||||
torch.isclose(actual, expected, atol=0.2, rtol=0.2)
|
||||
.float()
|
||||
.mean()
|
||||
.item()
|
||||
> 0.99
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(actual, expected, atol=0.06, rtol=0.06)
|
||||
assert torch.isfinite(actual).all()
|
||||
|
||||
actual = run()
|
||||
check(actual)
|
||||
if graph:
|
||||
saved = actual.clone()
|
||||
run()
|
||||
capture = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(capture):
|
||||
captured = run()
|
||||
for factor in (-0.5, 1.5):
|
||||
x.mul_(factor)
|
||||
weights.mul_(-0.75)
|
||||
capture.replay()
|
||||
torch.accelerator.synchronize()
|
||||
check(captured)
|
||||
torch.testing.assert_close(actual, saved, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI"
|
||||
)
|
||||
@pytest.mark.parametrize("m,n,k", MNKS)
|
||||
@pytest.mark.parametrize("scheme", ["bf16", "fp8_token_channel", "mxfp4"])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_mori_moe(m, n, k, scheme):
|
||||
if scheme == "mxfp4":
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not on_gfx950():
|
||||
pytest.skip("AITER MXFP4 W4A4 MoE requires gfx950")
|
||||
config = VllmConfig(
|
||||
parallel_config=ParallelConfig(
|
||||
data_parallel_size=2,
|
||||
enable_expert_parallel=True,
|
||||
all2all_backend="mori_high_throughput",
|
||||
),
|
||||
scheduler_config=SchedulerConfig.default_factory(max_num_batched_tokens=256),
|
||||
)
|
||||
env = {
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
"VLLM_ROCM_USE_AITER_MOE": "1",
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
|
||||
}
|
||||
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, scheme, m == 32)
|
||||
@@ -1,462 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for SM100 CUTLASS MXFP4 x MXFP4 grouped MoE kernels."""
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import torch_moe_single
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
random.seed(42)
|
||||
set_random_seed(42)
|
||||
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
def align(val: int, alignment: int = 128) -> int:
|
||||
return int((val + alignment - 1) // alignment * alignment)
|
||||
|
||||
|
||||
def calc_diff(x, y):
|
||||
x, y = x.double(), y.double()
|
||||
denominator = (x * x + y * y).sum()
|
||||
sim = 2 * (x * y).sum() / denominator
|
||||
return 1 - sim
|
||||
|
||||
|
||||
def is_sm100_supported() -> bool:
|
||||
return current_platform.is_cuda() and current_platform.is_device_capability_family(
|
||||
100
|
||||
)
|
||||
|
||||
|
||||
def compute_ref_output(
|
||||
input_tensor: torch.Tensor,
|
||||
weight_list: list[torch.Tensor],
|
||||
expert_offsets: list[int],
|
||||
expert_offset: int,
|
||||
num_experts: int,
|
||||
) -> torch.Tensor:
|
||||
"""Reference output using torch_moe_single with top-1 routing."""
|
||||
score = torch.full(
|
||||
(expert_offset, num_experts),
|
||||
-1e9,
|
||||
device=input_tensor.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
for g in range(num_experts):
|
||||
start = expert_offsets[g]
|
||||
end = expert_offsets[g + 1] if g + 1 < num_experts else expert_offset
|
||||
score[start:end, g] = 0.0
|
||||
|
||||
return torch_moe_single(
|
||||
input_tensor, torch.stack(weight_list, dim=0), score, topk=1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_sm100_supported(),
|
||||
reason="cutlass_mxfp4_group_mm requires CUDA SM100",
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [8, 16, 32])
|
||||
@pytest.mark.parametrize("out_dtype", [torch.bfloat16])
|
||||
def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype):
|
||||
"""Test the MXFP4 grouped GEMM kernel by:
|
||||
1. Creating random per-expert inputs and weights
|
||||
2. Quantizing both to MXFP4 using the CUDA kernel
|
||||
3. Running the CUTLASS grouped GEMM
|
||||
4. Comparing against BF16 reference
|
||||
"""
|
||||
device = "cuda"
|
||||
alignment = 128
|
||||
# N and K must be multiples of 128 for clean swizzle layout
|
||||
n_g = random.randint(1, 16) * alignment
|
||||
k_g = random.randint(1, 16) * alignment
|
||||
|
||||
expert_offset = 0
|
||||
expert_offsets_input = []
|
||||
problem_sizes = []
|
||||
input_list = []
|
||||
weight_list = []
|
||||
|
||||
for g in range(num_experts):
|
||||
m_g = random.randint(1, 256)
|
||||
expert_offsets_input.append(expert_offset)
|
||||
expert_offset += m_g
|
||||
problem_sizes.append([m_g, n_g, k_g])
|
||||
|
||||
input_list.append(
|
||||
torch.normal(0.0, std=0.5, size=(m_g, k_g), device=device, dtype=out_dtype)
|
||||
)
|
||||
weight_list.append(
|
||||
torch.normal(0.0, std=0.5, size=(n_g, k_g), device=device, dtype=out_dtype)
|
||||
)
|
||||
|
||||
input_tensor = torch.concat(input_list, dim=0) # [M_total, K]
|
||||
|
||||
# --- Quantize INPUTS via mxfp4_experts_quant ---
|
||||
input_bs_offsets = []
|
||||
tot = 0
|
||||
for g in range(num_experts):
|
||||
input_bs_offsets.append(tot)
|
||||
tot += align(problem_sizes[g][0], 128)
|
||||
input_bs_offsets.append(tot)
|
||||
|
||||
_inp_expert_offsets = torch.tensor(
|
||||
expert_offsets_input + [expert_offset], device=device, dtype=torch.int32
|
||||
)
|
||||
_inp_bs_offsets = torch.tensor(input_bs_offsets, device=device, dtype=torch.int32)
|
||||
|
||||
input_quant, input_sf = ops.mxfp4_experts_quant(
|
||||
input_tensor,
|
||||
_inp_expert_offsets,
|
||||
_inp_bs_offsets,
|
||||
num_experts,
|
||||
topk=1,
|
||||
)
|
||||
|
||||
# --- Quantize WEIGHTS via mxfp4_experts_quant ---
|
||||
# Treat each expert's N weight rows as an "expert" with N tokens
|
||||
weight_tensor = torch.concat(weight_list, dim=0) # [E*N, K]
|
||||
weight_expert_offsets = [g * n_g for g in range(num_experts)] + [num_experts * n_g]
|
||||
# N is always multiple of 128, so blockscale offsets are clean
|
||||
weight_bs_offsets = [g * n_g for g in range(num_experts)] + [num_experts * n_g]
|
||||
|
||||
_wt_expert_offsets = torch.tensor(
|
||||
weight_expert_offsets, device=device, dtype=torch.int32
|
||||
)
|
||||
_wt_bs_offsets = torch.tensor(weight_bs_offsets, device=device, dtype=torch.int32)
|
||||
|
||||
weight_quant, weight_sf = ops.mxfp4_experts_quant(
|
||||
weight_tensor,
|
||||
_wt_expert_offsets,
|
||||
_wt_bs_offsets,
|
||||
num_experts,
|
||||
topk=1,
|
||||
)
|
||||
|
||||
# Reshape weight quantized data to [E, N, K//2]
|
||||
weight_quant = weight_quant[: num_experts * n_g].view(num_experts, n_g, k_g // 2)
|
||||
|
||||
# Reshape weight scale factors to [E, N, K//32]
|
||||
# The quant kernel produces uint8 SF buffer. Each row has K//32 SFs.
|
||||
scales_per_row = k_g // MXFP4_BLOCK_SIZE
|
||||
weight_sf_flat = weight_sf.view(-1)[: num_experts * n_g * scales_per_row]
|
||||
weight_sf_3d = weight_sf_flat.view(num_experts, n_g, scales_per_row)
|
||||
|
||||
# Output
|
||||
output = torch.empty((expert_offset, n_g), device=device, dtype=out_dtype)
|
||||
|
||||
_problem_sizes = torch.tensor(problem_sizes, device=device, dtype=torch.int32)
|
||||
_expert_offsets = torch.tensor(
|
||||
expert_offsets_input, device=device, dtype=torch.int32
|
||||
)
|
||||
_input_bs = torch.tensor(input_bs_offsets[:-1], device=device, dtype=torch.int32)
|
||||
|
||||
# Run the MXFP4 grouped GEMM
|
||||
ops.cutlass_mxfp4_moe_mm(
|
||||
output,
|
||||
input_quant,
|
||||
weight_quant,
|
||||
input_sf,
|
||||
weight_sf_3d,
|
||||
_problem_sizes,
|
||||
_expert_offsets,
|
||||
_input_bs,
|
||||
)
|
||||
|
||||
# Reference: BF16 matmul
|
||||
ref_output = compute_ref_output(
|
||||
input_tensor=input_tensor,
|
||||
weight_list=weight_list,
|
||||
expert_offsets=expert_offsets_input,
|
||||
expert_offset=expert_offset,
|
||||
num_experts=num_experts,
|
||||
)
|
||||
|
||||
# Compare per-expert
|
||||
for g in range(num_experts):
|
||||
start = expert_offsets_input[g]
|
||||
end = expert_offsets_input[g + 1] if g + 1 < num_experts else expert_offset
|
||||
if start == end:
|
||||
continue
|
||||
baseline = ref_output[start:end]
|
||||
actual = output[start:end]
|
||||
diff = calc_diff(actual, baseline)
|
||||
print(
|
||||
f"m_g={end - start} n_g={n_g} k_g={k_g} "
|
||||
f"num_experts={num_experts}, "
|
||||
f"out_dtype={out_dtype}, diff={diff:.5f}"
|
||||
)
|
||||
# FP4 quantization is very lossy (~4 bits precision)
|
||||
# Comparing quantized vs full-precision gives cosine diff of 0.05-0.15
|
||||
assert diff < 0.15, f"Expert {g}: diff={diff:.5f} exceeds threshold"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_sm100_supported(),
|
||||
reason="mxfp4_experts_quant requires CUDA SM100",
|
||||
)
|
||||
def test_mxfp4_experts_quant_basic():
|
||||
"""Basic smoke test for the MXFP4 experts quantization kernel."""
|
||||
device = "cuda"
|
||||
num_experts = 4
|
||||
k = 256
|
||||
tokens_per_expert = 16
|
||||
|
||||
total_tokens = tokens_per_expert * num_experts
|
||||
input_tensor = torch.randn(total_tokens, k, device=device, dtype=torch.bfloat16) / 5
|
||||
|
||||
expert_offsets = [i * tokens_per_expert for i in range(num_experts + 1)]
|
||||
blockscale_offsets = [
|
||||
align(i * tokens_per_expert, 128) for i in range(num_experts + 1)
|
||||
]
|
||||
|
||||
_expert_offsets = torch.tensor(expert_offsets, device=device, dtype=torch.int32)
|
||||
_blockscale_offsets = torch.tensor(
|
||||
blockscale_offsets, device=device, dtype=torch.int32
|
||||
)
|
||||
|
||||
output, output_sf = ops.mxfp4_experts_quant(
|
||||
input_tensor,
|
||||
_expert_offsets,
|
||||
_blockscale_offsets,
|
||||
num_experts,
|
||||
topk=1,
|
||||
)
|
||||
|
||||
assert output.shape == (total_tokens, k // 2)
|
||||
assert output.dtype == torch.uint8
|
||||
assert output_sf.dtype == torch.uint8
|
||||
assert output.any(), "Quantized output is all zeros"
|
||||
print(
|
||||
f"MXFP4 experts quant: output shape={output.shape}, sf shape={output_sf.shape}"
|
||||
)
|
||||
print("PASSED")
|
||||
|
||||
|
||||
def untile_cutlass_scale(scale_raw: torch.Tensor, rows: int, K: int) -> torch.Tensor:
|
||||
"""Convert CUTLASS tiled scale back to flat [M, K//32] layout.
|
||||
|
||||
CUTLASS tiled layout: [numMTiles, numKTiles, 32(outerM), 4(innerM), 4(innerK)]
|
||||
Produced by: padded.reshape(numMTiles, 4, 32, numKTiles, 4).permute(0,3,2,1,4)
|
||||
To undo: tiled.permute(0, 3, 2, 1, 4).reshape(padded_M, padded_sK)
|
||||
"""
|
||||
num_scale_cols = K // MXFP4_BLOCK_SIZE
|
||||
num_m_tiles = (rows + 127) // 128
|
||||
num_k_tiles = (num_scale_cols + 3) // 4
|
||||
padded_M = num_m_tiles * 128
|
||||
padded_sK = num_k_tiles * 4
|
||||
|
||||
scale_bytes = scale_raw.view(torch.uint8).flatten()
|
||||
total_bytes = padded_M * padded_sK
|
||||
tiled = scale_bytes[:total_bytes].reshape(num_m_tiles, num_k_tiles, 32, 4, 4)
|
||||
undone = tiled.permute(0, 3, 2, 1, 4).contiguous()
|
||||
return undone.reshape(padded_M, padded_sK)[:rows, :num_scale_cols]
|
||||
|
||||
|
||||
def compute_reference_e8m0_scale(block_max: float) -> int:
|
||||
"""Compute the expected OCP MX spec E8M0 scale for a given block max.
|
||||
|
||||
The CUTLASS kernel uses round-to-nearest on the mantissa:
|
||||
rounded_bits = (float_bits + (1 << 21)) & 0xFF800000
|
||||
biased_exp = (rounded_bits >> 23) & 0xFF
|
||||
scale_exp = max(biased_exp - 2, 0)
|
||||
|
||||
This ensures max_val / scale <= 6.0 for most inputs.
|
||||
"""
|
||||
import struct
|
||||
|
||||
if block_max <= 0:
|
||||
return 0
|
||||
# Replicate the kernel's rounding logic in Python
|
||||
float_bytes = struct.pack("f", block_max)
|
||||
max_bits = struct.unpack("I", float_bytes)[0]
|
||||
rounded_bits = (max_bits + (1 << 21)) & 0xFF800000
|
||||
biased_exp = (rounded_bits >> 23) & 0xFF
|
||||
scale_exp = max(int(biased_exp) - 2, 0)
|
||||
scale_exp = min(scale_exp, 254)
|
||||
return scale_exp
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_sm100_supported(),
|
||||
reason="mxfp4_experts_quant requires CUDA SM100",
|
||||
)
|
||||
@pytest.mark.parametrize("k", [256, 7168])
|
||||
@pytest.mark.parametrize("m", [16, 64])
|
||||
def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
|
||||
"""Test that mxfp4_experts_quant computes E8M0 block scales correctly
|
||||
per OCP MX spec (not the NVFP4 formula).
|
||||
|
||||
The old buggy kernel used: floor(log2(max/6)) + 127
|
||||
The fixed kernel uses: round_nearest_exp(max) - 2
|
||||
|
||||
This test verifies:
|
||||
1. Scales match the expected OCP MX formula for all blocks
|
||||
2. No block max exceeds the representable range (no unexpected saturation)
|
||||
3. Reconstruction error is within expected bounds for MXFP4
|
||||
"""
|
||||
device = "cuda"
|
||||
|
||||
# Generate input with controlled range
|
||||
input_tensor = torch.randn(m, k, device=device, dtype=torch.bfloat16) * 0.5
|
||||
|
||||
# Quantize
|
||||
num_experts = 1
|
||||
expert_offsets = torch.tensor([0, m], device=device, dtype=torch.int32)
|
||||
num_k_tiles = (k // MXFP4_BLOCK_SIZE + 3) // 4
|
||||
blockscale_offsets = torch.tensor(
|
||||
[0, align(m, 128) * num_k_tiles], device=device, dtype=torch.int32
|
||||
)
|
||||
|
||||
output_fp4, output_sf = ops.mxfp4_experts_quant(
|
||||
input_tensor, expert_offsets, blockscale_offsets, num_experts, topk=1
|
||||
)
|
||||
|
||||
# Untile scale to flat layout for verification
|
||||
scale_flat = untile_cutlass_scale(output_sf, m, k)
|
||||
assert scale_flat.shape == (m, k // MXFP4_BLOCK_SIZE)
|
||||
|
||||
# Verify each block's scale matches the OCP MX spec formula
|
||||
num_blocks = k // MXFP4_BLOCK_SIZE
|
||||
mismatches = 0
|
||||
buggy_pattern = 0 # count blocks where scale is 1-2 lower than expected
|
||||
|
||||
for row in range(m):
|
||||
for blk in range(num_blocks):
|
||||
block_start = blk * MXFP4_BLOCK_SIZE
|
||||
block_end = block_start + MXFP4_BLOCK_SIZE
|
||||
block_max = (
|
||||
input_tensor[row, block_start:block_end].float().abs().max().item()
|
||||
)
|
||||
|
||||
actual_scale = scale_flat[row, blk].item()
|
||||
expected_scale = compute_reference_e8m0_scale(block_max)
|
||||
|
||||
if actual_scale != expected_scale:
|
||||
mismatches += 1
|
||||
if actual_scale < expected_scale:
|
||||
buggy_pattern += 1
|
||||
|
||||
total_blocks = m * num_blocks
|
||||
match_rate = (total_blocks - mismatches) / total_blocks
|
||||
|
||||
print(
|
||||
f" m={m}, k={k}: scale match rate = {match_rate * 100:.2f}% "
|
||||
f"({mismatches}/{total_blocks} mismatches)"
|
||||
)
|
||||
|
||||
# The fixed kernel should match the reference formula exactly
|
||||
assert match_rate > 0.99, (
|
||||
f"E8M0 scale match rate too low: {match_rate * 100:.2f}%. "
|
||||
f"Buggy pattern (scale too low): {buggy_pattern}/{mismatches}. "
|
||||
f"This suggests the NVFP4 formula bug is present."
|
||||
)
|
||||
|
||||
# Extra check: if most mismatches show scale < expected, it's the old bug
|
||||
if mismatches > 0:
|
||||
assert buggy_pattern / mismatches < 0.5, (
|
||||
f"Most scale mismatches show scale too LOW ({buggy_pattern}/{mismatches}). "
|
||||
"This is the signature of the NVFP4 formula bug in nvfp4_utils.cuh."
|
||||
)
|
||||
|
||||
# Verify reconstruction error is within MXFP4 expected bounds
|
||||
# Dequantize and check cosine similarity
|
||||
fp4_lut = torch.tensor(
|
||||
[0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6],
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
lo = (output_fp4 & 0x0F).long()
|
||||
hi = ((output_fp4 >> 4) & 0x0F).long()
|
||||
unpacked = torch.stack([lo, hi], dim=-1).reshape(m, k)
|
||||
fp4_vals = fp4_lut[unpacked]
|
||||
|
||||
scales_expanded = 2.0 ** (scale_flat.float() - 127.0)
|
||||
scales_expanded = scales_expanded.unsqueeze(-1).expand(-1, -1, MXFP4_BLOCK_SIZE)
|
||||
scales_expanded = scales_expanded.reshape(m, k)
|
||||
recon = (fp4_vals * scales_expanded).bfloat16()
|
||||
|
||||
# Cosine similarity should be > 0.99 for well-behaved MXFP4 quantization
|
||||
cos_sim = torch.nn.functional.cosine_similarity(
|
||||
recon.float().flatten().unsqueeze(0),
|
||||
input_tensor.float().flatten().unsqueeze(0),
|
||||
).item()
|
||||
max_abs_diff = (recon.float() - input_tensor.float()).abs().max().item()
|
||||
|
||||
print(
|
||||
f" Reconstruction: cosine_sim={cos_sim:.6f}, max_abs_diff={max_abs_diff:.4f}"
|
||||
)
|
||||
|
||||
assert cos_sim > 0.99, (
|
||||
f"Reconstruction cosine similarity too low: {cos_sim:.6f}. "
|
||||
f"Expected > 0.99 for correct MXFP4 quantization."
|
||||
)
|
||||
# With correct E8M0, max abs diff should be bounded by scale * 6
|
||||
# (worst case: value just below threshold rounds to wrong FP4 code)
|
||||
assert max_abs_diff < 1.0, (
|
||||
f"Max reconstruction error too large: {max_abs_diff:.4f}. "
|
||||
"Likely caused by incorrect E8M0 scale (values saturating to ±6)."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_sm100_supported(),
|
||||
reason="mxfp4_experts_quant requires CUDA SM100",
|
||||
)
|
||||
def test_mxfp4_experts_quant_no_saturation():
|
||||
"""Test that the E8M0 scale is large enough to avoid unexpected saturation.
|
||||
|
||||
With the buggy NVFP4 formula, the scale was too small causing most values
|
||||
to saturate to ±6 in FP4. The fixed OCP MX formula should ensure that
|
||||
block_max / scale <= 6.0 (the max E2M1 value) in almost all cases.
|
||||
"""
|
||||
device = "cuda"
|
||||
|
||||
m, k = 128, 1024
|
||||
# Use inputs with known range to make saturation detectable
|
||||
input_tensor = torch.randn(m, k, device=device, dtype=torch.bfloat16) * 0.5
|
||||
|
||||
num_experts = 1
|
||||
expert_offsets = torch.tensor([0, m], device=device, dtype=torch.int32)
|
||||
num_k_tiles = (k // MXFP4_BLOCK_SIZE + 3) // 4
|
||||
blockscale_offsets = torch.tensor(
|
||||
[0, align(m, 128) * num_k_tiles], device=device, dtype=torch.int32
|
||||
)
|
||||
|
||||
output_fp4, output_sf = ops.mxfp4_experts_quant(
|
||||
input_tensor, expert_offsets, blockscale_offsets, num_experts, topk=1
|
||||
)
|
||||
|
||||
# Check saturation rate: count FP4 values that are ±6 (codes 7 and 15)
|
||||
lo = output_fp4 & 0x0F
|
||||
hi = (output_fp4 >> 4) & 0x0F
|
||||
# Code 7 = +6.0, code 15 = -6.0
|
||||
saturated = ((lo == 7) | (lo == 15) | (hi == 7) | (hi == 15)).sum().item()
|
||||
total_values = m * k
|
||||
saturation_rate = saturated / total_values
|
||||
|
||||
print(
|
||||
f" Saturation rate: {saturation_rate * 100:.2f}% "
|
||||
f"({saturated}/{total_values} values at ±6)"
|
||||
)
|
||||
|
||||
# For Gaussian input with std=0.5, saturation should be very rare
|
||||
# (±6 * scale is far from the typical range).
|
||||
# The buggy kernel had ~30-50% saturation; fixed should be < 5%.
|
||||
assert saturation_rate < 0.05, (
|
||||
f"FP4 saturation rate too high: {saturation_rate * 100:.2f}%. "
|
||||
"This suggests the E8M0 scale is too small (NVFP4 formula bug). "
|
||||
"Expected < 5% for Gaussian(0, 0.5) input with correct OCP MX scale."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v", "-s"])
|
||||
@@ -1,176 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MXFP8 MoE backend selection for the AITER FlyDSL kernel (gfx950).
|
||||
|
||||
GPU-free: mocks the platform (gfx950) and the aiter fused-MoE enable flag, then
|
||||
exercises the oracle so the FlyDSL backend is auto-picked when usable (including
|
||||
under expert parallelism, since apply() forwards the expert_map as aiter's
|
||||
expert_mask) and skipped (native fallback) when the device is unsupported or the
|
||||
aiter runtime is disabled.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("This test can only run on ROCm.", allow_module_level=True)
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config # noqa: E402
|
||||
from vllm.model_executor.layers.fused_moe.activation import ( # noqa: E402
|
||||
MoEActivation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import ( # noqa: E402
|
||||
_AITER_SWIGLU_ALPHA,
|
||||
_AITER_SWIGLU_BETA,
|
||||
AiterMxfp8Experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import ( # noqa: E402
|
||||
Mxfp8EmulationTritonExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import ( # noqa: E402
|
||||
FusedMoEActivationFormat,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( # noqa: E402
|
||||
Fp8MoeBackend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import ( # noqa: E402
|
||||
_BACKEND_NAME_MAP,
|
||||
_SUPPORTED_BACKENDS,
|
||||
_mxfp8_backend_to_kernel_cls,
|
||||
_select_kernel_cls,
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
)
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops # noqa: E402 # isort: skip
|
||||
|
||||
_AITER_MOD = "vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe"
|
||||
|
||||
|
||||
def _config(ep_size: int = 1):
|
||||
# AiterMxfp8Experts hardcodes SwiGLU-OAI: match its required activation and
|
||||
# alpha/beta so is_supported_config doesn't reject the config on those grounds.
|
||||
cfg = make_dummy_moe_config(
|
||||
num_experts=128,
|
||||
experts_per_token=4,
|
||||
hidden_dim=6144,
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
)
|
||||
cfg = dataclasses.replace(
|
||||
cfg, swiglu_alpha=_AITER_SWIGLU_ALPHA, swiglu_beta=_AITER_SWIGLU_BETA
|
||||
)
|
||||
if ep_size != 1:
|
||||
cfg = dataclasses.replace(
|
||||
cfg,
|
||||
moe_parallel_config=dataclasses.replace(
|
||||
cfg.moe_parallel_config, ep_size=ep_size, use_ep=True
|
||||
),
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
def _gfx950():
|
||||
"""Patch the platform so the device gate (gfx950 / MX) passes off-ROCm."""
|
||||
return patch.multiple(
|
||||
f"{_AITER_MOD}.current_platform",
|
||||
is_rocm=lambda: True,
|
||||
supports_mx=lambda: True,
|
||||
)
|
||||
|
||||
|
||||
def _aiter_moe_enabled(present: bool):
|
||||
return patch.object(rocm_aiter_ops, "is_fused_moe_enabled", return_value=present)
|
||||
|
||||
|
||||
def test_aiter_mxfp8_registered():
|
||||
"""The FlyDSL backend is auto-selectable and reachable via --moe-backend aiter."""
|
||||
assert Fp8MoeBackend.AITER_MXFP8 in _SUPPORTED_BACKENDS
|
||||
assert _BACKEND_NAME_MAP["aiter"] is Fp8MoeBackend.AITER_MXFP8
|
||||
assert _mxfp8_backend_to_kernel_cls(Fp8MoeBackend.AITER_MXFP8) == [
|
||||
AiterMxfp8Experts
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ep_size", [1, 2])
|
||||
def test_ep_supported(ep_size):
|
||||
"""FlyDSL accepts both TP and EP: apply() forwards expert_map as expert_mask."""
|
||||
assert (
|
||||
AiterMxfp8Experts._supports_parallel_config(
|
||||
_config(ep_size).moe_parallel_config
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"present,ep_size,supported,reason_substr",
|
||||
[
|
||||
(True, 1, True, None), # gfx950 + aiter MoE + TP -> selectable
|
||||
(True, 2, True, None), # gfx950 + aiter MoE + EP -> selectable (expert_mask)
|
||||
(
|
||||
False,
|
||||
1,
|
||||
False,
|
||||
"does not support current device",
|
||||
), # disabled -> not selected
|
||||
],
|
||||
)
|
||||
def test_is_supported_config(present, ep_size, supported, reason_substr):
|
||||
with _gfx950(), _aiter_moe_enabled(present):
|
||||
ok, reason = AiterMxfp8Experts.is_supported_config(
|
||||
AiterMxfp8Experts,
|
||||
_config(ep_size),
|
||||
kMxfp8Static,
|
||||
kMxfp8Dynamic,
|
||||
FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert ok is supported
|
||||
if reason_substr is not None:
|
||||
assert reason_substr in reason
|
||||
|
||||
|
||||
def test_explicit_moe_backend_aiter():
|
||||
"""--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear
|
||||
ValueError when the aiter MoE runtime is disabled."""
|
||||
with _gfx950(), _aiter_moe_enabled(True):
|
||||
assert (
|
||||
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
|
||||
is AiterMxfp8Experts
|
||||
)
|
||||
assert (
|
||||
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(2))
|
||||
is AiterMxfp8Experts
|
||||
)
|
||||
with (
|
||||
_gfx950(),
|
||||
_aiter_moe_enabled(False),
|
||||
pytest.raises(ValueError, match="does not support current device"),
|
||||
):
|
||||
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
|
||||
|
||||
|
||||
def test_gfx950_picks_aiter():
|
||||
"""Auto-select on real ROCm hardware with aiter MoE enabled -> FlyDSL wins."""
|
||||
with (
|
||||
patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=True),
|
||||
_aiter_moe_enabled(True),
|
||||
):
|
||||
backend, experts_cls = select_mxfp8_moe_backend(_config())
|
||||
assert backend is Fp8MoeBackend.AITER_MXFP8
|
||||
assert experts_cls is AiterMxfp8Experts
|
||||
|
||||
|
||||
def test_gfx942_picks_emulation():
|
||||
"""Flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
|
||||
dot_scaled backend wins instead."""
|
||||
with patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=False):
|
||||
backend, experts_cls = select_mxfp8_moe_backend(_config())
|
||||
assert backend is Fp8MoeBackend.EMULATION
|
||||
assert experts_cls is Mxfp8EmulationTritonExperts
|
||||
@@ -1,291 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
dequantize_nvfp4_to_dtype,
|
||||
)
|
||||
from tests.kernels.utils import torch_moe
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import nvfp4_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
make_moe_prepare_and_finalize_no_dp_ep,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.has_device_capability(100):
|
||||
pytest.skip(
|
||||
"Nvfp4 Requires compute capability of 10 or above.", allow_module_level=True
|
||||
)
|
||||
|
||||
MNK_FACTORS = [
|
||||
(2, 1024, 1024),
|
||||
(2, 1024, 1536),
|
||||
(2, 3072, 1024),
|
||||
(64, 1024, 1024),
|
||||
(64, 3072, 1024),
|
||||
(64, 2048, 1536),
|
||||
(224, 1024, 1024),
|
||||
(224, 1024, 1536),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", [40, 64, 256])
|
||||
@pytest.mark.parametrize("topk", [1, 6, 8])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_cutlass_fp4_moe_no_graph(
|
||||
m: int, n: int, k: int, e: int, topk: int, dtype: torch.dtype, workspace_init
|
||||
):
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
quant_blocksize = 16
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
|
||||
(_, w1_q, w1_blockscale, w1_gs), (_, w2_q, w2_blockscale, w2_gs) = (
|
||||
make_test_weights(
|
||||
e,
|
||||
n,
|
||||
k,
|
||||
in_dtype=dtype,
|
||||
quant_dtype="nvfp4",
|
||||
block_shape=None, # use quant_blocksize?
|
||||
per_out_ch_quant=False,
|
||||
)
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
a1_gs = torch.ones((e,), device="cuda", dtype=torch.float32)
|
||||
a2_gs = torch.ones((e,), device="cuda", dtype=torch.float32)
|
||||
|
||||
assert w1_gs is not None
|
||||
assert w2_gs is not None
|
||||
assert w1_blockscale is not None
|
||||
assert w2_blockscale is not None
|
||||
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=(1 / w1_gs),
|
||||
g2_alphas=(1 / w2_gs),
|
||||
a1_gscale=a1_gs,
|
||||
a2_gscale=a2_gs,
|
||||
w1_scale=w1_blockscale,
|
||||
w2_scale=w2_blockscale,
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
CutlassExpertsFp4(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
),
|
||||
)
|
||||
|
||||
cutlass_output = kernel.apply(
|
||||
hidden_states=a,
|
||||
w1=w1_q,
|
||||
w2=w2_q,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=e,
|
||||
activation=mk.MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
# Reference check:
|
||||
a_global_scale = (
|
||||
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a.flatten(), dim=-1)
|
||||
).to(torch.float32)
|
||||
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a, a_global_scale)
|
||||
|
||||
a_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
a_fp4,
|
||||
a_scale_interleaved,
|
||||
a_global_scale,
|
||||
dtype=a.dtype,
|
||||
device=a.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
|
||||
w1_d = torch.empty((e, 2 * n, k), device="cuda", dtype=dtype)
|
||||
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
|
||||
|
||||
for idx in range(0, e):
|
||||
w1_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w1_q[idx],
|
||||
w1_blockscale[idx],
|
||||
w1_gs[idx],
|
||||
dtype=dtype,
|
||||
device=w1_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
w2_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w2_q[idx],
|
||||
w2_blockscale[idx],
|
||||
w2_gs[idx],
|
||||
dtype=dtype,
|
||||
device=w2_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
|
||||
torch_output = torch_moe(a_in_dtype, w1_d, w2_d, score, topk)
|
||||
|
||||
torch.testing.assert_close(torch_output, cutlass_output, atol=1e-1, rtol=1e-1)
|
||||
|
||||
|
||||
# step3.5-flash uses swiglustep activation (clipped SwiGLU with limit=7.0)
|
||||
# for MoE layers 43-44. This tests the non-fused activation fallback path
|
||||
# in run_cutlass_moe_fp4 (apply_moe_activation + separate fp4 quantization).
|
||||
# Model dims: e=288, topk=8, n=1280 (moe_intermediate_size), k=4096 (hidden)
|
||||
SWIGLUSTEP_MNK_FACTORS = [
|
||||
(2, 1280, 4096),
|
||||
(64, 1280, 4096),
|
||||
(224, 1280, 4096),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", SWIGLUSTEP_MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", [64, 288])
|
||||
@pytest.mark.parametrize("topk", [1, 8])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_cutlass_fp4_moe_swiglustep(
|
||||
m: int, n: int, k: int, e: int, topk: int, dtype: torch.dtype, workspace_init
|
||||
):
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
quant_blocksize = 16
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
|
||||
(_, w1_q, w1_blockscale, w1_gs), (_, w2_q, w2_blockscale, w2_gs) = (
|
||||
make_test_weights(
|
||||
e,
|
||||
n,
|
||||
k,
|
||||
in_dtype=dtype,
|
||||
quant_dtype="nvfp4",
|
||||
block_shape=None,
|
||||
per_out_ch_quant=False,
|
||||
)
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
a1_gs = torch.ones((e,), device="cuda", dtype=torch.float32)
|
||||
a2_gs = torch.ones((e,), device="cuda", dtype=torch.float32)
|
||||
|
||||
assert w1_gs is not None
|
||||
assert w2_gs is not None
|
||||
assert w1_blockscale is not None
|
||||
assert w2_blockscale is not None
|
||||
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=(1 / w1_gs),
|
||||
g2_alphas=(1 / w2_gs),
|
||||
a1_gscale=a1_gs,
|
||||
a2_gscale=a2_gs,
|
||||
w1_scale=w1_blockscale,
|
||||
w2_scale=w2_blockscale,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
make_moe_prepare_and_finalize_no_dp_ep(use_monolithic=False),
|
||||
CutlassExpertsFp4(
|
||||
moe_config=make_dummy_moe_config(),
|
||||
quant_config=quant_config,
|
||||
),
|
||||
)
|
||||
|
||||
cutlass_output = kernel.apply(
|
||||
hidden_states=a,
|
||||
w1=w1_q,
|
||||
w2=w2_q,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SWIGLUSTEP,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
# Reference: dequantize everything and run torch_moe with swiglustep
|
||||
a_global_scale = (
|
||||
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a.flatten(), dim=-1)
|
||||
).to(torch.float32)
|
||||
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a, a_global_scale)
|
||||
|
||||
a_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
a_fp4,
|
||||
a_scale_interleaved,
|
||||
a_global_scale,
|
||||
dtype=a.dtype,
|
||||
device=a.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
|
||||
w1_d = torch.empty((e, 2 * n, k), device="cuda", dtype=dtype)
|
||||
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
|
||||
|
||||
for idx in range(0, e):
|
||||
w1_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w1_q[idx],
|
||||
w1_blockscale[idx],
|
||||
w1_gs[idx],
|
||||
dtype=dtype,
|
||||
device=w1_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
w2_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w2_q[idx],
|
||||
w2_blockscale[idx],
|
||||
w2_gs[idx],
|
||||
dtype=dtype,
|
||||
device=w2_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
|
||||
torch_output = torch_moe(
|
||||
a_in_dtype,
|
||||
w1_d,
|
||||
w2_d,
|
||||
score,
|
||||
topk,
|
||||
activation=MoEActivation.SWIGLUSTEP,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(torch_output, cutlass_output, atol=1e-1, rtol=1e-1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_cutlass_fp4_moe_no_graph((2, 1024, 1024), 40, 1, torch.half)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,38 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import TritonExperts
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
|
||||
from .modular_kernel_tools.common import Config
|
||||
from .modular_kernel_tools.profile_modular_kernel import run
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="profile_modular_kernel requires a CUDA device",
|
||||
)
|
||||
def test_profile_modular_kernel_smoke(tmp_path):
|
||||
config = Config(
|
||||
Ms=[16],
|
||||
K=128,
|
||||
N=256,
|
||||
E=4,
|
||||
topks=[2],
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=None,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=TritonExperts,
|
||||
world_size=1,
|
||||
torch_trace_dir_path=str(tmp_path),
|
||||
)
|
||||
|
||||
run(config)
|
||||
|
||||
traces = list(tmp_path.glob("m*_*_trace.json"))
|
||||
assert traces, "profile_modular_kernel.run did not emit any chrome traces"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,136 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the AITER biased_grouped_topk expert-group count.
|
||||
|
||||
The AITER kernel is only instantiated for ``NUM_GRP`` in {1, 2, 4, 8}, so the
|
||||
group count derived from ``num_experts`` has to be rounded to a supported
|
||||
value that still divides ``num_experts``. Grouping is a no-op on this path
|
||||
(``topk_group == num_expert_group``), so any such value routes identically;
|
||||
the constraint is purely about which kernel exists.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
AITER_MAX_EXPERTS_PER_GROUP as MAX_EXPERTS_PER_GROUP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
AITER_SUPPORTED_NUM_GRP as SUPPORTED_NUM_GRP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
_aiter_can_use_biased_grouped_topk,
|
||||
_aiter_get_num_expert_group,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
grouped_topk,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_cdna
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "expected"),
|
||||
[
|
||||
(8, 1), # fits in a single group
|
||||
(32, 1), # exactly at the per-group limit
|
||||
(64, 2),
|
||||
(128, 4),
|
||||
(256, 8),
|
||||
(72, 8), # naive ceil gives 3 -> unsupported, rounds up to 8
|
||||
(96, 8), # naive ceil gives 3 -> unsupported, rounds up to 8
|
||||
(160, 8), # naive ceil gives 5 -> unsupported, rounds up to 8
|
||||
(192, 8), # naive ceil gives 6 -> unsupported, rounds up to 8
|
||||
# No supported NUM_GRP divides these within the group-size limit, so
|
||||
# the naive value is kept and the call site declines the kernel.
|
||||
(33, 3),
|
||||
(129, 43),
|
||||
(257, 257),
|
||||
(320, 10),
|
||||
(384, 12),
|
||||
],
|
||||
)
|
||||
def test_group_count_and_guard(num_experts, expected):
|
||||
g = _aiter_get_num_expert_group(num_experts)
|
||||
assert g == expected
|
||||
# The router asserts both unconditionally; rounding must never break them.
|
||||
assert num_experts % g == 0
|
||||
assert num_experts // g <= MAX_EXPERTS_PER_GROUP
|
||||
|
||||
supported = g in SUPPORTED_NUM_GRP
|
||||
# Ample topk, so only the membership check can decline.
|
||||
assert _aiter_can_use_biased_grouped_topk(num_experts, topk=64) == supported
|
||||
if supported:
|
||||
assert _aiter_can_use_biased_grouped_topk(num_experts, topk=g)
|
||||
assert not _aiter_can_use_biased_grouped_topk(num_experts, topk=g - 1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_cdna(), reason="CDNA ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts",
|
||||
[
|
||||
64, # g == 2, not rounded -- covers the no-op claim itself
|
||||
128, # g == 4, not rounded
|
||||
256, # g == 8, not rounded
|
||||
96, # naive 3 -> rounded to 8
|
||||
192, # naive 6 -> rounded to 8
|
||||
],
|
||||
)
|
||||
def test_rounded_group_count_routes_like_the_reference(num_experts):
|
||||
"""The rounded group count must route identically to ``grouped_topk``.
|
||||
|
||||
Gating is continuous random, so exact score ties -- the one place the two
|
||||
may legitimately differ -- have measure zero.
|
||||
"""
|
||||
torch.manual_seed(num_experts)
|
||||
device = "cuda"
|
||||
topk = 8
|
||||
num_tokens = 83 # not a multiple of any warp/tile size
|
||||
g = _aiter_get_num_expert_group(num_experts)
|
||||
assert _aiter_can_use_biased_grouped_topk(num_experts, topk)
|
||||
|
||||
gating = torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device)
|
||||
bias = torch.randn(num_experts, dtype=torch.float32, device=device)
|
||||
|
||||
ref_weights, ref_ids = grouped_topk(
|
||||
hidden_states=torch.empty(num_tokens, 1, device=device),
|
||||
gating_output=gating,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
num_expert_group=g,
|
||||
topk_group=g,
|
||||
scoring_func="sigmoid",
|
||||
e_score_correction_bias=bias,
|
||||
)
|
||||
|
||||
weights = torch.empty(num_tokens, topk, dtype=torch.float32, device=device)
|
||||
ids = torch.empty(num_tokens, topk, dtype=torch.int32, device=device)
|
||||
rocm_aiter_ops.biased_grouped_topk(
|
||||
gating,
|
||||
bias,
|
||||
weights,
|
||||
ids,
|
||||
num_expert_group=g,
|
||||
topk_group=g,
|
||||
need_renorm=True,
|
||||
)
|
||||
|
||||
# Neither side promises an order within a token's top-k, so sort by expert
|
||||
# id and carry the weights through the same permutation -- sorting the two
|
||||
# independently would not catch a weight attached to the wrong expert.
|
||||
order = ids.argsort(dim=-1)
|
||||
ref_order = ref_ids.argsort(dim=-1)
|
||||
torch.testing.assert_close(
|
||||
ids.gather(1, order), ref_ids.to(torch.int32).gather(1, ref_order)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
weights.gather(1, order),
|
||||
ref_weights.gather(1, ref_order),
|
||||
atol=1e-4,
|
||||
rtol=1e-4,
|
||||
)
|
||||
@@ -1,387 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# This is a test for the AITER ops.
|
||||
# It tests if the AITER ops are
|
||||
# 1. correctly registered as custom ops
|
||||
# 2. correctly defined the relationship between
|
||||
# implementation and fake function
|
||||
# 3. can be used with torch.compile
|
||||
# This file will be skipped if AITER is not installed
|
||||
# and the platform is not ROCm.
|
||||
|
||||
import importlib.util
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("This test can only run on ROCm.", allow_module_level=True)
|
||||
|
||||
# this import statement is needed to ensure the ops are registered
|
||||
import vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe # noqa: F401
|
||||
|
||||
# need to import once to ensure the ops are registered
|
||||
# Check if aiter package is installed
|
||||
aiter_available = importlib.util.find_spec("aiter") is not None
|
||||
|
||||
if not aiter_available:
|
||||
pytest.skip("These tests require AITER to run.", allow_module_level=True)
|
||||
|
||||
|
||||
def test_rocm_aiter_biased_grouped_topk_custom_op_registration():
|
||||
"""Test that the custom op is correctly registered."""
|
||||
# Check if the op exists in torch.ops.vllm
|
||||
assert hasattr(torch.ops.vllm, "rocm_aiter_biased_grouped_topk")
|
||||
|
||||
# Check if the op is callable
|
||||
assert callable(torch.ops.vllm.rocm_aiter_biased_grouped_topk)
|
||||
|
||||
|
||||
def test_rocm_aiter_grouped_topk_custom_op_registration():
|
||||
"""Test that the custom op is correctly registered."""
|
||||
# Check if the op exists in torch.ops.vllm
|
||||
assert hasattr(torch.ops.vllm, "rocm_aiter_grouped_topk")
|
||||
|
||||
# Check if the op is callable
|
||||
assert callable(torch.ops.vllm.rocm_aiter_grouped_topk)
|
||||
|
||||
|
||||
def test_rocm_aiter_biased_grouped_topk_torch_compile_compatibility():
|
||||
"""Test that the op can be used with torch.compile."""
|
||||
# Create test tensors
|
||||
token = 64
|
||||
expert = 256
|
||||
num_expert_group = 8
|
||||
topk = 8
|
||||
topk_group = 4
|
||||
renormalize = True
|
||||
scale_factor = 1.0
|
||||
|
||||
gating_output = torch.randn((token, expert), dtype=torch.bfloat16, device="cuda")
|
||||
e_score_correction_bias = torch.randn(
|
||||
(expert,), dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
|
||||
device = gating_output.device
|
||||
topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device)
|
||||
topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device)
|
||||
|
||||
# Define a function that uses the op
|
||||
def biased_grouped_topk_fn(
|
||||
gating_output, e_score_correction_bias, topk_weights, topk_ids
|
||||
):
|
||||
return torch.ops.vllm.rocm_aiter_biased_grouped_topk(
|
||||
gating_output,
|
||||
e_score_correction_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
renormalize,
|
||||
scale_factor,
|
||||
)
|
||||
|
||||
# Verify the op's fake implementation
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.rocm_aiter_biased_grouped_topk,
|
||||
(gating_output, e_score_correction_bias, topk_weights, topk_ids),
|
||||
kwargs={
|
||||
"num_expert_group": num_expert_group,
|
||||
"topk_group": topk_group,
|
||||
"need_renorm": renormalize,
|
||||
"routed_scaling_factor": scale_factor,
|
||||
},
|
||||
test_utils=("test_faketensor"),
|
||||
)
|
||||
|
||||
# Compile the function with appropriate settings
|
||||
compiled_fn = torch.compile(
|
||||
biased_grouped_topk_fn,
|
||||
fullgraph=True,
|
||||
backend="inductor",
|
||||
mode="reduce-overhead",
|
||||
dynamic=False,
|
||||
)
|
||||
|
||||
topk_weights_original = torch.empty(
|
||||
(token, topk), dtype=torch.float32, device=device
|
||||
)
|
||||
topk_ids_original = torch.empty((token, topk), dtype=torch.int32, device=device)
|
||||
|
||||
topk_weights_compiled = torch.empty(
|
||||
(token, topk), dtype=torch.float32, device=device
|
||||
)
|
||||
topk_ids_compiled = torch.empty((token, topk), dtype=torch.int32, device=device)
|
||||
|
||||
# Run both compiled (V1 graph mode) and uncompiled versions (V1 eager mode)
|
||||
biased_grouped_topk_fn(
|
||||
gating_output, e_score_correction_bias, topk_weights_original, topk_ids_original
|
||||
)
|
||||
compiled_fn(
|
||||
gating_output, e_score_correction_bias, topk_weights_compiled, topk_ids_compiled
|
||||
)
|
||||
|
||||
# Sort the results for comparison since the order might not be deterministic
|
||||
topk_ids_original, indices_original = torch.sort(topk_ids_original)
|
||||
topk_weights_original = torch.gather(topk_weights_original, 1, indices_original)
|
||||
|
||||
topk_ids_compiled, indices_compiled = torch.sort(topk_ids_compiled)
|
||||
topk_weights_compiled = torch.gather(topk_weights_compiled, 1, indices_compiled)
|
||||
|
||||
# Verify results match
|
||||
assert torch.allclose(
|
||||
topk_weights_original, topk_weights_compiled, rtol=1e-2, atol=1e-2
|
||||
)
|
||||
assert torch.allclose(topk_ids_original, topk_ids_compiled)
|
||||
|
||||
|
||||
def test_rocm_aiter_grouped_topk_torch_compile_compatibility():
|
||||
"""Test that the op can be used with torch.compile."""
|
||||
# Create test tensors
|
||||
token = 64
|
||||
expert = 256
|
||||
num_expert_group = 8
|
||||
topk = 8
|
||||
topk_group = 4
|
||||
renormalize = True
|
||||
scoring_func = "softmax"
|
||||
scale_factor = 1.0
|
||||
|
||||
gating_output = torch.randn((token, expert), dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
device = gating_output.device
|
||||
topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device)
|
||||
topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device)
|
||||
|
||||
# Define a function that uses the op
|
||||
def grouped_topk_fn(gating_output, topk_weights, topk_ids, scoring_func):
|
||||
return torch.ops.vllm.rocm_aiter_grouped_topk(
|
||||
gating_output,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
renormalize,
|
||||
scoring_func,
|
||||
scale_factor,
|
||||
)
|
||||
|
||||
# Verify the op's fake implementation
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.rocm_aiter_grouped_topk,
|
||||
(gating_output, topk_weights, topk_ids),
|
||||
kwargs={
|
||||
"num_expert_group": num_expert_group,
|
||||
"topk_group": topk_group,
|
||||
"need_renorm": renormalize,
|
||||
"scoring_func": scoring_func,
|
||||
"routed_scaling_factor": scale_factor,
|
||||
},
|
||||
test_utils=("test_faketensor"),
|
||||
)
|
||||
|
||||
# Compile the function with appropriate settings
|
||||
compiled_fn = torch.compile(
|
||||
grouped_topk_fn,
|
||||
fullgraph=True,
|
||||
backend="inductor",
|
||||
mode="reduce-overhead",
|
||||
dynamic=False,
|
||||
)
|
||||
|
||||
topk_weights_original = torch.empty(
|
||||
(token, topk), dtype=torch.float32, device=device
|
||||
)
|
||||
topk_ids_original = torch.empty((token, topk), dtype=torch.int32, device=device)
|
||||
|
||||
topk_weights_compiled = torch.empty(
|
||||
(token, topk), dtype=torch.float32, device=device
|
||||
)
|
||||
topk_ids_compiled = torch.empty((token, topk), dtype=torch.int32, device=device)
|
||||
|
||||
# Run both compiled (V1 graph mode) and uncompiled versions (V1 eager mode)
|
||||
grouped_topk_fn(
|
||||
gating_output, topk_weights_original, topk_ids_original, scoring_func
|
||||
)
|
||||
compiled_fn(gating_output, topk_weights_compiled, topk_ids_compiled, scoring_func)
|
||||
|
||||
# Sort the results for comparison since the order might not be deterministic
|
||||
topk_ids_original, indices_original = torch.sort(topk_ids_original)
|
||||
topk_weights_original = torch.gather(topk_weights_original, 1, indices_original)
|
||||
|
||||
topk_ids_compiled, indices_compiled = torch.sort(topk_ids_compiled)
|
||||
topk_weights_compiled = torch.gather(topk_weights_compiled, 1, indices_compiled)
|
||||
|
||||
# Verify results match
|
||||
assert torch.allclose(
|
||||
topk_weights_original, topk_weights_compiled, rtol=1e-2, atol=1e-2
|
||||
)
|
||||
assert torch.allclose(topk_ids_original, topk_ids_compiled)
|
||||
|
||||
|
||||
def test_rocm_aiter_topk_gating_custom_op_registration():
|
||||
assert callable(torch.ops.vllm.rocm_aiter_topk_gating)
|
||||
|
||||
|
||||
def _sort_routing(weights: torch.Tensor, ids: torch.Tensor):
|
||||
ids_sorted, order = torch.sort(ids, dim=-1)
|
||||
weights_sorted = torch.gather(weights, 1, order)
|
||||
return weights_sorted, ids_sorted
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 64])
|
||||
@pytest.mark.parametrize("num_experts", [256, 512])
|
||||
@pytest.mark.parametrize("topk", [8, 10])
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
def test_rocm_aiter_topk_gating_matches_softmax_reference(
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
):
|
||||
"""Numerical check vs softmax + topk, including Qwen3.8 (E=512, k=10)."""
|
||||
torch.manual_seed(0)
|
||||
gating_output = torch.randn(
|
||||
(num_tokens, num_experts), dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
scores = torch.softmax(gating_output.float(), dim=-1)
|
||||
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
|
||||
|
||||
topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda")
|
||||
topk_ids = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda")
|
||||
token_expert_indices = torch.empty(
|
||||
(num_tokens, topk), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
torch.ops.vllm.rocm_aiter_topk_gating(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
0,
|
||||
"",
|
||||
)
|
||||
|
||||
# BF16 logits can tie at the top-k boundary, so a different tied expert is
|
||||
# valid. Check selection against the cutoff and weights against the scores
|
||||
# of the experts the kernel actually selected.
|
||||
assert ((topk_ids >= 0) & (topk_ids < num_experts)).all()
|
||||
sorted_ids = torch.sort(topk_ids, dim=-1).values
|
||||
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
|
||||
selected_scores = torch.gather(scores, 1, topk_ids.long())
|
||||
assert (selected_scores >= cutoff).all()
|
||||
expected_weights = selected_scores
|
||||
if renormalize:
|
||||
expected_weights = expected_weights / expected_weights.sum(dim=-1, keepdim=True)
|
||||
torch.testing.assert_close(topk_weights, expected_weights, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def test_rocm_aiter_topk_gating_shared_expert_suffix():
|
||||
"""Fused shared expert: softmax top-k on the prefix, sigmoid appended.
|
||||
|
||||
Matches the Qwen3.8 FSE launch: 512 routed experts, top-10, one shared
|
||||
column, weights wide enough for the suffix, and the shared id prefilled.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_experts, topk, num_shared = 4, 512, 10, 1
|
||||
routed = torch.randn((num_tokens, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
shared = torch.randn((num_tokens, num_shared), dtype=torch.bfloat16, device="cuda")
|
||||
shared[0, 0] = 40
|
||||
gating_output = torch.cat([routed, shared], dim=-1).contiguous()
|
||||
width = topk + num_shared
|
||||
shared_ids = torch.arange(
|
||||
num_experts, num_experts + num_shared, device="cuda", dtype=torch.int32
|
||||
)
|
||||
topk_ids = torch.full((num_tokens, width), -1, dtype=torch.int32, device="cuda")
|
||||
topk_ids[:, topk:] = shared_ids
|
||||
topk_weights = torch.empty((num_tokens, width), dtype=torch.float32, device="cuda")
|
||||
token_expert_indices = torch.empty(
|
||||
(num_tokens, topk), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
torch.ops.vllm.rocm_aiter_topk_gating(
|
||||
topk_weights,
|
||||
topk_ids[:, :topk],
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
True,
|
||||
num_shared,
|
||||
"sigmoid",
|
||||
)
|
||||
|
||||
scores = torch.softmax(routed.float(), dim=-1)
|
||||
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
|
||||
routed_ids = topk_ids[:, :topk]
|
||||
assert ((routed_ids >= 0) & (routed_ids < num_experts)).all()
|
||||
sorted_ids = torch.sort(routed_ids, dim=-1).values
|
||||
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
|
||||
selected = torch.gather(scores, 1, routed_ids.long())
|
||||
assert (selected >= cutoff).all()
|
||||
expected = selected / selected.sum(dim=-1, keepdim=True)
|
||||
torch.testing.assert_close(topk_weights[:, :topk], expected, atol=2e-2, rtol=2e-2)
|
||||
torch.testing.assert_close(
|
||||
topk_weights[:, topk:],
|
||||
torch.sigmoid(shared.float()),
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
)
|
||||
assert torch.equal(topk_ids[:, topk:], shared_ids.expand_as(topk_ids[:, topk:]))
|
||||
|
||||
|
||||
def test_rocm_aiter_topk_gating_torch_compile_compatibility():
|
||||
token, expert, topk, renormalize = 4, 512, 10, True
|
||||
gating_output = torch.randn((token, expert), dtype=torch.bfloat16, device="cuda")
|
||||
device = gating_output.device
|
||||
|
||||
def gating_fn(gating_output, topk_weights, topk_ids, token_expert_indices):
|
||||
return torch.ops.vllm.rocm_aiter_topk_gating(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
0,
|
||||
"",
|
||||
)
|
||||
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.rocm_aiter_topk_gating,
|
||||
(
|
||||
torch.empty((token, topk), dtype=torch.float32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
gating_output,
|
||||
),
|
||||
kwargs={
|
||||
"renormalize": renormalize,
|
||||
"num_shared_experts": 0,
|
||||
"shared_expert_scoring_func": "",
|
||||
},
|
||||
test_utils=("test_faketensor"),
|
||||
)
|
||||
|
||||
compiled_fn = torch.compile(
|
||||
gating_fn,
|
||||
fullgraph=True,
|
||||
backend="inductor",
|
||||
mode="reduce-overhead",
|
||||
dynamic=False,
|
||||
)
|
||||
|
||||
def alloc():
|
||||
return (
|
||||
torch.empty((token, topk), dtype=torch.float32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
w0, i0, t0 = alloc()
|
||||
w1, i1, t1 = alloc()
|
||||
gating_fn(gating_output, w0, i0, t0)
|
||||
compiled_fn(gating_output, w1, i1, t1)
|
||||
w0, i0 = _sort_routing(w0, i0)
|
||||
w1, i1 = _sort_routing(w1, i1)
|
||||
assert torch.allclose(w0, w1, rtol=1e-2, atol=1e-2)
|
||||
assert torch.equal(i0, i1)
|
||||
@@ -1,303 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Routed-expert capture on the monolithic (fused router + experts) MoE path.
|
||||
|
||||
A monolithic kernel that ``supports_routing_replay_capture`` must write, into
|
||||
the leading rows of ``routing_replay_out``, exactly the experts it routed each
|
||||
token to. The router logits plant an unambiguous top-k per token, so the test
|
||||
checks the reported experts against that answer and not just their range.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
TrtLlmBf16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
|
||||
TrtLlmFp8ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
|
||||
TrtLlmNvFp4ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEExpertsMonolithic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
try:
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
except ImportError:
|
||||
pytest.skip("flashinfer not available", allow_module_level=True)
|
||||
|
||||
if not has_flashinfer_trtllm_fused_moe() or not current_platform.is_cuda():
|
||||
pytest.skip(
|
||||
"Requires FlashInfer TRT-LLM fused MoE on CUDA",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip(
|
||||
"TRT-LLM fused MoE kernels require SM100+",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
NUM_EXPERTS = 32
|
||||
NUM_GROUPS = 4 # DeepSeekV3 grouped routing
|
||||
TOPK_GROUPS = 2
|
||||
HIDDEN = 1024
|
||||
INTERMEDIATE = 1024
|
||||
DEVICE = torch.device("cuda:0")
|
||||
|
||||
# A kernel under test, and its call: (router_logits, routing_replay_out) -> None.
|
||||
Kernel = tuple[FusedMoEExpertsMonolithic, Callable[[torch.Tensor, torch.Tensor], None]]
|
||||
|
||||
|
||||
def _planted_routing(num_tokens: int, top_k: int) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Router logits with one unambiguous answer under every routing method.
|
||||
|
||||
Each token gives +4 to ``top_k`` experts drawn from at most two expert
|
||||
groups and -4 to the rest, so softmax top-k, sigmoid top-k and DeepSeekV3
|
||||
grouped top-k all pick the same experts, with margins no rounding can flip.
|
||||
Returns the logits and the planted experts of each token, sorted.
|
||||
"""
|
||||
group_size = NUM_EXPERTS // NUM_GROUPS
|
||||
planted = []
|
||||
for _ in range(num_tokens):
|
||||
groups = torch.randperm(NUM_GROUPS)[:TOPK_GROUPS]
|
||||
candidates = (groups[:, None] * group_size + torch.arange(group_size)).flatten()
|
||||
planted.append(candidates[torch.randperm(len(candidates))[:top_k]])
|
||||
experts = torch.stack(planted)
|
||||
logits = torch.full((num_tokens, NUM_EXPERTS), -4.0).scatter_(1, experts, 4.0)
|
||||
return logits.to(DEVICE), experts.sort(dim=-1).values.to(DEVICE)
|
||||
|
||||
|
||||
def _routing_kwargs(routing_method: RoutingMethodType) -> dict:
|
||||
if routing_method != RoutingMethodType.DeepSeekV3:
|
||||
return {}
|
||||
return dict(
|
||||
num_expert_group=NUM_GROUPS,
|
||||
topk_group=TOPK_GROUPS,
|
||||
e_score_correction_bias=torch.zeros(
|
||||
NUM_EXPERTS, device=DEVICE, dtype=torch.bfloat16
|
||||
),
|
||||
routed_scaling_factor=1.0,
|
||||
)
|
||||
|
||||
|
||||
def _moe_config(top_k: int, routing_method: RoutingMethodType) -> FusedMoEConfig:
|
||||
return FusedMoEConfig(
|
||||
num_experts=NUM_EXPERTS,
|
||||
experts_per_token=top_k,
|
||||
hidden_dim=HIDDEN,
|
||||
intermediate_size=INTERMEDIATE,
|
||||
num_local_experts=NUM_EXPERTS,
|
||||
num_logical_experts=NUM_EXPERTS,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
activation=MoEActivation.SILU,
|
||||
device=DEVICE,
|
||||
routing_method=routing_method,
|
||||
max_num_tokens=16,
|
||||
)
|
||||
|
||||
|
||||
def _bf16_kernel(top_k: int, routing_method: RoutingMethodType) -> Kernel:
|
||||
from flashinfer import shuffle_matrix_a
|
||||
from flashinfer.fused_moe import convert_to_block_layout
|
||||
|
||||
def block_major_k(w: torch.Tensor) -> torch.Tensor:
|
||||
"""(E, M, K) -> the BlockMajorK layout ``trtllm_bf16_moe`` expects."""
|
||||
return torch.stack(
|
||||
[
|
||||
convert_to_block_layout(shuffle_matrix_a(e.view(torch.uint8), 64), 128)
|
||||
for e in w
|
||||
]
|
||||
).view(torch.bfloat16)
|
||||
|
||||
experts = TrtLlmBf16ExpertsMonolithic(
|
||||
moe_config=_moe_config(top_k, routing_method),
|
||||
quant_config=FusedMoEQuantConfig.make(
|
||||
quant_dtype=None,
|
||||
per_act_token_quant=False,
|
||||
per_out_ch_quant=False,
|
||||
block_shape=None,
|
||||
),
|
||||
)
|
||||
w13 = block_major_k(
|
||||
torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, device=DEVICE).bfloat16()
|
||||
* 0.1
|
||||
)
|
||||
w2 = block_major_k(
|
||||
torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, device=DEVICE).bfloat16() * 0.1
|
||||
)
|
||||
|
||||
def forward(router_logits, routing_replay_out):
|
||||
hidden_states = torch.randn(len(router_logits), HIDDEN, device=DEVICE) * 0.1
|
||||
experts.apply(
|
||||
hidden_states=hidden_states.bfloat16(),
|
||||
w1=w13,
|
||||
w2=w2,
|
||||
router_logits=router_logits,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=NUM_EXPERTS,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_replay_out=routing_replay_out,
|
||||
**_routing_kwargs(routing_method),
|
||||
)
|
||||
|
||||
return experts, forward
|
||||
|
||||
|
||||
def _fp8_block_kernel(top_k: int, routing_method: RoutingMethodType) -> Kernel:
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
_shuffle_deepseek_fp8_moe_weights,
|
||||
)
|
||||
|
||||
block = 128
|
||||
w13, w2 = _shuffle_deepseek_fp8_moe_weights(
|
||||
torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, device=DEVICE).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, device=DEVICE).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
)
|
||||
experts = TrtLlmFp8ExpertsMonolithic(
|
||||
moe_config=_moe_config(top_k, routing_method),
|
||||
quant_config=fp8_w8a8_moe_quant_config(
|
||||
w1_scale=torch.ones(
|
||||
NUM_EXPERTS, 2 * INTERMEDIATE // block, HIDDEN // block, device=DEVICE
|
||||
),
|
||||
w2_scale=torch.ones(
|
||||
NUM_EXPERTS, HIDDEN // block, INTERMEDIATE // block, device=DEVICE
|
||||
),
|
||||
block_shape=[block, block],
|
||||
per_act_token_quant=False,
|
||||
),
|
||||
)
|
||||
|
||||
def forward(router_logits, routing_replay_out):
|
||||
num_tokens = len(router_logits)
|
||||
hidden_states = torch.randn(num_tokens, HIDDEN, device=DEVICE) * 0.1
|
||||
experts.apply(
|
||||
hidden_states=hidden_states.to(torch.float8_e4m3fn),
|
||||
w1=w13,
|
||||
w2=w2,
|
||||
router_logits=router_logits,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=NUM_EXPERTS,
|
||||
expert_map=None,
|
||||
# Per-block activation scales, (num_tokens, hidden / 128).
|
||||
a1q_scale=torch.ones(num_tokens, HIDDEN // block, device=DEVICE),
|
||||
apply_router_weight_on_input=False,
|
||||
routing_replay_out=routing_replay_out,
|
||||
**_routing_kwargs(routing_method),
|
||||
)
|
||||
|
||||
return experts, forward
|
||||
|
||||
|
||||
def _nvfp4_kernel(top_k: int, routing_method: RoutingMethodType) -> Kernel:
|
||||
from flashinfer import fp4_quantize
|
||||
|
||||
block = 16
|
||||
global_scale = torch.tensor(1.0, device=DEVICE)
|
||||
|
||||
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Packed fp4 values and their fp8 per-16 block scales, (E, M, K/16)."""
|
||||
q, scale = fp4_quantize(
|
||||
w, global_scale, block, sf_use_ue8m0=False, is_sf_swizzled_layout=False
|
||||
)
|
||||
return q, scale.view(torch.float8_e4m3fn).reshape(*w.shape[:2], -1)
|
||||
|
||||
w13, w13_scale = quantize(
|
||||
torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, device=DEVICE).bfloat16()
|
||||
)
|
||||
w2, w2_scale = quantize(
|
||||
torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, device=DEVICE).bfloat16()
|
||||
)
|
||||
ones = torch.ones(NUM_EXPERTS, device=DEVICE)
|
||||
experts = TrtLlmNvFp4ExpertsMonolithic(
|
||||
moe_config=_moe_config(top_k, routing_method),
|
||||
quant_config=FusedMoEQuantConfig.make(
|
||||
quant_dtype="nvfp4",
|
||||
per_act_token_quant=False,
|
||||
per_out_ch_quant=False,
|
||||
block_shape=None,
|
||||
w1_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
g1_alphas=ones,
|
||||
g2_alphas=ones,
|
||||
a1_gscale=global_scale,
|
||||
a2_gscale=ones,
|
||||
),
|
||||
)
|
||||
|
||||
def forward(router_logits, routing_replay_out):
|
||||
hidden_states = torch.randn(len(router_logits), HIDDEN, device=DEVICE) * 0.1
|
||||
# apply() reinterprets the packed uint8 activation scales itself.
|
||||
hidden_q, hidden_scale = fp4_quantize(
|
||||
hidden_states.bfloat16(),
|
||||
global_scale,
|
||||
block,
|
||||
sf_use_ue8m0=False,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
experts.apply(
|
||||
hidden_states=hidden_q,
|
||||
w1=w13,
|
||||
w2=w2,
|
||||
router_logits=router_logits,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=NUM_EXPERTS,
|
||||
expert_map=None,
|
||||
a1q_scale=hidden_scale,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_replay_out=routing_replay_out,
|
||||
**_routing_kwargs(routing_method),
|
||||
)
|
||||
|
||||
return experts, forward
|
||||
|
||||
|
||||
KERNELS = {"bf16": _bf16_kernel, "fp8_block": _fp8_block_kernel, "nvfp4": _nvfp4_kernel}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kernel", list(KERNELS))
|
||||
@pytest.mark.parametrize(
|
||||
"routing_method", [RoutingMethodType.Renormalize, RoutingMethodType.DeepSeekV3]
|
||||
)
|
||||
@pytest.mark.parametrize("top_k", [2, 4])
|
||||
@pytest.mark.parametrize("num_tokens", [2, 7, 16])
|
||||
def test_routing_replay_reports_the_routed_experts(
|
||||
kernel: str, routing_method: RoutingMethodType, top_k: int, num_tokens: int
|
||||
) -> None:
|
||||
torch.manual_seed(0)
|
||||
experts, forward = KERNELS[kernel](top_k, routing_method)
|
||||
assert experts.supports_routing_replay_capture()
|
||||
router_logits, planted = _planted_routing(num_tokens, top_k)
|
||||
# One spare row: the kernel must write the batch's rows and nothing else.
|
||||
routing_replay_out = torch.full(
|
||||
(num_tokens + 1, top_k), -1, dtype=torch.int16, device=DEVICE
|
||||
)
|
||||
|
||||
forward(router_logits, routing_replay_out)
|
||||
|
||||
reported = routing_replay_out[:num_tokens].long().sort(dim=-1).values
|
||||
assert torch.equal(reported, planted), (reported, planted)
|
||||
assert (routing_replay_out[num_tokens:] == -1).all()
|
||||
@@ -1,209 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test script for the token-to-expert routing simulator.
|
||||
|
||||
This script demonstrates how to use the routing simulator to test
|
||||
different routing strategies and analyze their performance, including
|
||||
integration tests with FusedMoEFactory layer.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.routing_simulator_router import (
|
||||
DistributionBasedRouting,
|
||||
RoutingSimulator,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def device():
|
||||
"""Fixture to provide the appropriate device for testing."""
|
||||
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16, 256])
|
||||
@pytest.mark.parametrize("hidden_size", [64, 1024])
|
||||
@pytest.mark.parametrize("num_experts", [16, 128])
|
||||
@pytest.mark.parametrize("top_k", [1, 4])
|
||||
def test_basic_functionality(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
top_k: int,
|
||||
device,
|
||||
):
|
||||
"""Test basic functionality of the routing simulator."""
|
||||
# Test each routing strategy
|
||||
strategies = RoutingSimulator.get_available_strategies()
|
||||
|
||||
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
|
||||
router_logits = torch.randn(num_tokens, num_experts, device=device)
|
||||
|
||||
for strategy in strategies:
|
||||
# Simulate routing
|
||||
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
strategy_name=strategy,
|
||||
top_k=top_k,
|
||||
)
|
||||
|
||||
# Check output shapes
|
||||
assert topk_weights.shape == (
|
||||
num_tokens,
|
||||
top_k,
|
||||
), f"Wrong weights shape for {strategy}"
|
||||
assert topk_ids.shape == (
|
||||
num_tokens,
|
||||
top_k,
|
||||
), f"Wrong ids shape for {strategy}"
|
||||
|
||||
# Check that expert IDs are valid
|
||||
assert topk_ids.min() >= 0, f"Invalid expert ID (negative) for {strategy}"
|
||||
assert topk_ids.max() < num_experts, (
|
||||
f"Invalid expert ID (too large) for {strategy}"
|
||||
)
|
||||
|
||||
|
||||
def test_routing_strategy_integration(monkeypatch, device):
|
||||
"""Test that the routing strategy environment variable works with
|
||||
FusedMoEFactory."""
|
||||
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
|
||||
|
||||
# Test parameters
|
||||
num_tokens = 32
|
||||
hidden_size = 16
|
||||
num_experts = 4
|
||||
top_k = 2
|
||||
|
||||
# Create test data
|
||||
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
|
||||
router_logits = torch.randn(num_tokens, num_experts, device=device)
|
||||
|
||||
# Test different routing strategies
|
||||
strategies = RoutingSimulator.get_available_strategies()
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
temp_file = tempfile.mkstemp()[1]
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
local_rank=0,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
)
|
||||
initialize_model_parallel(
|
||||
tensor_model_parallel_size=1,
|
||||
pipeline_model_parallel_size=1,
|
||||
)
|
||||
|
||||
for strategy in strategies:
|
||||
fused_moe = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=0,
|
||||
use_grouped_topk=False,
|
||||
renormalize=True,
|
||||
prefix=strategy,
|
||||
)
|
||||
|
||||
# Set environment variable
|
||||
env_name = "VLLM_MOE_ROUTING_SIMULATION_STRATEGY"
|
||||
monkeypatch.setenv(env_name, strategy)
|
||||
|
||||
# Temporarily override the envs lookup so the router factory
|
||||
# reads the monkeypatched value instead of the module-load-time
|
||||
# default. Use monkeypatch.setitem so the original lambda is
|
||||
# restored automatically at teardown.
|
||||
monkeypatch.setitem(
|
||||
envs.environment_variables,
|
||||
env_name,
|
||||
lambda s=strategy: s,
|
||||
)
|
||||
|
||||
# Test the select_experts method
|
||||
topk_weights, topk_ids = fused_moe.router.select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
)
|
||||
|
||||
# Verify output shapes
|
||||
assert topk_weights.shape == (num_tokens, top_k), (
|
||||
f"Wrong weights shape for {strategy}"
|
||||
)
|
||||
assert topk_ids.shape == (num_tokens, top_k), (
|
||||
f"Wrong ids shape for {strategy}"
|
||||
)
|
||||
|
||||
# Verify expert IDs are valid
|
||||
assert topk_ids.min() >= 0, f"Invalid expert ID (negative) for {strategy}"
|
||||
assert topk_ids.max() < num_experts, (
|
||||
f"Invalid expert ID (too large) for {strategy}"
|
||||
)
|
||||
|
||||
|
||||
def test_distribution_based_routing_with_custom_strategy():
|
||||
"""Test registering and using DistributionBasedRouting with custom
|
||||
parameters."""
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# Register custom distribution-based strategy
|
||||
custom_strategy = DistributionBasedRouting(distribution="normal", mean=2.0, std=0.5)
|
||||
RoutingSimulator.register_strategy("custom_normal", custom_strategy)
|
||||
|
||||
# Test data
|
||||
num_tokens = 60
|
||||
hidden_size = 48
|
||||
num_experts = 6
|
||||
top_k = 3
|
||||
|
||||
hidden_states = torch.randn(num_tokens, hidden_size, device=device)
|
||||
router_logits = torch.randn(num_tokens, num_experts, device=device)
|
||||
|
||||
# Use the custom strategy
|
||||
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
strategy_name="custom_normal",
|
||||
top_k=top_k,
|
||||
)
|
||||
|
||||
# Check output shapes
|
||||
assert topk_weights.shape == (num_tokens, top_k)
|
||||
assert topk_ids.shape == (num_tokens, top_k)
|
||||
|
||||
# Check that expert IDs are valid
|
||||
assert topk_ids.min() >= 0
|
||||
assert topk_ids.max() < num_experts
|
||||
|
||||
|
||||
def test_instance_compatibility():
|
||||
"""Test that static methods work correctly."""
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# Test static method directly
|
||||
hidden_states = torch.randn(10, 8, device=device)
|
||||
router_logits = torch.randn(10, 4, device=device)
|
||||
|
||||
topk_weights, topk_ids = RoutingSimulator.simulate_routing(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
strategy_name="uniform_random",
|
||||
top_k=2,
|
||||
)
|
||||
|
||||
assert topk_weights.shape == (10, 2)
|
||||
assert topk_ids.shape == (10, 2)
|
||||
@@ -1,228 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for FusedMoEFactory with routed_input_transform.
|
||||
|
||||
Verifies that applying routed_input_transform inside FusedMoEFactory
|
||||
produces the same results as applying the transform manually outside.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer, set_random_seed
|
||||
|
||||
|
||||
class SimpleLinear(nn.Module):
|
||||
"""A simple linear transform mimicking latent projection in latent MoE."""
|
||||
|
||||
def __init__(self, in_features: int, out_features: int, dtype: torch.dtype):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(
|
||||
torch.randn(out_features, in_features, device="cuda", dtype=dtype) / 10
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return nn.functional.linear(x, self.weight)
|
||||
|
||||
|
||||
class SimpleSharedExperts(nn.Module):
|
||||
"""A simple 2-layer MLP mimicking shared experts."""
|
||||
|
||||
def __init__(self, hidden_size: int, intermediate_size: int, dtype: torch.dtype):
|
||||
super().__init__()
|
||||
self.up = nn.Linear(
|
||||
hidden_size, intermediate_size * 2, bias=False, device="cuda", dtype=dtype
|
||||
)
|
||||
self.down = nn.Linear(
|
||||
intermediate_size, hidden_size, bias=False, device="cuda", dtype=dtype
|
||||
)
|
||||
with torch.no_grad():
|
||||
self.up.weight.div_(10)
|
||||
self.down.weight.div_(10)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
gate_up = self.up(x)
|
||||
gate, up = gate_up.chunk(2, dim=-1)
|
||||
return self.down(nn.functional.silu(gate) * up)
|
||||
|
||||
|
||||
def _assert_close(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
atol: float,
|
||||
rtol: float,
|
||||
label: str,
|
||||
) -> None:
|
||||
"""assert_close that prints diff diagnostics on both success and failure."""
|
||||
actual_nans = int(actual.isnan().sum().item())
|
||||
expected_nans = int(expected.isnan().sum().item())
|
||||
actual_zeros = int((actual == 0).sum().item())
|
||||
expected_zeros = int((expected == 0).sum().item())
|
||||
n_total = actual.numel()
|
||||
|
||||
diff = (actual - expected).abs()
|
||||
max_diff = diff.max().item()
|
||||
mean_diff = diff.mean().item()
|
||||
n_exceed = int((diff > atol).sum().item())
|
||||
pct_exceed = n_exceed / n_total * 100
|
||||
|
||||
print(
|
||||
f"[{label}] "
|
||||
f"shape={list(actual.shape)}, "
|
||||
f"max_diff={max_diff:.6e}, "
|
||||
f"mean_diff={mean_diff:.6e}, "
|
||||
f"exceed_atol({atol})={n_exceed}/{n_total} ({pct_exceed:.2f}%), "
|
||||
f"actual=[{actual.min().item():.4f}, {actual.max().item():.4f}], "
|
||||
f"expected=[{expected.min().item():.4f}, {expected.max().item():.4f}], "
|
||||
f"nan(actual/expected)={actual_nans}/{expected_nans}, "
|
||||
f"zeros(actual/expected)={actual_zeros}/{expected_zeros}"
|
||||
)
|
||||
|
||||
assert actual_nans == 0, (
|
||||
f"{label}: actual has {actual_nans}/{n_total} NaN values "
|
||||
f"(expected has {expected_nans}). "
|
||||
f"This indicates a kernel bug, not a precision issue."
|
||||
)
|
||||
assert expected_nans == 0, (
|
||||
f"{label}: expected has {expected_nans}/{n_total} NaN values. "
|
||||
f"This indicates a kernel bug, not a precision issue."
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual,
|
||||
expected,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
msg=(
|
||||
f"{label}: max_diff={max_diff:.6e}, mean_diff={mean_diff:.6e}, "
|
||||
f"exceed_atol({atol})={n_exceed}/{n_total} ({pct_exceed:.2f}%)"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_cuda():
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32])
|
||||
@pytest.mark.parametrize("hidden_size,latent_size", [(256, 128), (128, 64)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
is_torch_equal_or_newer("2.10.0"),
|
||||
reason="Test fails with PyTorch 2.10.0 see: https://github.com/vllm-project/vllm/issues/33995",
|
||||
)
|
||||
def test_routed_input_transform_inside_vs_outside(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
latent_size: int,
|
||||
dtype: torch.dtype,
|
||||
use_rocm_aiter: bool,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Compare FusedMoEFactory with transform inside vs manually applying outside.
|
||||
Method A (inside): FusedMoEFactory with routed_input_transform
|
||||
Method B (outside): Manually transform, then FusedMoEFactory without transform
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_rocm_aiter else "0")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1" if use_rocm_aiter else "0")
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
set_random_seed(42)
|
||||
|
||||
num_experts = 8
|
||||
top_k = 2
|
||||
intermediate_size = hidden_size * 2
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.compilation_config.static_forward_context = dict()
|
||||
|
||||
shared_experts = SimpleSharedExperts(hidden_size, intermediate_size, dtype)
|
||||
routed_transform = SimpleLinear(hidden_size, latent_size, dtype)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
# Method A: FusedMoEFactory WITH routed_input_transform
|
||||
moe_with_transform = FusedMoEFactory(
|
||||
shared_experts=shared_experts,
|
||||
routed_input_transform=routed_transform,
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=latent_size,
|
||||
intermediate_size=intermediate_size,
|
||||
renormalize=True,
|
||||
params_dtype=dtype,
|
||||
tp_size=1,
|
||||
dp_size=1,
|
||||
pcp_size=1,
|
||||
prefix="moe_with_transform",
|
||||
)
|
||||
|
||||
# Method B: FusedMoEFactory WITHOUT routed_input_transform
|
||||
# Note: shared_experts=None because when transform is done outside,
|
||||
moe_without_transform = FusedMoEFactory(
|
||||
shared_experts=None,
|
||||
routed_input_transform=None,
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=latent_size,
|
||||
intermediate_size=intermediate_size,
|
||||
renormalize=True,
|
||||
params_dtype=dtype,
|
||||
tp_size=1,
|
||||
dp_size=1,
|
||||
pcp_size=1,
|
||||
prefix="moe_without_transform",
|
||||
)
|
||||
|
||||
# Weights are created via torch.empty (uninitialized).
|
||||
# Initialize with seeded random values for reproducibility.
|
||||
with torch.no_grad():
|
||||
moe_with_transform.w13_weight.normal_()
|
||||
moe_with_transform.w13_weight.div_(10)
|
||||
moe_with_transform.w2_weight.normal_()
|
||||
moe_with_transform.w2_weight.div_(10)
|
||||
moe_without_transform.w13_weight.copy_(moe_with_transform.w13_weight)
|
||||
moe_without_transform.w2_weight.copy_(moe_with_transform.w2_weight)
|
||||
|
||||
moe_with_transform.quant_method.process_weights_after_loading(
|
||||
moe_with_transform
|
||||
)
|
||||
moe_without_transform.quant_method.process_weights_after_loading(
|
||||
moe_without_transform
|
||||
)
|
||||
|
||||
hidden_states = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype)
|
||||
router_logits = torch.randn(num_tokens, num_experts, device="cuda", dtype=dtype)
|
||||
|
||||
with set_forward_context(None, vllm_config, num_tokens=num_tokens):
|
||||
# Method A: combined output (shared + routed)
|
||||
combined_A = moe_with_transform(hidden_states, router_logits)
|
||||
|
||||
# Method B: manually transform, get routed output, add shared
|
||||
transformed_hidden = routed_transform(hidden_states)
|
||||
routed_out_B = moe_without_transform(transformed_hidden, router_logits)
|
||||
shared_out_B = shared_experts(hidden_states)
|
||||
combined_B = shared_out_B + routed_out_B
|
||||
|
||||
torch.testing.assert_close(
|
||||
combined_A,
|
||||
combined_B,
|
||||
atol=1e-3,
|
||||
rtol=1e-3,
|
||||
msg="Combined output should match: transform inside vs outside",
|
||||
)
|
||||
@@ -1,318 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import (
|
||||
persistent_masked_m_silu_mul_quant,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
DeepGemmQuantScaleFMT,
|
||||
has_deep_gemm,
|
||||
transform_sf_into_required_layout,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="persistent_masked_m_silu_mul_quant requires a CUDA-alike or XPU device.",
|
||||
)
|
||||
|
||||
CASES = [
|
||||
(1, 1, 128, fp8_dtype),
|
||||
(1, 4, 128 * 1, fp8_dtype),
|
||||
(2, 4, 128 * 2, fp8_dtype),
|
||||
(1, 4, 128 * 3, fp8_dtype),
|
||||
(8, 16, 128 * 4, fp8_dtype),
|
||||
(8, 16, 128 * 5, fp8_dtype),
|
||||
(8, 16, 128 * 6, fp8_dtype),
|
||||
(8, 16, 128 * 7, fp8_dtype),
|
||||
(8, 16, 128 * 8, fp8_dtype),
|
||||
(8, 16, 128 * 9, fp8_dtype),
|
||||
(8, 64, 7168, fp8_dtype),
|
||||
(8, 128, 128 * 33, fp8_dtype),
|
||||
(1, 4, 128 * 10, fp8_dtype),
|
||||
(8, 128, 7168, fp8_dtype),
|
||||
(8, 512, 7168, fp8_dtype),
|
||||
(8, 1024, 7168, fp8_dtype),
|
||||
(17, 31, 768, fp8_dtype),
|
||||
(32, 64, 256, fp8_dtype),
|
||||
(256, 8, 7168, fp8_dtype),
|
||||
(256, 32, 7168, fp8_dtype),
|
||||
(256, 64, 7168, fp8_dtype),
|
||||
# Only add a few fnuz tests to help with long CI times.
|
||||
(8, 512, 7168, torch.float8_e4m3fnuz),
|
||||
(8, 1024, 7168, torch.float8_e4m3fnuz),
|
||||
]
|
||||
|
||||
|
||||
def as_uint8(x) -> torch.Tensor:
|
||||
return (
|
||||
torch.empty(x.shape, dtype=x.dtype, device=x.device).copy_(x).view(torch.uint8)
|
||||
)
|
||||
|
||||
|
||||
def silu(x: torch.Tensor) -> torch.Tensor:
|
||||
x_f32 = x.to(torch.float32)
|
||||
act_f32 = x_f32 / (1.0 + torch.exp(-x_f32))
|
||||
if current_platform.is_cuda():
|
||||
# C++ kernel returns bf16
|
||||
return act_f32.to(torch.bfloat16)
|
||||
# Triton fallback stays in f32
|
||||
return act_f32
|
||||
|
||||
|
||||
def do_quant(x: torch.Tensor, group_size: int, ceil_ue8m0: bool):
|
||||
fp8_min_val, fp8_max_val = get_fp8_min_max()
|
||||
eps_bf16 = torch.tensor([1e-10], device=x.device, dtype=torch.bfloat16)
|
||||
one_bf16 = torch.tensor([1.0], device=x.device, dtype=torch.bfloat16)
|
||||
fp8_max_bf16 = torch.tensor([fp8_max_val], device=x.device, dtype=torch.bfloat16)
|
||||
fp8_min_bf16 = torch.tensor([fp8_min_val], device=x.device, dtype=torch.bfloat16)
|
||||
fp8_max_inv = one_bf16 / fp8_max_bf16
|
||||
assert fp8_max_inv.dtype == torch.bfloat16
|
||||
|
||||
assert x.size(-1) % group_size == 0
|
||||
num_groups = x.numel() // group_size
|
||||
x_og_shape = x.shape
|
||||
|
||||
if current_platform.is_cuda():
|
||||
# C++ kernel computes entirely in bf16
|
||||
x = x.to(torch.bfloat16)
|
||||
x = x.view((-1, group_size))
|
||||
amax = x.abs().amax(dim=1).clamp(min=eps_bf16)
|
||||
assert amax.dtype == torch.bfloat16
|
||||
s = amax * fp8_max_inv
|
||||
|
||||
if ceil_ue8m0:
|
||||
s = torch.exp2(
|
||||
torch.ceil(torch.log2(s).to(torch.bfloat16)).to(torch.bfloat16)
|
||||
).to(torch.bfloat16)
|
||||
|
||||
inv_s = one_bf16 / s
|
||||
inv_s = inv_s.view((num_groups, 1))
|
||||
xq = torch.clamp(
|
||||
x * inv_s, min=fp8_min_bf16.item(), max=fp8_max_bf16.item()
|
||||
).to(fp8_dtype)
|
||||
else:
|
||||
# Triton fallback computes in f32. Use multiply-by-reciprocal
|
||||
# to match Triton's constexpr evaluation of 1.0/fp8_max.
|
||||
fp8_min_f, fp8_max_f = get_fp8_min_max()
|
||||
|
||||
x = x.to(torch.float32).view((-1, group_size))
|
||||
amax = x.abs().amax(dim=1).clamp(min=1e-10)
|
||||
s = amax * (1.0 / fp8_max_f)
|
||||
if ceil_ue8m0:
|
||||
s = torch.exp2(torch.ceil(torch.log2(s)))
|
||||
inv_s = (1.0 / s).view((num_groups, 1))
|
||||
xq = torch.clamp(x * inv_s, min=fp8_min_f, max=fp8_max_f).to(fp8_dtype)
|
||||
|
||||
xq = xq.view(x_og_shape)
|
||||
xs = s.view((-1, xq.size(-1) // group_size))
|
||||
return xq, xs
|
||||
|
||||
|
||||
def silu_mul_quant(
|
||||
gate: torch.Tensor, up: torch.Tensor, group_size: int, ceil_ue8m0: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert gate.size(-1) % group_size == 0
|
||||
assert up.size(-1) % group_size == 0
|
||||
|
||||
assert gate.dtype == torch.bfloat16
|
||||
assert up.dtype == torch.bfloat16
|
||||
|
||||
act = silu(gate)
|
||||
|
||||
# act & mul
|
||||
a_m = act * up
|
||||
|
||||
q, s = do_quant(a_m, group_size, ceil_ue8m0)
|
||||
return q, s
|
||||
|
||||
|
||||
def pack_scales(x: torch.Tensor, tokens_per_expert: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack float32 scales into a int32 tensor."""
|
||||
assert x.dtype == torch.float32
|
||||
E, T, G = x.size()
|
||||
|
||||
# Add i32_padding here so we can view it as a i32 tensor later on.
|
||||
i32_padding = round_up(G, 4) - G
|
||||
ref_s_i8 = torch.empty((E, T, G + i32_padding), dtype=torch.uint8, device=DEVICE)
|
||||
for e in range(E):
|
||||
nt = tokens_per_expert[e].item()
|
||||
ref_s_i8[e, :nt, :G] = x[e, :nt].view(torch.int32) >> 23
|
||||
|
||||
ref_s_i32 = ref_s_i8.view(torch.int32)
|
||||
|
||||
return ref_s_i32
|
||||
|
||||
|
||||
def ref_with_scale_fmt(
|
||||
E: int,
|
||||
T: int,
|
||||
H: int,
|
||||
group_size: int,
|
||||
tokens_per_expert: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
up: torch.Tensor,
|
||||
scale_fmt: DeepGemmQuantScaleFMT,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The precision types of the operations triggered by this function
|
||||
match closely with the kernel implementation so we compare more
|
||||
accurately.
|
||||
"""
|
||||
scale_dtype = (
|
||||
torch.int32 if scale_fmt == DeepGemmQuantScaleFMT.UE8M0 else torch.float32
|
||||
)
|
||||
ceil_ue8m0 = scale_fmt in [
|
||||
DeepGemmQuantScaleFMT.UE8M0,
|
||||
DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0,
|
||||
]
|
||||
|
||||
ref_q = torch.empty((E, T, H), dtype=fp8_dtype, device=DEVICE)
|
||||
ref_s_f32 = torch.empty(
|
||||
(E, T, cdiv(H, group_size)), dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
|
||||
for e in range(E):
|
||||
nt = tokens_per_expert[e].item()
|
||||
if nt == 0:
|
||||
continue
|
||||
ref_q[e, :nt], ref_s_f32[e, :nt] = silu_mul_quant(
|
||||
gate[e, :nt], up[e, :nt], group_size, ceil_ue8m0=ceil_ue8m0
|
||||
)
|
||||
|
||||
if scale_dtype == torch.float32:
|
||||
return ref_q, ref_s_f32
|
||||
|
||||
assert scale_dtype == torch.int32
|
||||
return ref_q, pack_scales(ref_s_f32, tokens_per_expert)
|
||||
|
||||
|
||||
def token_random(E, T, H2, tokens_per_expert):
|
||||
"""Initialize each token in a random range so we test a range of
|
||||
scale values.
|
||||
"""
|
||||
y = torch.empty((E, T, H2), dtype=torch.bfloat16, device=DEVICE)
|
||||
for e in range(E):
|
||||
for t in range(tokens_per_expert[e].item()):
|
||||
exp = random.choice(range(1, 20))
|
||||
y[e, t].uniform_(-(2**exp), 2**exp)
|
||||
return y
|
||||
|
||||
|
||||
@pytest.mark.parametrize("E,T,H,fp8_type", CASES)
|
||||
@torch.inference_mode()
|
||||
def test_silu_mul_fp8_quant_deep_gemm(E: int, T: int, H: int, fp8_type: torch.dtype):
|
||||
group_size = 128
|
||||
set_random_seed(42)
|
||||
|
||||
tokens_per_expert = torch.randint(
|
||||
low=0,
|
||||
high=T,
|
||||
size=(E,),
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
# Input tensor of shape (E, T, 2*H)
|
||||
y = token_random(E, T, 2 * H, tokens_per_expert)
|
||||
|
||||
gate = y[..., :H].to(torch.bfloat16)
|
||||
up = y[..., H:].to(torch.bfloat16)
|
||||
|
||||
scale_fmts = [
|
||||
DeepGemmQuantScaleFMT.FLOAT32,
|
||||
DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0,
|
||||
DeepGemmQuantScaleFMT.UE8M0,
|
||||
]
|
||||
|
||||
# Run the SiLU V2 kernel
|
||||
for scale_fmt in scale_fmts:
|
||||
y_q, y_s = persistent_masked_m_silu_mul_quant(
|
||||
y,
|
||||
tokens_per_expert,
|
||||
group_size=group_size,
|
||||
quant_scale_fmt=scale_fmt,
|
||||
)
|
||||
|
||||
ref_y_q, ref_y_s = ref_with_scale_fmt(
|
||||
E, T, H, group_size, tokens_per_expert, gate, up, scale_fmt=scale_fmt
|
||||
)
|
||||
|
||||
# deepgemm scales transform
|
||||
dg_scales = None
|
||||
if (
|
||||
has_deep_gemm()
|
||||
and current_platform.has_device_capability(100)
|
||||
and scale_fmt == DeepGemmQuantScaleFMT.UE8M0
|
||||
):
|
||||
_q, _s = ref_with_scale_fmt(
|
||||
E,
|
||||
T,
|
||||
H,
|
||||
group_size,
|
||||
tokens_per_expert,
|
||||
gate,
|
||||
up,
|
||||
scale_fmt=DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0,
|
||||
)
|
||||
dg_scales = transform_sf_into_required_layout(
|
||||
sf=_s,
|
||||
mn=_q.size(1),
|
||||
k=_q.size(2),
|
||||
recipe=(1, 128, 128),
|
||||
num_groups=_q.size(0),
|
||||
is_sfa=True,
|
||||
)
|
||||
|
||||
expected_scale_dtype = (
|
||||
torch.int32 if scale_fmt == DeepGemmQuantScaleFMT.UE8M0 else torch.float32
|
||||
)
|
||||
assert y_s.dtype == expected_scale_dtype
|
||||
assert ref_y_s.dtype == expected_scale_dtype
|
||||
|
||||
for e in range(E):
|
||||
nt = tokens_per_expert[e].item()
|
||||
|
||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
||||
# ROCm and XPU both run the Triton fallback kernel, whose f32
|
||||
# math intrinsics (tl.exp) may differ from PyTorch's torch.exp
|
||||
# by 1 ULP. At FP8 quantization boundaries this can flip one
|
||||
# representable value. Allow 1 FP8 quantum of tolerance.
|
||||
torch.testing.assert_close(
|
||||
y_q[e, :nt].to(torch.float32),
|
||||
ref_y_q[e, :nt].to(torch.float32),
|
||||
atol=32.0,
|
||||
rtol=0.2,
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
y_q[e, :nt].to(torch.float32),
|
||||
ref_y_q[e, :nt].to(torch.float32),
|
||||
)
|
||||
|
||||
if scale_fmt == DeepGemmQuantScaleFMT.UE8M0:
|
||||
G = H // group_size
|
||||
y_s_sliced = as_uint8(y_s[e])
|
||||
ref_s_sliced = as_uint8(ref_y_s[e])
|
||||
torch.testing.assert_close(y_s_sliced[:nt, :G], ref_s_sliced[:nt, :G])
|
||||
if dg_scales is not None:
|
||||
dg_sliced = as_uint8(dg_scales[e])
|
||||
torch.testing.assert_close(y_s_sliced[:nt, :G], dg_sliced[:nt, :G])
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
y_s[e, :nt],
|
||||
ref_y_s[e, :nt],
|
||||
)
|
||||
@@ -1,230 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_per_token_group_quant_fp8_colmajor,
|
||||
silu_mul_per_token_group_quant_fp8_colmajor,
|
||||
silu_mul_quant_fp8_packed_triton,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_e8m0_used
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
FLOAT8_DTYPE = torch.float8_e4m3fn
|
||||
GROUP_SIZE = 128
|
||||
|
||||
|
||||
def reference_quant(x: torch.Tensor, use_ue8m0: bool):
|
||||
"""Reference triton quant kernel from,
|
||||
vllm.model_executor.layers.quantization.utils.fp8_utils
|
||||
"""
|
||||
x_q = torch.empty_like(x, device=x.device, dtype=FLOAT8_DTYPE)
|
||||
|
||||
# Allocate the scale tensor in column-major format.
|
||||
shape = (x.shape[-1] // GROUP_SIZE,) + x.shape[:-1]
|
||||
x_s = torch.empty(shape, device=x.device, dtype=torch.float32).permute(-1, -2)
|
||||
|
||||
M = x.numel() // GROUP_SIZE
|
||||
N = GROUP_SIZE
|
||||
BLOCK = triton.next_power_of_2(N)
|
||||
# heuristics for number of warps
|
||||
num_warps = min(max(BLOCK // 256, 1), 8)
|
||||
num_stages = 1
|
||||
|
||||
finfo = torch.finfo(FLOAT8_DTYPE)
|
||||
fp8_min = finfo.min
|
||||
fp8_max = finfo.max
|
||||
|
||||
_per_token_group_quant_fp8_colmajor[(M,)](
|
||||
x,
|
||||
x_q,
|
||||
x_s,
|
||||
GROUP_SIZE,
|
||||
x.shape[1],
|
||||
x.stride(0),
|
||||
x_s.stride(1),
|
||||
eps=1e-10,
|
||||
fp8_min=fp8_min,
|
||||
fp8_max=fp8_max,
|
||||
use_ue8m0=use_ue8m0,
|
||||
BLOCK=BLOCK,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
return x_q, x_s
|
||||
|
||||
|
||||
def reference(x: torch.Tensor, use_ue8m0: bool) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
T, N = x.size()
|
||||
ref_act_out = torch.empty((T, N // 2), dtype=torch.bfloat16, device=DEVICE)
|
||||
torch.ops._C.silu_and_mul(ref_act_out, x)
|
||||
return reference_quant(ref_act_out, use_ue8m0)
|
||||
|
||||
|
||||
def reference_with_clamp(
|
||||
x: torch.Tensor, use_ue8m0: bool, clamp_limit: float
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pre-clamp inputs (gate from above, up symmetric) at the input dtype to
|
||||
match the C++ compute() template, then run the standard silu_and_mul +
|
||||
quant reference."""
|
||||
N_2 = x.size(1) // 2
|
||||
dtype = x.dtype
|
||||
gate = x[..., :N_2].to(torch.float32).clamp(max=clamp_limit).to(dtype)
|
||||
up = (
|
||||
x[..., N_2:]
|
||||
.to(torch.float32)
|
||||
.clamp(min=-clamp_limit, max=clamp_limit)
|
||||
.to(dtype)
|
||||
)
|
||||
return reference(torch.cat([gate, up], dim=-1), use_ue8m0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("T", [128, 256, 512])
|
||||
@pytest.mark.parametrize("N", [128 * 2, 256 * 2, 768 * 2, 2048 * 2, 7168 * 2])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="ROCm does not support DeepGemm.",
|
||||
)
|
||||
def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
|
||||
set_random_seed(42)
|
||||
|
||||
input = torch.rand((T, N), dtype=torch.bfloat16, device=DEVICE)
|
||||
|
||||
use_ue8m0 = is_deep_gemm_e8m0_used()
|
||||
|
||||
# Test
|
||||
output, output_scales = silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
input, use_ue8m0=use_ue8m0
|
||||
)
|
||||
|
||||
# Reference
|
||||
ref_output, ref_output_scales = reference(input, use_ue8m0)
|
||||
|
||||
torch.testing.assert_close(output.to(torch.float32), ref_output.to(torch.float32))
|
||||
torch.testing.assert_close(output_scales, ref_output_scales)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"counts", [[0, 1, 127, 128, 129, 0, 257, 13], [128] * 8, [0] * 8]
|
||||
)
|
||||
@pytest.mark.parametrize("group_size", [32, 128])
|
||||
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
|
||||
@pytest.mark.parametrize("hidden_size", [256, 2048])
|
||||
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm graph test")
|
||||
def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
counts: list[int],
|
||||
group_size: int,
|
||||
scale_format: str,
|
||||
hidden_size: int,
|
||||
clamp_limit: float | None,
|
||||
):
|
||||
"""Live rows match dense quantization; gaps stay untouched as counts change."""
|
||||
alignment = 128
|
||||
capacity = sum(triton.cdiv(n, alignment) * alignment for n in counts)
|
||||
# Exercise unused capacity as well as the gaps between experts.
|
||||
capacity = max(capacity, alignment)
|
||||
if counts != [128] * 8:
|
||||
capacity += alignment
|
||||
x = torch.randn((capacity, 2 * hidden_size), device=DEVICE, dtype=torch.bfloat16)
|
||||
x *= 16
|
||||
clean_x = x.clone()
|
||||
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
|
||||
output_dtype = (
|
||||
FLOAT8_DTYPE if scale_format == "packed" else current_platform.fp8_dtype()
|
||||
)
|
||||
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=output_dtype)
|
||||
|
||||
def set_counts(values):
|
||||
positions = []
|
||||
mask = torch.zeros(capacity, device=DEVICE, dtype=torch.bool)
|
||||
start = 0
|
||||
for count in values:
|
||||
mask[start : start + count] = True
|
||||
positions.append(start + count)
|
||||
start += triton.cdiv(count, alignment) * alignment
|
||||
ends.copy_(torch.tensor(positions, device=DEVICE, dtype=torch.int32))
|
||||
return mask
|
||||
|
||||
def quantize(x, output=None, **kwargs):
|
||||
if scale_format == "packed":
|
||||
return silu_mul_quant_fp8_packed_triton(
|
||||
x,
|
||||
group_size=group_size,
|
||||
output_q=output,
|
||||
clamp_limit=clamp_limit,
|
||||
**kwargs,
|
||||
)
|
||||
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
x,
|
||||
group_size=group_size,
|
||||
output=output,
|
||||
clamp_limit=clamp_limit,
|
||||
use_ue8m0=scale_format == "ceil_ue8m0",
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def invoke():
|
||||
return quantize(
|
||||
x,
|
||||
output=output,
|
||||
expert_ends=ends,
|
||||
expert_alignment=alignment,
|
||||
)
|
||||
|
||||
set_counts(counts)
|
||||
invoke()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual_q, actual_s = invoke()
|
||||
|
||||
for values in (counts, counts[::-1]):
|
||||
live = set_counts(values)
|
||||
x.copy_(clean_x)
|
||||
x[~live] = float("nan")
|
||||
output.view(torch.uint8).fill_(91)
|
||||
graph.replay()
|
||||
reference_q, reference_s = quantize(x)
|
||||
assert actual_q.dtype == reference_q.dtype
|
||||
torch.testing.assert_close(
|
||||
actual_q.view(torch.uint8)[live],
|
||||
reference_q.view(torch.uint8)[live],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
torch.testing.assert_close(actual_s[live], reference_s[live], rtol=0, atol=0)
|
||||
assert torch.all(actual_q.view(torch.uint8)[~live] == 91)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("T", [128, 256, 512])
|
||||
@pytest.mark.parametrize("N", [128 * 2, 256 * 2, 768 * 2, 2048 * 2, 7168 * 2])
|
||||
@pytest.mark.parametrize("clamp_limit", [7.0, 10.0])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="ROCm does not support DeepGemm.",
|
||||
)
|
||||
def test_silu_mul_fp8_quant_deep_gemm_clamp(T: int, N: int, clamp_limit: float):
|
||||
set_random_seed(42)
|
||||
|
||||
# Use a wide distribution so values routinely exceed both clamp limits and
|
||||
# the clamp branch is actually exercised (uniform [0, 1) inputs would never
|
||||
# trigger it).
|
||||
input = torch.randn((T, N), dtype=torch.bfloat16, device=DEVICE) * 8.0
|
||||
|
||||
use_ue8m0 = is_deep_gemm_e8m0_used()
|
||||
|
||||
# Test
|
||||
output, output_scales = silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
input, use_ue8m0=use_ue8m0, clamp_limit=clamp_limit
|
||||
)
|
||||
|
||||
# Reference
|
||||
ref_output, ref_output_scales = reference_with_clamp(input, use_ue8m0, clamp_limit)
|
||||
|
||||
torch.testing.assert_close(output.to(torch.float32), ref_output.to(torch.float32))
|
||||
torch.testing.assert_close(output_scales, ref_output_scales)
|
||||
@@ -1,224 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the persistent cp.async-pipelined SITU + block-FP8
|
||||
quant kernel (``situ_and_mul_quant``).
|
||||
|
||||
Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference
|
||||
activation + block-FP8 quant, drive random per-row scale magnitudes, exercise
|
||||
the masked contiguous layout, and compare per valid row.
|
||||
|
||||
The kernel computes SITU in fp32 and converts to fp8 with hardware rounding,
|
||||
which torch cannot reproduce bit-exactly. It is therefore validated against the
|
||||
*ideal* FP8 quantizer of the exact fp64 activation: the difference must stay
|
||||
within the unavoidable FP8 block-quant error (within 1 fp8 code for ~all
|
||||
elements, never off by more than 2).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import situ_and_mul_quant
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
DEVICE = current_platform.device_type
|
||||
FP8_MAX = torch.finfo(fp8_dtype).max
|
||||
|
||||
# The kernel uses fp8 e4m3 conversion, so it is CUDA-only and needs SM89+
|
||||
# (Ada/Hopper). There is no Triton fallback.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not current_platform.has_device_capability(89),
|
||||
reason="situ_and_mul_quant persistent kernel requires CUDA SM89+ (fp8 e4m3).",
|
||||
)
|
||||
|
||||
# Kimi-K3 baked SITU params. Only (d=3072, beta=4.0, linear_beta=25.0,
|
||||
# group_size=128, fp16/bf16) routes to the persistent cp.async-pipelined kernel;
|
||||
# anything else falls back to the scalar group kernel.
|
||||
SITU_BETA = 4.0
|
||||
SITU_LINEAR_BETA = 25.0
|
||||
SITU_D = 3072
|
||||
GROUP_SIZE = 128
|
||||
|
||||
# (num_tokens, num_valid_tokens). None means all rows are valid. 2048/4096 rows
|
||||
# span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8).
|
||||
CASES = [
|
||||
(1, None),
|
||||
(17, None),
|
||||
(128, None),
|
||||
(2048, None),
|
||||
(4096, None),
|
||||
(2048, 1500), # DeepEP v2 contiguous padding
|
||||
(2048, 1),
|
||||
(4096, 4095),
|
||||
(512, 0), # whole tensor is padding
|
||||
]
|
||||
|
||||
|
||||
def token_random(num_tokens: int, twod: int, dtype: torch.dtype) -> torch.Tensor:
|
||||
"""Per-row random exponent so groups span many scale magnitudes."""
|
||||
base = torch.randn(num_tokens, twod, dtype=torch.float32, device=DEVICE)
|
||||
exps = torch.randint(1, 13, (num_tokens, 1), device=DEVICE).float()
|
||||
return (base * (2.0**exps)).to(dtype)
|
||||
|
||||
|
||||
def ref_situ_block_fp8_quant(
|
||||
inp: torch.Tensor, beta: float, linear_beta: float, group_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Ideal FP8 block-quant of the exact fp64 SITU activation.
|
||||
|
||||
The kernel rounds the activation to ``scalar_t`` before quantizing (it stores
|
||||
``(float)(scalar_t)situ(...)``), so the reference does the same before taking
|
||||
the per-group absmax.
|
||||
"""
|
||||
d = inp.shape[-1] // 2
|
||||
gate, up = inp.double().chunk(2, dim=-1)
|
||||
gate_out = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
|
||||
up_out = linear_beta * torch.tanh(up / linear_beta) if linear_beta > 0 else up
|
||||
act = (gate_out * up_out).to(inp.dtype).float()
|
||||
|
||||
m = act.shape[0]
|
||||
ng = d // group_size
|
||||
a = act.view(m, ng, group_size)
|
||||
scale = a.abs().amax(dim=2).clamp_min(1e-30) / FP8_MAX
|
||||
q = (a / scale.unsqueeze(2)).clamp(-FP8_MAX, FP8_MAX).to(fp8_dtype)
|
||||
return q.view(m, d), scale
|
||||
|
||||
|
||||
def fp8_code_dist(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
||||
"""Signed-magnitude fp8 code distance (valid for e4m3fn and e4m3fnuz)."""
|
||||
|
||||
def ordered(x: torch.Tensor) -> torch.Tensor:
|
||||
x = x.view(torch.uint8).to(torch.int32)
|
||||
return torch.where((x >> 7) == 1, -(x & 0x7F), x & 0x7F)
|
||||
|
||||
return (ordered(a) - ordered(b)).abs()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens,valid", CASES)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
|
||||
@torch.inference_mode()
|
||||
def test_situ_and_mul_quant_pipelined(
|
||||
num_tokens: int, valid: int | None, dtype: torch.dtype
|
||||
) -> None:
|
||||
set_random_seed(42)
|
||||
d = SITU_D
|
||||
inp = token_random(num_tokens, 2 * d, dtype)
|
||||
|
||||
# Sentinels: padding rows must keep their bytes (out) and get scale 1.0.
|
||||
out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype)
|
||||
scale = torch.full(
|
||||
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
out_sentinel = out.clone()
|
||||
num_valid_tokens = (
|
||||
None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE)
|
||||
)
|
||||
|
||||
situ_and_mul_quant(
|
||||
out,
|
||||
scale,
|
||||
inp,
|
||||
beta=SITU_BETA,
|
||||
linear_beta=SITU_LINEAR_BETA,
|
||||
group_size=GROUP_SIZE,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
)
|
||||
|
||||
# Padding contract: skipped rows keep their bytes; their scale becomes 1.0.
|
||||
if valid is not None and valid < num_tokens:
|
||||
assert torch.equal(scale[valid:], torch.ones_like(scale[valid:]))
|
||||
assert torch.equal(
|
||||
out[valid:].view(torch.uint8), out_sentinel[valid:].view(torch.uint8)
|
||||
)
|
||||
|
||||
rows = num_tokens if valid is None else valid
|
||||
if rows == 0:
|
||||
return
|
||||
|
||||
ref_q, ref_s = ref_situ_block_fp8_quant(
|
||||
inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE
|
||||
)
|
||||
|
||||
assert torch.isfinite(scale[:rows]).all()
|
||||
assert not torch.isnan(out[:rows].float()).any()
|
||||
|
||||
# Scales: the kernel's fp32 SITU shifts the per-group absmax only
|
||||
# marginally vs the fp64 reference.
|
||||
torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5)
|
||||
|
||||
# FP8 codes: must land within 1 code of the ideal quantizer for ~all
|
||||
# elements, never off by more than 2.
|
||||
dist = fp8_code_dist(out[:rows], ref_q)
|
||||
assert dist.max().item() <= 2
|
||||
assert (dist <= 1).float().mean().item() >= 0.99
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)])
|
||||
@torch.inference_mode()
|
||||
def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
|
||||
"""The kernel expands num_valid_tokens by topk to bound rows."""
|
||||
set_random_seed(7)
|
||||
d = SITU_D
|
||||
num_tokens = tokens * topk + 37 # padding tail past the valid rows
|
||||
inp = token_random(num_tokens, 2 * d, torch.bfloat16)
|
||||
|
||||
out = torch.full((num_tokens, d), 17.0, device=DEVICE).to(fp8_dtype)
|
||||
scale = torch.full(
|
||||
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
out_sentinel = out.clone()
|
||||
num_valid_tokens = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
situ_and_mul_quant(
|
||||
out,
|
||||
scale,
|
||||
inp,
|
||||
beta=SITU_BETA,
|
||||
linear_beta=SITU_LINEAR_BETA,
|
||||
group_size=GROUP_SIZE,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
topk=topk,
|
||||
)
|
||||
|
||||
rows = tokens * topk
|
||||
assert torch.equal(scale[rows:], torch.ones_like(scale[rows:]))
|
||||
assert torch.equal(
|
||||
out[rows:].view(torch.uint8), out_sentinel[rows:].view(torch.uint8)
|
||||
)
|
||||
|
||||
ref_q, ref_s = ref_situ_block_fp8_quant(
|
||||
inp[:rows], SITU_BETA, SITU_LINEAR_BETA, GROUP_SIZE
|
||||
)
|
||||
torch.testing.assert_close(scale[:rows], ref_s, rtol=3e-2, atol=1e-5)
|
||||
assert fp8_code_dist(out[:rows], ref_q).max().item() <= 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half])
|
||||
@torch.inference_mode()
|
||||
def test_situ_and_mul_quant_pipelined_deterministic(dtype: torch.dtype) -> None:
|
||||
"""Two runs on the same input must be byte-identical."""
|
||||
set_random_seed(0)
|
||||
d = SITU_D
|
||||
num_tokens = 2048
|
||||
inp = token_random(num_tokens, 2 * d, dtype)
|
||||
|
||||
def run() -> tuple[torch.Tensor, torch.Tensor]:
|
||||
out = torch.empty(num_tokens, d, dtype=fp8_dtype, device=DEVICE)
|
||||
scale = torch.empty(
|
||||
num_tokens, d // GROUP_SIZE, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
situ_and_mul_quant(
|
||||
out,
|
||||
scale,
|
||||
inp,
|
||||
beta=SITU_BETA,
|
||||
linear_beta=SITU_LINEAR_BETA,
|
||||
group_size=GROUP_SIZE,
|
||||
)
|
||||
return out, scale
|
||||
|
||||
out1, s1 = run()
|
||||
out2, s2 = run()
|
||||
assert torch.equal(out1.view(torch.uint8), out2.view(torch.uint8))
|
||||
assert torch.equal(s1, s2)
|
||||
@@ -1,740 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.dsv4_topk import dsv4_topk
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
fused_topk_bias,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _torch_topk_softplus_sqrt(
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
):
|
||||
scores = F.softplus(gating_output.float()).sqrt()
|
||||
original_scores = scores
|
||||
|
||||
if hash_indices_table is not None:
|
||||
assert input_ids is not None
|
||||
if bias_vl is None or image_sentinel_lo == 0:
|
||||
topk_ids = hash_indices_table[input_ids.long()]
|
||||
else:
|
||||
image_mask = (input_ids.long() >= image_sentinel_lo) & (
|
||||
input_ids.long() < image_sentinel_lo + 5
|
||||
)
|
||||
topk_ids = hash_indices_table[input_ids.long()]
|
||||
vl_ids = (scores + bias_vl).topk(topk, dim=-1).indices
|
||||
topk_ids = torch.where(
|
||||
image_mask.unsqueeze(-1), vl_ids.to(topk_ids.dtype), topk_ids
|
||||
)
|
||||
else:
|
||||
if bias_vl is not None and image_sentinel_lo > 0:
|
||||
# Image tokens carry five consecutive in-vocab sentinel ids
|
||||
# starting at image_sentinel_lo and select experts with bias_vl
|
||||
# instead of the regular bias.
|
||||
assert input_ids is not None
|
||||
assert e_score_correction_bias is not None
|
||||
image_mask = (
|
||||
(input_ids.long() >= image_sentinel_lo)
|
||||
& (input_ids.long() < image_sentinel_lo + 5)
|
||||
).unsqueeze(-1)
|
||||
row_bias = torch.where(
|
||||
image_mask,
|
||||
bias_vl.unsqueeze(0),
|
||||
e_score_correction_bias.unsqueeze(0),
|
||||
)
|
||||
scores_for_choice = scores + row_bias
|
||||
elif e_score_correction_bias is not None:
|
||||
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
|
||||
else:
|
||||
scores_for_choice = scores
|
||||
# Match the fused kernel's deterministic tie-break: lower expert ids
|
||||
# win when scores are equal. torch.topk does not guarantee which tied
|
||||
# index it selects at the k-th boundary.
|
||||
topk_ids = torch.argsort(
|
||||
scores_for_choice, dim=-1, descending=True, stable=True
|
||||
)[:, :topk]
|
||||
|
||||
topk_weights = original_scores.gather(1, topk_ids.long())
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights = topk_weights * routed_scaling_factor
|
||||
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
def _dsv4_moe_config(monkeypatch, default_vllm_config, has_hash_routing, backend):
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
|
||||
from vllm.platforms import rocm
|
||||
|
||||
# Exercise both native MX and monolithic AITER selection on any ROCm device.
|
||||
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
|
||||
monkeypatch.setattr(rocm, "on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_enabled", lambda: True)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_fused_moe_enabled", lambda: True)
|
||||
default_vllm_config.kernel_config.moe_backend = "triton"
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=8,
|
||||
top_k=2,
|
||||
hidden_size=256,
|
||||
intermediate_size=256,
|
||||
params_dtype=torch.bfloat16,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=torch.zeros(8),
|
||||
hash_indices_table=(
|
||||
torch.tensor([[0, 1], [2, 3]], dtype=torch.int32)
|
||||
if has_hash_routing
|
||||
else None
|
||||
),
|
||||
prefix="hash_routing_selection",
|
||||
)
|
||||
return layer, replace(layer.routed_experts.moe_config, moe_backend=backend)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
@pytest.mark.parametrize("has_hash_routing", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"backend,expected_experts",
|
||||
[
|
||||
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
|
||||
("aiter", "AiterMxfp4Experts"),
|
||||
("triton_unfused", "UnfusedOAITritonExperts"),
|
||||
],
|
||||
)
|
||||
def test_hash_routing_backend_selection(
|
||||
dist_init,
|
||||
default_vllm_config,
|
||||
monkeypatch,
|
||||
has_hash_routing,
|
||||
backend,
|
||||
expected_experts,
|
||||
):
|
||||
"""Hash tables reach backend selection and hash-capable kernels."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
|
||||
layer, config = _dsv4_moe_config(
|
||||
monkeypatch, default_vllm_config, has_hash_routing, backend
|
||||
)
|
||||
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
assert experts_cls.__name__ == expected_experts
|
||||
# Monolithic kernels route internally, so they read the table from the layer.
|
||||
assert (layer.routed_experts.hash_indices_table is not None) == has_hash_routing
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
def test_hash_routing_rejects_monolithic_without_support(
|
||||
dist_init, default_vllm_config, monkeypatch
|
||||
):
|
||||
"""A monolithic kernel that does not implement hash routing is rejected."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
"_supports_hash_routing",
|
||||
staticmethod(lambda: False),
|
||||
)
|
||||
_, config = _dsv4_moe_config(
|
||||
monkeypatch, default_vllm_config, True, "aiter_triton_mxfp4_bf16"
|
||||
)
|
||||
with pytest.raises(ValueError, match="hash routing"):
|
||||
select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
|
||||
|
||||
def test_torch_topk_softplus_sqrt_breaks_ties_by_expert_id():
|
||||
gating_output = torch.tensor([[2.0, 1.0, 1.0, 0.0]])
|
||||
|
||||
_, topk_ids = _torch_topk_softplus_sqrt(
|
||||
gating_output,
|
||||
topk=2,
|
||||
renormalize=False,
|
||||
routed_scaling_factor=1.0,
|
||||
)
|
||||
|
||||
assert topk_ids.tolist() == [[0, 1]]
|
||||
|
||||
|
||||
def test_sqrtsoftplus_bias_uses_deepseek_v4_routing_method():
|
||||
assert (
|
||||
get_routing_method_type(
|
||||
scoring_func="sqrtsoftplus",
|
||||
top_k=8,
|
||||
renormalize=True,
|
||||
num_expert_group=None,
|
||||
has_e_score_bias=True,
|
||||
)
|
||||
== RoutingMethodType.DeepseekV4
|
||||
)
|
||||
assert (
|
||||
get_routing_method_type(
|
||||
scoring_func="sqrtsoftplus",
|
||||
top_k=8,
|
||||
renormalize=False,
|
||||
num_expert_group=None,
|
||||
has_e_score_bias=True,
|
||||
)
|
||||
== RoutingMethodType.Unspecified
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 33, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [1024, 2048])
|
||||
@pytest.mark.parametrize("num_experts", [128, 256, 384, 512])
|
||||
@pytest.mark.parametrize("topk", [6, 8, 16])
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 1.5])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
|
||||
def test_fused_topk_softplus_sqrt(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
e_score_correction_bias = torch.randn(
|
||||
(num_experts,), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
# Different kernels may return the topk experts in different orders when
|
||||
# scores tie; sort by expert id before comparing.
|
||||
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
|
||||
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
|
||||
torch.testing.assert_close(sorted_ref_ids, sorted_ids, atol=0, rtol=0)
|
||||
|
||||
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
|
||||
sorted_w = topk_weights.gather(1, idx_ops)
|
||||
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 33, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [1024, 2048])
|
||||
@pytest.mark.parametrize("num_experts", [256, 384, 512])
|
||||
@pytest.mark.parametrize("topk", [6, 8, 16])
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 2.5])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
|
||||
def test_fused_topk_softplus_sqrt_hash(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
vocab_size = 1024
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
# Per-token fixed expert selection: for each vocab id pick `topk` distinct
|
||||
# experts.
|
||||
hash_indices_table = torch.stack(
|
||||
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
|
||||
).to(device="cuda", dtype=torch.long)
|
||||
input_ids = torch.randint(
|
||||
0, vocab_size, (num_tokens,), dtype=torch.long, device="cuda"
|
||||
)
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=None,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
input_tokens=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
|
||||
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
|
||||
torch.testing.assert_close(sorted_ref_ids, sorted_ids, atol=0, rtol=0)
|
||||
|
||||
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
|
||||
sorted_w = topk_weights.gather(1, idx_ops)
|
||||
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="The DeepSeek V4 fast path is CUDA-only.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "num_experts", "indices_type"),
|
||||
[
|
||||
(0, 256, torch.int32),
|
||||
(17, 256, torch.int32),
|
||||
(17, 384, torch.int64),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("use_padding", [False, True])
|
||||
def test_dsv4_fast_topk(
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
indices_type: torch.dtype,
|
||||
use_padding: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
gating_output = torch.randn(
|
||||
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=6,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=correction_bias,
|
||||
)
|
||||
is_padding = torch.arange(num_tokens, device="cuda") % 2 == 1
|
||||
if use_padding:
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.router."
|
||||
"fused_topk_bias_router._get_padding_mask",
|
||||
lambda _: is_padding,
|
||||
)
|
||||
gating_output[is_padding] = float("nan")
|
||||
topk_weights_ref[is_padding] = 0
|
||||
topk_ids_ref[is_padding] = -1
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
gating_output,
|
||||
gating_output,
|
||||
"sqrtsoftplus",
|
||||
correction_bias,
|
||||
6,
|
||||
True,
|
||||
indices_type=indices_type,
|
||||
routed_scaling_factor=1.5,
|
||||
)
|
||||
|
||||
assert topk_ids.dtype == indices_type
|
||||
torch.testing.assert_close(topk_ids_ref.to(indices_type), topk_ids, atol=0, rtol=0)
|
||||
torch.testing.assert_close(
|
||||
topk_weights_ref,
|
||||
topk_weights,
|
||||
atol=2e-5,
|
||||
rtol=2e-5,
|
||||
)
|
||||
|
||||
if use_padding and num_tokens:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
weights, ids = dsv4_topk(
|
||||
gating_output, correction_bias, indices_type, 1.5, is_padding=is_padding
|
||||
)
|
||||
is_padding.fill_(True)
|
||||
graph.replay()
|
||||
assert (ids == -1).all()
|
||||
assert (weights == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize("use_hash", [False, True])
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
@pytest.mark.parametrize("use_padding_mask", [False, True])
|
||||
@pytest.mark.parametrize("pad_with_nan", [False, True])
|
||||
@pytest.mark.parametrize("num_experts", [128, 256, 384])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
|
||||
def test_fused_topk_softplus_sqrt_padding(
|
||||
use_hash: bool,
|
||||
use_bias: bool,
|
||||
use_padding_mask: bool,
|
||||
pad_with_nan: bool,
|
||||
num_experts: int,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
"""Verify explicit padding and NaN-padded rows do not affect real rows."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 8
|
||||
topk = 6
|
||||
indices_dtype = torch.int32
|
||||
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
# Exercise top-k cutoff ties without relying on random low-precision collisions.
|
||||
gating_output[0] = 0
|
||||
|
||||
padding_rows = torch.zeros(num_tokens, dtype=torch.bool, device="cuda")
|
||||
padding_rows[1::2] = True
|
||||
if pad_with_nan:
|
||||
gating_output[padding_rows] = float("nan")
|
||||
is_padding = padding_rows if use_padding_mask else None
|
||||
|
||||
# A negative correction bias makes explicit pad rows look selectable unless
|
||||
# the kernel uses the is_padding guard.
|
||||
e_score_correction_bias = None
|
||||
if use_bias:
|
||||
e_score_correction_bias = (
|
||||
-torch.rand((num_experts,), dtype=torch.float32, device="cuda") - 1.0
|
||||
)
|
||||
|
||||
input_ids = None
|
||||
hash_indices_table = None
|
||||
if use_hash:
|
||||
vocab_size = 64
|
||||
hash_indices_table = torch.stack(
|
||||
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
|
||||
).to(device="cuda", dtype=indices_dtype)
|
||||
input_ids = torch.randint(
|
||||
0, vocab_size, (num_tokens,), dtype=indices_dtype, device="cuda"
|
||||
)
|
||||
|
||||
topk_weights = torch.empty(num_tokens, topk, dtype=torch.float32, device="cuda")
|
||||
topk_ids = torch.empty(num_tokens, topk, dtype=indices_dtype, device="cuda")
|
||||
token_expert_indices = torch.empty(
|
||||
num_tokens, topk, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
ops.topk_hash_softplus_sqrt(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=1.0,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
input_tokens=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
is_padding=is_padding,
|
||||
)
|
||||
|
||||
if use_padding_mask:
|
||||
pad_ids = topk_ids[padding_rows]
|
||||
pad_weights = topk_weights[padding_rows]
|
||||
assert torch.equal(pad_ids, torch.full_like(pad_ids, -1)), (
|
||||
f"Explicit pad rows should contain only -1 ids, got {pad_ids.tolist()}"
|
||||
)
|
||||
assert (pad_weights == 0).all(), (
|
||||
"Explicit pad rows should have all-zero weights, "
|
||||
f"got {pad_weights.tolist()}"
|
||||
)
|
||||
|
||||
if pad_with_nan:
|
||||
nan_pad_weights = topk_weights[padding_rows]
|
||||
assert torch.isfinite(nan_pad_weights).all(), (
|
||||
f"NaN-padded rows have non-finite weights, got {nan_pad_weights.tolist()}"
|
||||
)
|
||||
assert (nan_pad_weights == 0).all(), (
|
||||
"NaN-padded rows should have all-zero weights, "
|
||||
f"got {nan_pad_weights.tolist()}"
|
||||
)
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=1.0,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
rows_to_compare = torch.ones(num_tokens, dtype=torch.bool, device="cuda")
|
||||
if use_padding_mask or pad_with_nan:
|
||||
rows_to_compare = ~padding_rows
|
||||
|
||||
sorted_ref_ids, idx_ref = topk_ids_ref[rows_to_compare].sort(dim=-1)
|
||||
sorted_ids, idx_ops = topk_ids[rows_to_compare].sort(dim=-1)
|
||||
torch.testing.assert_close(
|
||||
sorted_ref_ids, sorted_ids.to(sorted_ref_ids.dtype), atol=0, rtol=0
|
||||
)
|
||||
|
||||
sorted_w_ref = topk_weights_ref[rows_to_compare].gather(1, idx_ref)
|
||||
sorted_w = topk_weights[rows_to_compare].gather(1, idx_ops)
|
||||
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def _make_mixed_input_ids(
|
||||
num_tokens: int, image_sentinel_lo: int, dtype: torch.dtype = torch.long
|
||||
) -> torch.Tensor:
|
||||
"""Mix of text, image-sentinel, and above-sentinel ids.
|
||||
|
||||
Every third token is an image sentinel id (cycling through the five slots
|
||||
[image_sentinel_lo, image_sentinel_lo + 5)); every sixth token carries an
|
||||
id just above the sentinel block, simulating the named special tokens
|
||||
that follow it — those must route as regular text tokens.
|
||||
"""
|
||||
input_ids = torch.randint(
|
||||
0, image_sentinel_lo, (num_tokens,), dtype=torch.long, device="cuda"
|
||||
)
|
||||
pos = torch.arange(num_tokens, device="cuda")
|
||||
image_rows = pos % 3 == 0
|
||||
input_ids[image_rows] = image_sentinel_lo + (pos[image_rows] // 3) % 5
|
||||
above_rows = pos % 6 == 3
|
||||
input_ids[above_rows] = image_sentinel_lo + 5
|
||||
return input_ids.to(dtype)
|
||||
|
||||
|
||||
def _assert_topk_matches(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights_ref: torch.Tensor,
|
||||
topk_ids_ref: torch.Tensor,
|
||||
) -> None:
|
||||
# Different kernels may return the topk experts in different orders when
|
||||
# scores tie; sort by expert id before comparing.
|
||||
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
|
||||
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
|
||||
torch.testing.assert_close(
|
||||
sorted_ref_ids, sorted_ids.to(sorted_ref_ids.dtype), atol=0, rtol=0
|
||||
)
|
||||
|
||||
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
|
||||
sorted_w = topk_weights.gather(1, idx_ops.to(torch.long))
|
||||
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "topk", "renormalize", "dtype"),
|
||||
[
|
||||
(256, 6, True, torch.float32), # Triton dsv4 fast path
|
||||
(256, 8, True, torch.bfloat16), # CUDA topk kernel
|
||||
(384, 6, False, torch.half),
|
||||
(512, 16, True, torch.float32),
|
||||
],
|
||||
)
|
||||
def test_fused_topk_softplus_sqrt_bias_vl(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
"""Image sentinel tokens must select experts with bias_vl.
|
||||
|
||||
Rows with ids just above the sentinel block (image_sentinel_lo + 5)
|
||||
simulate the named special tokens that follow it and must route through
|
||||
the regular bias path; the reference comparison covers them.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 64
|
||||
hidden_size = 1024
|
||||
vocab_size = 128
|
||||
image_sentinel_lo = vocab_size - 8
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
e_score_correction_bias = torch.randn(
|
||||
(num_experts,), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
|
||||
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
|
||||
assert (input_ids == image_sentinel_lo + 5).any()
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
input_ids=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=1.5,
|
||||
input_tokens=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "topk", "renormalize", "dtype", "indices_dtype"),
|
||||
[
|
||||
(256, 6, True, torch.float32, torch.long), # specialized hash kernel
|
||||
(384, 6, True, torch.float32, torch.int32), # specialized hash kernel
|
||||
(384, 8, False, torch.bfloat16, torch.long), # generic hash kernel
|
||||
(512, 6, True, torch.float32, torch.long), # generic hash kernel
|
||||
],
|
||||
)
|
||||
def test_fused_topk_softplus_sqrt_hash_bias_vl(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
dtype: torch.dtype,
|
||||
indices_dtype: torch.dtype,
|
||||
):
|
||||
"""Hash MoE: text rows use tid2eid, image rows use topk(score + bias_vl).
|
||||
|
||||
Sentinel ids are in-vocab; rows with ids above the sentinel block
|
||||
(image_sentinel_lo + 5) must still come straight from the hash table.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 64
|
||||
hidden_size = 1024
|
||||
vocab_size = 64
|
||||
image_sentinel_lo = vocab_size - 8
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
hash_indices_table = torch.stack(
|
||||
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
|
||||
).to(device="cuda", dtype=indices_dtype)
|
||||
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
|
||||
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
|
||||
image_rows = (input_ids >= image_sentinel_lo) & (input_ids < image_sentinel_lo + 5)
|
||||
assert image_rows.any() and (~image_rows).any()
|
||||
assert (input_ids == image_sentinel_lo + 5).any()
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=2.5,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=None,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
input_tokens=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
routed_scaling_factor=2.5,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
|
||||
|
||||
# Text rows must come straight from the hash table, image rows from
|
||||
# topk(score + bias_vl).
|
||||
text_ids = hash_indices_table[input_ids[~image_rows].long()]
|
||||
torch.testing.assert_close(
|
||||
topk_ids[~image_rows].to(text_ids.dtype), text_ids, atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="The DeepSeek V4 fast path is CUDA-only.",
|
||||
)
|
||||
def test_dsv4_fast_topk_bias_vl():
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 33
|
||||
num_experts = 256
|
||||
vocab_size = 128
|
||||
image_sentinel_lo = vocab_size - 8
|
||||
gating_output = torch.randn(
|
||||
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
|
||||
bias_vl = torch.randn(num_experts, dtype=torch.float32, device="cuda")
|
||||
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=6,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=correction_bias,
|
||||
input_ids=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
topk_weights, topk_ids = dsv4_topk(
|
||||
gating_output,
|
||||
correction_bias,
|
||||
torch.int64,
|
||||
1.5,
|
||||
input_ids=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
assert topk_ids.dtype == torch.int64
|
||||
torch.testing.assert_close(topk_ids_ref.to(torch.int64), topk_ids, atol=0, rtol=0)
|
||||
torch.testing.assert_close(topk_weights_ref, topk_weights, atol=2e-5, rtol=2e-5)
|
||||
@@ -1,261 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MoE with non-gated activations (*_no_mul).
|
||||
|
||||
These tests verify that MoE layers work correctly with activations like
|
||||
silu_no_mul, gelu_no_mul, relu2_no_mul where the activation output dimension
|
||||
equals N (not N // 2 like gated activations).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
apply_moe_activation_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Triton MoE kernels require a CUDA-alike or XPU device.",
|
||||
),
|
||||
pytest.mark.skipif(
|
||||
current_platform.is_cuda_alike()
|
||||
and not current_platform.has_device_capability(80),
|
||||
reason="Triton MoE kernels require compute capability >= 8.0 on CUDA/ROCm.",
|
||||
),
|
||||
]
|
||||
|
||||
# Test parameters
|
||||
M_SIZES = [1, 16, 64]
|
||||
N_SIZES = [128, 256]
|
||||
K_SIZES = [64, 128]
|
||||
TOPK_VALUES = [1, 2]
|
||||
NUM_EXPERTS = 8
|
||||
NO_MUL_ACTIVATIONS = [
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
]
|
||||
|
||||
|
||||
def test_apply_moe_activation_supported_contract():
|
||||
supported = {
|
||||
activation
|
||||
for activation in MoEActivation
|
||||
if apply_moe_activation_supported(activation)
|
||||
}
|
||||
|
||||
assert supported == set(MoEActivation) - {MoEActivation.RELU2}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
activation
|
||||
for activation in MoEActivation
|
||||
if apply_moe_activation_supported(activation)
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_supported_apply_moe_activation_executes(activation: MoEActivation):
|
||||
output_width = 64
|
||||
input_width = output_width * 2 if activation.is_gated else output_width
|
||||
input = torch.randn(2, input_width, device="cuda", dtype=torch.bfloat16)
|
||||
output = torch.empty(2, output_width, device="cuda", dtype=torch.bfloat16)
|
||||
activation_config = ApplyMoEActivationConfig()
|
||||
if activation == MoEActivation.SITU:
|
||||
activation_config = ApplyMoEActivationConfig(activation_situ_beta=1.0)
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
activation_config = ApplyMoEActivationConfig(clamp_limit=7.0)
|
||||
|
||||
apply_moe_activation(activation, output, input, activation_config=activation_config)
|
||||
|
||||
assert torch.isfinite(output).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("activation", list(MoEActivation))
|
||||
def test_triton_activation_metadata_tracks_shared_apply(activation: MoEActivation):
|
||||
assert TritonExperts._supports_activation(
|
||||
activation
|
||||
) == apply_moe_activation_supported(activation)
|
||||
|
||||
|
||||
def make_test_tensors(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
device: str = DEVICE,
|
||||
):
|
||||
"""Create test tensors for MoE with non-gated activation.
|
||||
|
||||
For non-gated activations (*_no_mul):
|
||||
- w1: (E, N, K) - projects from K to N
|
||||
- w2: (E, K, N) - projects from N back to K (note: N, not N//2)
|
||||
"""
|
||||
hidden_states = torch.randn(m, k, dtype=dtype, device=device)
|
||||
|
||||
# For non-gated: w1 projects K -> N, w2 projects N -> K
|
||||
w1 = torch.randn(num_experts, n, k, dtype=dtype, device=device) * 0.1
|
||||
w2 = torch.randn(num_experts, k, n, dtype=dtype, device=device) * 0.1
|
||||
|
||||
topk_weights = torch.ones(m, topk, dtype=torch.float32, device=device) / topk
|
||||
topk_ids = torch.randint(0, num_experts, (m, topk), device=device)
|
||||
|
||||
return hidden_states, w1, w2, topk_weights, topk_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", M_SIZES)
|
||||
@pytest.mark.parametrize("n", N_SIZES)
|
||||
@pytest.mark.parametrize("k", K_SIZES)
|
||||
@pytest.mark.parametrize("topk", TOPK_VALUES)
|
||||
@pytest.mark.parametrize("activation", NO_MUL_ACTIVATIONS)
|
||||
@torch.inference_mode()
|
||||
def test_triton_experts_no_mul_activation(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
topk: int,
|
||||
activation: MoEActivation,
|
||||
):
|
||||
hidden_states, w1, w2, topk_weights, topk_ids = make_test_tensors(
|
||||
m, n, k, NUM_EXPERTS, topk
|
||||
)
|
||||
|
||||
experts = TritonExperts(
|
||||
moe_config=make_dummy_moe_config(),
|
||||
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
)
|
||||
|
||||
ws1_shape, ws2_shape, out_shape = experts.workspace_shapes(
|
||||
M=m,
|
||||
N=n,
|
||||
K=k,
|
||||
topk=topk,
|
||||
global_num_experts=NUM_EXPERTS,
|
||||
local_num_experts=NUM_EXPERTS,
|
||||
expert_tokens_meta=None,
|
||||
activation=activation,
|
||||
)
|
||||
|
||||
# Verify workspace shapes are correct for no_mul activation
|
||||
# workspace1 should handle activation_out_dim = N (not N//2)
|
||||
assert ws1_shape == (m, topk, max(n, k)), (
|
||||
f"workspace1 shape mismatch: expected {(m, topk, max(n, k))}, got {ws1_shape}"
|
||||
)
|
||||
# workspace2 should handle max(N, K) for intermediate_cache1/cache3
|
||||
assert ws2_shape == (m, topk, max(n, k)), (
|
||||
f"workspace2 shape mismatch: expected {(m, topk, max(n, k))}, got {ws2_shape}"
|
||||
)
|
||||
assert out_shape == (m, k), (
|
||||
f"output shape mismatch: expected {(m, k)}, got {out_shape}"
|
||||
)
|
||||
|
||||
workspace1 = torch.empty(
|
||||
ws1_shape[0] * ws1_shape[1] * ws1_shape[2],
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
workspace2 = torch.empty(
|
||||
ws2_shape[0] * ws2_shape[1] * ws2_shape[2],
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
output = torch.zeros(m, k, dtype=hidden_states.dtype, device=hidden_states.device)
|
||||
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=NUM_EXPERTS,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=workspace1,
|
||||
workspace2=workspace2,
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
assert output.shape == (m, k), f"Expected shape {(m, k)}, got {output.shape}"
|
||||
assert not torch.isnan(output).any(), "Output contains NaN"
|
||||
assert not torch.isinf(output).any(), "Output contains Inf"
|
||||
assert output.abs().sum() > 0, "Output is all zeros"
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_workspace_shapes_no_mul_vs_gated():
|
||||
"""Test that workspace shapes differ correctly between gated and non-gated."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
|
||||
|
||||
M, N, K, topk = 64, 256, 128, 2
|
||||
|
||||
experts = TritonExperts(
|
||||
moe_config=make_dummy_moe_config(),
|
||||
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
)
|
||||
|
||||
ws1_no_mul, _, out_no_mul = experts.workspace_shapes(
|
||||
M, N, K, topk, 8, 8, None, MoEActivation.SILU_NO_MUL
|
||||
)
|
||||
|
||||
ws1_gated, _, out_gated = experts.workspace_shapes(
|
||||
M, N, K, topk, 8, 8, None, MoEActivation.SILU
|
||||
)
|
||||
|
||||
# For no_mul: activation_out_dim = N
|
||||
# For gated: activation_out_dim = N // 2
|
||||
# workspace1 should use max(activation_out_dim, K)
|
||||
activation_out_dim_no_mul = N
|
||||
activation_out_dim_gated = N // 2
|
||||
|
||||
assert ws1_no_mul[2] == max(activation_out_dim_no_mul, K), (
|
||||
f"no_mul workspace1 last dim should be max({activation_out_dim_no_mul}, {K})"
|
||||
)
|
||||
assert ws1_gated[2] == max(activation_out_dim_gated, K), (
|
||||
f"gated workspace1 last dim should be max({activation_out_dim_gated}, {K})"
|
||||
)
|
||||
|
||||
# Output shapes should be the same
|
||||
assert out_no_mul == out_gated == (M, K)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_adjust_n_for_activation():
|
||||
"""Test the adjust_N_for_activation method."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
|
||||
|
||||
experts = TritonExperts(
|
||||
moe_config=make_dummy_moe_config(),
|
||||
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
)
|
||||
|
||||
N = 256
|
||||
|
||||
# Gated activations should return N // 2
|
||||
assert experts.adjust_N_for_activation(N, MoEActivation.SILU) == N // 2
|
||||
assert experts.adjust_N_for_activation(N, MoEActivation.GELU) == N // 2
|
||||
|
||||
# Non-gated activations should return N
|
||||
assert experts.adjust_N_for_activation(N, MoEActivation.SILU_NO_MUL) == N
|
||||
assert experts.adjust_N_for_activation(N, MoEActivation.GELU_NO_MUL) == N
|
||||
assert experts.adjust_N_for_activation(N, MoEActivation.RELU2_NO_MUL) == N
|
||||
@@ -1,169 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from https://github.com/sgl-project/sglang/blob/main/test/srt/test_triton_moe_channel_fp8_kernel.py
|
||||
import itertools
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import fused_moe
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.get_device_capability() < (9, 0):
|
||||
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
|
||||
if current_platform.is_fp8_fnuz():
|
||||
pytest.skip(
|
||||
"Tests in this file require float8_e4m3fn and platform does not support",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
|
||||
"""Matrix multiplication function that supports per-token input
|
||||
quantization and per-column weight quantization"""
|
||||
A = A.to(torch.float32)
|
||||
B = B.to(torch.float32)
|
||||
|
||||
assert A.shape[-1] == B.shape[-1], "Dimension mismatch"
|
||||
assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor"
|
||||
|
||||
# Reshape input
|
||||
M = A.numel() // A.shape[-1]
|
||||
B = B.t() # Transpose weight matrix
|
||||
N, K = B.shape
|
||||
origin_C_shape = A.shape[:-1] + (K,)
|
||||
A = A.reshape(M, N)
|
||||
|
||||
# As is per-token [M, 1], Bs is per-column [1, K]
|
||||
C = torch.matmul(A, B) # [M, K]
|
||||
C = As * C * Bs.view(1, -1) # Broadcast per-column scale
|
||||
|
||||
return C.reshape(origin_C_shape).to(output_dtype)
|
||||
|
||||
|
||||
def fp8_mask(a, mask):
|
||||
dtype = a.dtype
|
||||
return a.view(torch.int8)[mask].view(dtype)
|
||||
|
||||
|
||||
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk):
|
||||
"""This function performs fused moe with per-column int8
|
||||
quantization using native torch."""
|
||||
B, D = a.shape
|
||||
# Perform per-token quantization
|
||||
a_q, a_s = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True)
|
||||
# Repeat tokens to match topk
|
||||
a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
# Also repeat the scale
|
||||
a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1]
|
||||
|
||||
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
|
||||
# Calculate routing
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
topk_weight = topk_weight.view(-1)
|
||||
topk_ids = topk_ids.view(-1)
|
||||
# Process each expert
|
||||
for i in range(w1.shape[0]):
|
||||
mask = topk_ids == i
|
||||
if mask.sum():
|
||||
# First MLP layer: note that a_s is now per-token
|
||||
inter_out = native_w8a8_per_token_matmul(
|
||||
fp8_mask(a_q, mask),
|
||||
w1[i],
|
||||
fp8_mask(a_s, mask),
|
||||
w1_s[i],
|
||||
output_dtype=a.dtype,
|
||||
)
|
||||
# Activation function
|
||||
act_out = SiluAndMul().forward_native(inter_out)
|
||||
# Quantize activation output with per-token
|
||||
act_out_q, act_out_s = ops.scaled_fp8_quant(
|
||||
act_out, use_per_token_if_dynamic=True
|
||||
)
|
||||
|
||||
# Second MLP layer
|
||||
out[mask] = native_w8a8_per_token_matmul(
|
||||
act_out_q, w2[i], act_out_s, w2_s[i], output_dtype=a.dtype
|
||||
)
|
||||
# Apply routing weights and sum
|
||||
return (
|
||||
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
|
||||
).sum(dim=1)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_cuda():
|
||||
"""Sets the default CUDA device before each test in this module."""
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
|
||||
DTYPES = [torch.half, torch.bfloat16]
|
||||
M = [1, 33]
|
||||
N = [128, 1024]
|
||||
K = [256, 4096]
|
||||
E = [8]
|
||||
TOP_KS = [2, 6]
|
||||
SEEDS = [0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"M, N, K, E, topk, dtype, seed",
|
||||
itertools.product(M, N, K, E, TOP_KS, DTYPES, SEEDS),
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_fp8_fused_moe(M, N, K, E, topk, dtype, seed):
|
||||
torch.manual_seed(seed)
|
||||
# Initialize int8 quantization parameters
|
||||
factor_for_scale = 1e-2
|
||||
finfo = torch.finfo(torch.float8_e4m3fn)
|
||||
fp8_max = finfo.max
|
||||
fp8_min = finfo.min
|
||||
|
||||
# Input tensor
|
||||
# M * K
|
||||
a = torch.randn((M, K), dtype=dtype) / 10
|
||||
|
||||
# Generate int8 weights
|
||||
w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2
|
||||
w1 = (w1_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
|
||||
|
||||
w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2
|
||||
w2 = (w2_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
|
||||
|
||||
# Generate scale for each column (per-column quantization)
|
||||
w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * factor_for_scale
|
||||
w2_s = torch.rand(E, K, device=w2_fp32.device) * factor_for_scale
|
||||
score = torch.randn((M, E), dtype=dtype)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk)
|
||||
out = fused_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
score,
|
||||
topk,
|
||||
renormalize=False,
|
||||
quant_config=fp8_w8a8_moe_quant_config(
|
||||
per_act_token_quant=True,
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
block_shape=None, # Not using block quantization
|
||||
),
|
||||
)
|
||||
|
||||
# Check results
|
||||
rel_diff = torch.mean(
|
||||
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
|
||||
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
|
||||
assert rel_diff < 0.05
|
||||
@@ -1,201 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the FlashInfer TRTLLM BF16 MoE backend
|
||||
(`TrtLlmBf16ExpertsModular`).
|
||||
|
||||
This mirrors the TRTLLM NvFP4 modular test shape: construct the modular
|
||||
expert wrapper directly, pass production-format BlockMajorK weights, and
|
||||
compare against a torch MoE reference using the original BF16 weights.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from tests.kernels.utils import torch_moe
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
TrtLlmBf16ExpertsModular,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
|
||||
TrtLlmBf16LoRAExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
convert_to_unquantized_kernel_format,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if pytest and (
|
||||
not has_flashinfer_trtllm_fused_moe()
|
||||
or not current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip(
|
||||
"Requires flashinfer TRTLLM fused MoE BF16 backend (SM100)",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
|
||||
MNK_FACTORS = [
|
||||
(2, 160, 2560),
|
||||
(2, 1024, 1024),
|
||||
(64, 2048, 1536),
|
||||
(64, 1024, 4096),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", [128])
|
||||
@pytest.mark.parametrize("topk", [8])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_bf16_moe_modular_no_graph(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
e: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
workspace_init,
|
||||
):
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
|
||||
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
|
||||
# The FlashInfer conversion may rewrite unpadded input storage in place.
|
||||
# Preserve the original layout for the independent torch reference.
|
||||
reference_w1 = w1.clone()
|
||||
reference_w2 = w2.clone()
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
scores = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(scores, topk)
|
||||
topk_weights = topk_weights.contiguous()
|
||||
topk_ids = topk_ids.to(torch.int32).contiguous()
|
||||
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=dtype,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
trtllm_w1, trtllm_w2 = convert_to_unquantized_kernel_format(
|
||||
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
|
||||
moe_config,
|
||||
w1,
|
||||
w2,
|
||||
)
|
||||
expected_n = (n + 127) // 128 * 128
|
||||
assert moe_config.intermediate_size_per_partition == expected_n
|
||||
assert trtllm_w2.numel() == e * k * expected_n
|
||||
|
||||
trtllm_experts = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
TrtLlmBf16ExpertsModular(
|
||||
moe_config=moe_config,
|
||||
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
),
|
||||
)
|
||||
|
||||
trtllm_output = trtllm_experts.apply(
|
||||
hidden_states=a,
|
||||
w1=trtllm_w1,
|
||||
w2=trtllm_w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
torch_output = torch_moe(
|
||||
a,
|
||||
reference_w1,
|
||||
reference_w2,
|
||||
score,
|
||||
topk,
|
||||
activation=MoEActivation.SILU,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
torch_output,
|
||||
trtllm_output,
|
||||
atol=1e-1,
|
||||
rtol=2e-1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_lora_delta", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_bf16_lora_accepts_checkpoint_shaped_weights(has_lora_delta):
|
||||
"""LoRA dispatch accepts both 3D parameters and legacy 4D packed views."""
|
||||
config = make_dummy_moe_config(
|
||||
num_experts=128, hidden_dim=256, intermediate_size=128
|
||||
)
|
||||
experts = TrtLlmBf16LoRAExperts(config, FUSED_MOE_UNQUANTIZED_CONFIG)
|
||||
x = torch.randn(2, 256, device="cuda", dtype=torch.bfloat16) / 10
|
||||
w1, w2 = convert_to_unquantized_kernel_format(
|
||||
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
|
||||
config,
|
||||
torch.randn(128, 256, 256, device="cuda", dtype=torch.bfloat16) / 10,
|
||||
torch.randn(128, 256, 128, device="cuda", dtype=torch.bfloat16) / 10,
|
||||
)
|
||||
assert w1.ndim == w2.ndim == 3
|
||||
topk = (
|
||||
torch.tensor([[0], [1]], device="cuda", dtype=torch.int32),
|
||||
torch.ones(2, 1, device="cuda", dtype=torch.float32),
|
||||
)
|
||||
delta = torch.randn(2, 1, 256, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
def invoke(w1, w2):
|
||||
result = experts.invoke_routed_moe(
|
||||
hidden_states=x,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_ids_and_weights=topk,
|
||||
gemm1_lora_delta=delta if has_lora_delta else None,
|
||||
global_num_experts=128,
|
||||
a1q_scale=None,
|
||||
output=torch.empty_like(x),
|
||||
)
|
||||
if has_lora_delta:
|
||||
# Only compare rows belonging to real tokens, excluding padding.
|
||||
indices = result[2].flatten().long()
|
||||
return result[0][indices], result[3][indices]
|
||||
return result
|
||||
|
||||
actual = invoke(w1, w2)
|
||||
expected = invoke(w1.view(128, 4, 256, 64), w2.view(128, 2, 256, 64))
|
||||
for actual_tensor, expected_tensor in zip(actual, expected):
|
||||
torch.testing.assert_close(actual_tensor, expected_tensor)
|
||||
@@ -1,514 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the FlashInfer TRTLLM NvFP4 MoE backend
|
||||
(`TrtLlmNvFp4ExpertsModular`).
|
||||
|
||||
Covers the activations the wrapper claims to support — SiLU, RELU^2 (non-gated),
|
||||
and GELU — including a Gemma4-shaped case (128 experts, top-k 8,
|
||||
intermediate_size 704) that exercises the non-256-aligned padding path.
|
||||
"""
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import check_deferred_moe_finalize, make_test_quant_config
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
convert_swizzled_to_linear,
|
||||
dequantize_nvfp4_to_dtype,
|
||||
)
|
||||
from tests.kernels.utils import torch_moe
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.custom_op import CustomOp, op_registry
|
||||
from vllm.model_executor.layers.activation import SiluAndMulWithClamp, SituAndMul
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
|
||||
TrtLlmNvFp4ExpertsModular,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_static_weights_for_trtllm_fp4_moe,
|
||||
reorder_w1w3_to_w3w1,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if pytest and (
|
||||
not has_flashinfer_trtllm_fused_moe()
|
||||
or not current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip(
|
||||
"Requires flashinfer TRTLLM fused MoE and NvFP4 (SM100)",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
|
||||
# The (64, 704, 4096) row matches Gemma4's MoE shape and exercises the
|
||||
# non-256-aligned intermediate (padded inside the wrapper).
|
||||
MNK_FACTORS = [
|
||||
(2, 1024, 1024),
|
||||
(64, 2048, 1536),
|
||||
(64, 704, 4096),
|
||||
]
|
||||
|
||||
_SWIGLU_LIMIT = 0.1
|
||||
_LARGE_OUTPUT1_SCALE = 32768.0
|
||||
_CLAMP_OP_NAME = "test_silu_and_mul_with_clamp"
|
||||
_SITU_OP_NAME = "test_situ_and_mul"
|
||||
|
||||
# Test-only fixed-limit clamp. ``custom_op_name`` makes the class itself
|
||||
# valid as an ``activation=`` argument to ``torch_moe`` (which only looks
|
||||
# up ``activation.custom_op_name`` in ``op_registry``), so no
|
||||
# ``MoEActivation`` enum extension is needed.
|
||||
if _CLAMP_OP_NAME not in op_registry:
|
||||
|
||||
@CustomOp.register(_CLAMP_OP_NAME)
|
||||
class _SiluAndMulWithClampTest(SiluAndMulWithClamp):
|
||||
custom_op_name = _CLAMP_OP_NAME
|
||||
|
||||
def __init__(self, *, compile_native: bool = True) -> None:
|
||||
super().__init__(_SWIGLU_LIMIT, compile_native=compile_native)
|
||||
|
||||
|
||||
if _SITU_OP_NAME not in op_registry:
|
||||
|
||||
@CustomOp.register(_SITU_OP_NAME)
|
||||
class _SituAndMulTest(SituAndMul):
|
||||
custom_op_name = _SITU_OP_NAME
|
||||
|
||||
def __init__(self, *, compile_native: bool = True) -> None:
|
||||
super().__init__(4.0, 25.0, compile_native=compile_native)
|
||||
|
||||
|
||||
SILU_WITH_CLAMP = op_registry[_CLAMP_OP_NAME]
|
||||
SITU = op_registry[_SITU_OP_NAME]
|
||||
|
||||
|
||||
def _make_trtllm_fp4_moe_kernel(
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
per_token_activation: bool = False,
|
||||
) -> mk.FusedMoEKernel:
|
||||
experts = TrtLlmNvFp4ExpertsModular(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
per_token_activation=per_token_activation,
|
||||
)
|
||||
# Mimic the production weight-loader path so per-expert tensors that
|
||||
# are normally precomputed in process_weights_after_loading (g1_scale_c
|
||||
# and the rescaled gemm1_clamp_limit) get materialized. The test's
|
||||
# synthetic quant_config has g1_alphas/g2_alphas already at their
|
||||
# post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas
|
||||
# (same tensor) and use input_scale=1 to make the in-place
|
||||
# weight_scale_2 *= input_scale step a no-op.
|
||||
fake_layer = torch.nn.Module()
|
||||
fake_layer.w13_weight_scale_2 = quant_config.g1_alphas
|
||||
fake_layer.w2_weight_scale_2 = quant_config.g2_alphas
|
||||
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
|
||||
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
|
||||
experts.process_weights_after_loading(fake_layer)
|
||||
return mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
experts,
|
||||
)
|
||||
|
||||
|
||||
ACTIVATION_CASES = [
|
||||
pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, False, id="silu"),
|
||||
pytest.param(
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SILU,
|
||||
None,
|
||||
True,
|
||||
id="silu_per_token",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SILU,
|
||||
SILU_WITH_CLAMP,
|
||||
_SWIGLU_LIMIT,
|
||||
False,
|
||||
id="silu_clamp",
|
||||
),
|
||||
pytest.param(MoEActivation.SITU, SITU, None, False, id="situ"),
|
||||
pytest.param(
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
None,
|
||||
False,
|
||||
id="relu2_no_mul",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
None,
|
||||
True,
|
||||
id="relu2_no_mul_per_token",
|
||||
),
|
||||
pytest.param(MoEActivation.GELU, MoEActivation.GELU, None, False, id="gelu"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", [128])
|
||||
@pytest.mark.parametrize("topk", [8])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"activation,torch_activation,swiglu_limit,per_token_activation",
|
||||
ACTIVATION_CASES,
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_fp4_moe_no_graph(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
e: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
activation: MoEActivation,
|
||||
torch_activation: MoEActivation | type[SiluAndMulWithClamp] | type[SituAndMul],
|
||||
swiglu_limit: float | None,
|
||||
per_token_activation: bool,
|
||||
workspace_init,
|
||||
):
|
||||
# FlashInfer's trtllm_batched_gemm_runner has no precompiled tile
|
||||
# config for non-gated RELU^2 at non-256-aligned intermediate_size
|
||||
# (e.g. Gemma4's 704). Other activations (SiLU/GELU) work at the
|
||||
# same shape. Tracked upstream in FlashInfer; unrelated to this
|
||||
# PR's GELU enablement (Gemma4 uses GeGLU, not non-gated RELU^2).
|
||||
if activation == MoEActivation.RELU2_NO_MUL and (m, n, k) == (64, 704, 4096):
|
||||
pytest.skip(
|
||||
"FlashInfer trtllm_batched_gemm_runner: no valid tile config "
|
||||
"for non-gated RELU^2 at intermediate_size=704 "
|
||||
"(getValidConfigIndices throws). Tracked upstream."
|
||||
)
|
||||
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
|
||||
quant_blocksize = 16
|
||||
is_gated_act = activation.is_gated
|
||||
|
||||
w1_q, w2_q, quant_config = make_test_quant_config(
|
||||
e,
|
||||
n,
|
||||
k,
|
||||
in_dtype=dtype,
|
||||
quant_dtype="nvfp4",
|
||||
block_shape=None,
|
||||
per_act_token_quant=False,
|
||||
make_gate=is_gated_act,
|
||||
# The TRT-LLM FP4 MoE kernel rejects swizzled (padded) activation
|
||||
# scales — its numel-based vec_size check requires numel == M*K/16.
|
||||
# Match what oracle/nvfp4.py does for this backend.
|
||||
is_scale_swizzled=False,
|
||||
)
|
||||
quant_config.gemm1_clamp_limit = swiglu_limit
|
||||
if swiglu_limit is not None:
|
||||
assert quant_config.g1_alphas is not None
|
||||
assert quant_config.a2_gscale is not None
|
||||
assert torch.all(quant_config.a2_gscale == 1)
|
||||
# With a2_gscale == 1, g1_alphas is the TRTLLM
|
||||
# output1_scale_gate_scalar. Make it large enough to catch
|
||||
# clamp/output-scale coupling in the FlashInfer kernel wrapper.
|
||||
quant_config.g1_alphas.fill_(_LARGE_OUTPUT1_SCALE)
|
||||
|
||||
w1_q_linear = w1_q
|
||||
w2_q_linear = w2_q
|
||||
w1_scale_reference = quant_config.w1_scale
|
||||
w2_scale_reference = quant_config.w2_scale
|
||||
assert w1_scale_reference is not None
|
||||
assert w2_scale_reference is not None
|
||||
w1_scale_linear = torch.stack(
|
||||
[
|
||||
convert_swizzled_to_linear(scale, w1_q.size(1), k, quant_blocksize)
|
||||
for scale in w1_scale_reference
|
||||
]
|
||||
)
|
||||
w2_scale_linear = torch.stack(
|
||||
[
|
||||
convert_swizzled_to_linear(scale, w2_q.size(1), n, quant_blocksize)
|
||||
for scale in w2_scale_reference
|
||||
]
|
||||
)
|
||||
w1_q_for_kernel = w1_q
|
||||
if is_gated_act:
|
||||
w1_q_for_kernel, w1_scale_linear = reorder_w1w3_to_w3w1(
|
||||
w1_q.clone(), w1_scale_linear
|
||||
)
|
||||
w1_q, w1_scale, w2_q, w2_scale = prepare_static_weights_for_trtllm_fp4_moe(
|
||||
w1_q_for_kernel,
|
||||
w2_q,
|
||||
w1_scale_linear,
|
||||
w2_scale_linear,
|
||||
hidden_size=k,
|
||||
intermediate_size=n,
|
||||
num_experts=e,
|
||||
is_gated_activation=is_gated_act,
|
||||
)
|
||||
quant_config._w1.scale = w1_scale
|
||||
quant_config._w2.scale = w2_scale
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=activation,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=dtype,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
activation_situ_beta=4.0 if activation == MoEActivation.SITU else None,
|
||||
activation_situ_linear_beta=(
|
||||
25.0 if activation == MoEActivation.SITU else None
|
||||
),
|
||||
)
|
||||
|
||||
trtllm_experts = _make_trtllm_fp4_moe_kernel(
|
||||
moe_config,
|
||||
quant_config,
|
||||
per_token_activation=per_token_activation,
|
||||
)
|
||||
if activation == MoEActivation.SITU:
|
||||
torch.testing.assert_close(
|
||||
trtllm_experts.fused_experts.g1_scale_c, quant_config.a2_gscale
|
||||
)
|
||||
|
||||
trtllm_output = trtllm_experts.apply(
|
||||
hidden_states=a,
|
||||
w1=w1_q,
|
||||
w2=w2_q,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
# Reference: round-trip activations and weights through FP4
|
||||
# quant/dequant so the comparison isolates kernel/activation behavior
|
||||
# from quantization error.
|
||||
if per_token_activation:
|
||||
from flashinfer import SfLayout, nvfp4_quantize
|
||||
|
||||
a_fp4, a_scale_interleaved, per_token_scale = nvfp4_quantize(
|
||||
a,
|
||||
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
|
||||
sfLayout=SfLayout.layout_linear,
|
||||
per_token_activation=True,
|
||||
)
|
||||
a_global_scale = 1.0 / per_token_scale.unsqueeze(1)
|
||||
else:
|
||||
assert quant_config.a1_gscale is not None
|
||||
a_global_scale = quant_config.a1_gscale[0]
|
||||
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a, a_global_scale)
|
||||
a_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
a_fp4,
|
||||
a_scale_interleaved,
|
||||
a_global_scale,
|
||||
dtype=a.dtype,
|
||||
device=a.device,
|
||||
block_size=quant_blocksize,
|
||||
is_sf_linear_layout=per_token_activation,
|
||||
)
|
||||
|
||||
w1_d = torch.empty(
|
||||
(e, (2 if is_gated_act else 1) * n, k), device="cuda", dtype=dtype
|
||||
)
|
||||
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
|
||||
for idx in range(e):
|
||||
w1_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w1_q_linear[idx],
|
||||
w1_scale_reference[idx],
|
||||
(1 / quant_config.g1_alphas[idx]),
|
||||
dtype=dtype,
|
||||
device=w1_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
w2_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w2_q_linear[idx],
|
||||
w2_scale_reference[idx],
|
||||
(1 / quant_config.g2_alphas[idx]),
|
||||
dtype=dtype,
|
||||
device=w2_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
|
||||
torch_output = torch_moe(
|
||||
a_in_dtype, w1_d, w2_d, score, topk, activation=torch_activation
|
||||
)
|
||||
|
||||
if swiglu_limit is None:
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
torch_output.float().flatten(),
|
||||
trtllm_output.float().flatten(),
|
||||
dim=0,
|
||||
)
|
||||
assert cosine > 0.99
|
||||
torch.testing.assert_close(torch_output, trtllm_output, atol=2e-1, rtol=2e-1)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_fp4_moe_reprocess_does_not_refold_swiglu_params():
|
||||
"""Weight reloads rerun post-processing on the same experts object; the
|
||||
g1_alphas fold of clamp/beta must start from the unfolded values."""
|
||||
e, n, k = 8, 256, 256
|
||||
clamp, beta = 7.0, 1.0
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
_, _, quant_config = make_test_quant_config(
|
||||
e,
|
||||
n,
|
||||
k,
|
||||
in_dtype=torch.bfloat16,
|
||||
quant_dtype="nvfp4",
|
||||
block_shape=None,
|
||||
per_act_token_quant=False,
|
||||
is_scale_swizzled=False,
|
||||
)
|
||||
quant_config.gemm1_clamp_limit = clamp
|
||||
quant_config.gemm1_beta = beta
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=2,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=16,
|
||||
)
|
||||
experts = TrtLlmNvFp4ExpertsModular(
|
||||
moe_config=moe_config, quant_config=quant_config
|
||||
)
|
||||
g1_alphas = quant_config.g1_alphas
|
||||
assert g1_alphas is not None
|
||||
layer = torch.nn.Module()
|
||||
layer.w13_weight_scale_2 = g1_alphas
|
||||
layer.w2_weight_scale_2 = quant_config.g2_alphas
|
||||
layer.w13_input_scale = torch.ones_like(g1_alphas)
|
||||
layer.w2_input_scale = torch.ones_like(g1_alphas)
|
||||
|
||||
for new_alphas in (0.5, 0.25):
|
||||
g1_alphas.fill_(new_alphas)
|
||||
experts.process_weights_after_loading(layer)
|
||||
expected = torch.full_like(g1_alphas, 1.0 / new_alphas)
|
||||
torch.testing.assert_close(experts.gemm1_clamp_limit, clamp * expected)
|
||||
torch.testing.assert_close(experts.gemm1_beta, beta * expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,chunk_size", [(1, None), (16, None), (16, 5)])
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_fp4_moe_deferred_finalize(
|
||||
m: int,
|
||||
chunk_size: int | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
workspace_init,
|
||||
):
|
||||
"""TRTLLM-Gen NvFP4 modular experts can leave the top-k finalize to the caller."""
|
||||
e, topk, n, k = 128, 8, 1024, 1024
|
||||
dtype = torch.bfloat16
|
||||
if chunk_size is not None:
|
||||
monkeypatch.setattr(
|
||||
TrtLlmNvFp4ExpertsModular, "_get_chunk_size", lambda self: chunk_size
|
||||
)
|
||||
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
w1_q, w2_q, quant_config = make_test_quant_config(
|
||||
e, n, k, in_dtype=dtype, quant_dtype="nvfp4", is_scale_swizzled=False
|
||||
)
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
# One rank of a TP group, which deferral needs.
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=2 * n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
moe_parallel_config=replace(
|
||||
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
|
||||
),
|
||||
in_dtype=dtype,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
kernel = _make_trtllm_fp4_moe_kernel(moe_config, quant_config)
|
||||
|
||||
check_deferred_moe_finalize(
|
||||
moe_config,
|
||||
lambda: kernel.apply(
|
||||
hidden_states=a,
|
||||
w1=w1_q,
|
||||
w2=w2_q,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
),
|
||||
router_weights=topk_weights,
|
||||
chunked=chunk_size is not None,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_trtllm_fp4_moe_no_graph(
|
||||
64,
|
||||
704,
|
||||
4096,
|
||||
128,
|
||||
8,
|
||||
torch.bfloat16,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU,
|
||||
None,
|
||||
False,
|
||||
None,
|
||||
)
|
||||
@@ -1,642 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
select_unquantized_moe_backend,
|
||||
)
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
skipif_not_cuda_rocm = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_rocm()),
|
||||
reason="Only supported on CUDA/ROCm platforms.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("amx_supported", "in_dtype", "expect_amx_kernel"),
|
||||
[
|
||||
(True, torch.bfloat16, True),
|
||||
(False, torch.bfloat16, False),
|
||||
(True, torch.float16, False),
|
||||
],
|
||||
)
|
||||
def test_x86_cpu_unquantized_kernel_selection(
|
||||
amx_supported: bool,
|
||||
in_dtype: torch.dtype,
|
||||
expect_amx_kernel: bool,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUUnquantizedExperts,
|
||||
X86CPUUnquantizedExperts,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(current_platform, "is_cpu", return_value=True),
|
||||
patch.object(
|
||||
current_platform,
|
||||
"get_cpu_architecture",
|
||||
return_value=CpuArchEnum.X86,
|
||||
),
|
||||
patch("torch.cpu._is_amx_tile_supported", return_value=amx_supported),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(
|
||||
hidden_dim=128,
|
||||
intermediate_size=128,
|
||||
in_dtype=in_dtype,
|
||||
)
|
||||
kernel_cls = next(
|
||||
cls
|
||||
for cls in backend_to_kernel_cls(UnquantizedMoeBackend.CPU)
|
||||
if cls.is_supported_config(
|
||||
cls,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
CPUUnquantizedExperts.activation_format(),
|
||||
)[0]
|
||||
)
|
||||
|
||||
expected_kernel_cls = (
|
||||
X86CPUUnquantizedExperts if expect_amx_kernel else CPUUnquantizedExperts
|
||||
)
|
||||
assert kernel_cls is expected_kernel_cls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "in_dtype", "expect_arm_kernel"),
|
||||
[
|
||||
("linux", torch.bfloat16, True),
|
||||
("linux", torch.float16, False),
|
||||
("darwin", torch.bfloat16, False),
|
||||
],
|
||||
)
|
||||
def test_arm_cpu_unquantized_kernel_selection(
|
||||
platform: str,
|
||||
in_dtype: torch.dtype,
|
||||
expect_arm_kernel: bool,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ArmCPUUnquantizedExperts,
|
||||
CPUUnquantizedExperts,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(current_platform, "is_cpu", return_value=True),
|
||||
patch.object(
|
||||
current_platform,
|
||||
"get_cpu_architecture",
|
||||
return_value=CpuArchEnum.ARM,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.cpu_moe.sys.platform",
|
||||
platform,
|
||||
),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(
|
||||
hidden_dim=128,
|
||||
intermediate_size=128,
|
||||
in_dtype=in_dtype,
|
||||
)
|
||||
kernel_cls = next(
|
||||
cls
|
||||
for cls in backend_to_kernel_cls(UnquantizedMoeBackend.CPU)
|
||||
if cls.is_supported_config(
|
||||
cls,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
CPUUnquantizedExperts.activation_format(),
|
||||
)[0]
|
||||
)
|
||||
|
||||
expected_kernel_cls = (
|
||||
ArmCPUUnquantizedExperts if expect_arm_kernel else CPUUnquantizedExperts
|
||||
)
|
||||
assert kernel_cls is expected_kernel_cls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"platform_method,expected_backend",
|
||||
[
|
||||
("is_cuda", UnquantizedMoeBackend.TRITON), # Default CUDA without FlashInfer
|
||||
("is_rocm", UnquantizedMoeBackend.TRITON), # ROCm without AITER
|
||||
("is_cpu", UnquantizedMoeBackend.CPU),
|
||||
("is_xpu", UnquantizedMoeBackend.XPU),
|
||||
("is_tpu", UnquantizedMoeBackend.TPU),
|
||||
("is_out_of_tree", UnquantizedMoeBackend.OOT),
|
||||
],
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe."
|
||||
"has_flashinfer_cutlass_fused_moe",
|
||||
return_value=False,
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe."
|
||||
"has_flashinfer_trtllm_fused_moe",
|
||||
return_value=False,
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops.is_fused_moe_enabled",
|
||||
return_value=False,
|
||||
)
|
||||
def test_select_default_backend_by_platform(
|
||||
mock_aiter_enabled,
|
||||
mock_has_flashinfer_trtllm,
|
||||
mock_has_flashinfer_cutlass,
|
||||
monkeypatch,
|
||||
platform_method,
|
||||
expected_backend,
|
||||
):
|
||||
"""Test default backend selection per platform with all optional
|
||||
accelerators (FlashInfer, AITER) disabled.
|
||||
|
||||
Patch the feature probes directly because their cached results can outlive
|
||||
changes to has_flashinfer().
|
||||
"""
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
|
||||
) as mock_platform:
|
||||
# Set all platform checks to False
|
||||
mock_platform.is_cuda.return_value = False
|
||||
mock_platform.is_rocm.return_value = False
|
||||
mock_platform.is_cpu.return_value = False
|
||||
mock_platform.is_xpu.return_value = False
|
||||
mock_platform.is_tpu.return_value = False
|
||||
mock_platform.is_out_of_tree.return_value = False
|
||||
|
||||
# Set only the specified platform to True
|
||||
getattr(mock_platform, platform_method).return_value = True
|
||||
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=False),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(current_platform, platform_method, return_value=True),
|
||||
):
|
||||
# CPU's grouped-gemm kernels require hidden/intermediate sizes
|
||||
# aligned to 32; the size-1 defaults only work for backends that
|
||||
# don't check shapes at selection time.
|
||||
moe_config = (
|
||||
make_dummy_moe_config(hidden_dim=128, intermediate_size=128)
|
||||
if expected_backend == UnquantizedMoeBackend.CPU
|
||||
else make_dummy_moe_config()
|
||||
)
|
||||
selected_backend, expert_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == expected_backend
|
||||
if expected_backend in [
|
||||
UnquantizedMoeBackend.OOT,
|
||||
UnquantizedMoeBackend.TPU,
|
||||
]:
|
||||
assert expert_cls is None
|
||||
else:
|
||||
assert expert_cls is not None
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.has_flashinfer",
|
||||
return_value=False,
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops.is_fused_moe_enabled",
|
||||
return_value=True,
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific backend selection test"
|
||||
)
|
||||
def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer):
|
||||
"""Test ROCm backend selection when AITER is available."""
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
|
||||
) as mock_platform:
|
||||
mock_platform.is_cuda.return_value = False
|
||||
mock_platform.is_rocm.return_value = True
|
||||
mock_platform.is_cpu.return_value = False
|
||||
mock_platform.is_xpu.return_value = False
|
||||
mock_platform.is_tpu.return_value = False
|
||||
mock_platform.is_out_of_tree.return_value = False
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
selected_backend, expert_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config,
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.AITER
|
||||
assert expert_cls is not None
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.has_flashinfer",
|
||||
return_value=False,
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops."
|
||||
"is_fused_moe_enabled",
|
||||
return_value=True,
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops."
|
||||
"is_rdna_aiter_enabled",
|
||||
return_value=False,
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific backend selection test"
|
||||
)
|
||||
def test_select_rocm_aiter_backend_non_gated_activation_falls_back(
|
||||
mock_rdna_disabled, mock_aiter_enabled, mock_has_flashinfer, monkeypatch
|
||||
):
|
||||
"""Test ROCm backend selection falls back (not raises) for non-gated MoE."""
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
|
||||
) as mock_platform:
|
||||
mock_platform.is_cuda.return_value = False
|
||||
mock_platform.is_rocm.return_value = True
|
||||
mock_platform.is_cpu.return_value = False
|
||||
mock_platform.is_xpu.return_value = False
|
||||
mock_platform.is_tpu.return_value = False
|
||||
mock_platform.is_out_of_tree.return_value = False
|
||||
|
||||
moe_config = make_dummy_moe_config(activation=MoEActivation.SILU_NO_MUL)
|
||||
assert moe_config.is_act_and_mul is False
|
||||
|
||||
selected_backend, expert_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config,
|
||||
)
|
||||
|
||||
assert selected_backend != UnquantizedMoeBackend.AITER
|
||||
assert expert_cls is not None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific backend selection test"
|
||||
)
|
||||
def test_explicit_aiter_backend_non_gated_activation_still_raises():
|
||||
"""Explicit `--moe-backend aiter` still raises for non-gated MoE;
|
||||
only the env-var opt-in path falls back."""
|
||||
moe_config = make_dummy_moe_config(activation=MoEActivation.SILU_NO_MUL)
|
||||
moe_config.moe_backend = "aiter" # explicit pin, not "auto"
|
||||
assert moe_config.is_act_and_mul is False
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
select_unquantized_moe_backend(moe_config=moe_config)
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
|
||||
return_value=(True, None),
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
|
||||
)
|
||||
def test_select_cuda_flashinfer_trtllm_backend(
|
||||
mock_is_supported_trtllm_monolithic,
|
||||
):
|
||||
"""Test CUDA backend selection when FlashInfer TRTLLM is available and enabled."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(current_platform, "has_device_capability", return_value=True),
|
||||
):
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_backend = "flashinfer_trtllm"
|
||||
# TRTLLM requires EP and does not support DP
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.use_dp = False
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
assert experts_cls is not None
|
||||
assert experts_cls.__name__ == "TrtLlmBf16ExpertsMonolithic"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
|
||||
return_value=(False, "monolithic unsupported"),
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsModular.is_supported_config",
|
||||
return_value=(True, None),
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
|
||||
)
|
||||
def test_select_cuda_flashinfer_trtllm_modular_backend(
|
||||
mock_is_supported_trtllm_modular,
|
||||
mock_is_supported_trtllm_monolithic,
|
||||
):
|
||||
"""Test CUDA backend selection falls back to FlashInfer TRTLLM modular."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(current_platform, "has_device_capability", return_value=True),
|
||||
):
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_backend = "flashinfer_trtllm"
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.use_dp = False
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
assert experts_cls is not None
|
||||
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"all2all_backend",
|
||||
[
|
||||
"mori_high_throughput",
|
||||
"mori_low_latency",
|
||||
"flashinfer_nvlink_two_sided",
|
||||
"flashinfer_nvlink_one_sided",
|
||||
],
|
||||
)
|
||||
def test_select_cuda_flashinfer_trtllm_modular_for_standard_all2all(
|
||||
mock_supports_current_device,
|
||||
all2all_backend,
|
||||
):
|
||||
"""Test non-AG/RS standard-format all2all backends select modular BF16."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(
|
||||
current_platform, "is_device_capability_family", return_value=False
|
||||
),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
|
||||
moe_config.moe_backend = "flashinfer_trtllm"
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
moe_config.moe_parallel_config.ep_size = 2
|
||||
moe_config.moe_parallel_config.all2all_backend = all2all_backend
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
assert experts_cls is not None
|
||||
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
)
|
||||
@pytest.mark.parametrize("require_decomposed_backend", [False, True])
|
||||
def test_require_decomposed_backend_rejects_monolithic_kernels(
|
||||
mock_supports_current_device,
|
||||
require_decomposed_backend,
|
||||
):
|
||||
"""Expert substitution adjusts routes, so the kernel must not route itself."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
TrtLlmBf16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEActivationFormat,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.routing_method = RoutingMethodType.Renormalize
|
||||
moe_config.require_decomposed_backend = require_decomposed_backend
|
||||
|
||||
supported, reason = TrtLlmBf16ExpertsMonolithic.is_supported_config(
|
||||
TrtLlmBf16ExpertsMonolithic,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
|
||||
assert supported != require_decomposed_backend
|
||||
assert (reason is not None and "routes adjusted" in reason) == (
|
||||
require_decomposed_backend
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
)
|
||||
def test_select_cuda_deepep_ht_falls_back_from_trtllm(
|
||||
mock_supports_current_device,
|
||||
):
|
||||
"""Test DeepEP HT avoids the unsupported BF16 TRTLLM modular path."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(
|
||||
current_platform, "is_device_capability_family", return_value=False
|
||||
),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
|
||||
moe_config.moe_backend = "auto"
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
moe_config.moe_parallel_config.ep_size = 2
|
||||
moe_config.moe_parallel_config.all2all_backend = "deepep_high_throughput"
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
assert experts_cls.__name__ == "TritonExperts"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"routing_method",
|
||||
[RoutingMethodType.Renormalize, RoutingMethodType.MiniMax2],
|
||||
)
|
||||
@pytest.mark.parametrize("moe_backend", ["auto", "flashinfer_trtllm"])
|
||||
def test_select_cuda_flashinfer_trtllm_ag_rs_uses_monolithic(
|
||||
mock_supports_current_device,
|
||||
routing_method,
|
||||
moe_backend,
|
||||
):
|
||||
"""Test AG/RS stays on BF16 TRTLLM monolithic when TRTLLM is supported."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(
|
||||
current_platform, "is_device_capability_family", return_value=False
|
||||
),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
|
||||
moe_config.moe_backend = moe_backend
|
||||
moe_config.routing_method = routing_method
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
moe_config.moe_parallel_config.ep_size = 2
|
||||
moe_config.moe_parallel_config.all2all_backend = "allgather_reducescatter"
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
assert experts_cls is not None
|
||||
assert experts_cls.__name__ == "TrtLlmBf16ExpertsMonolithic"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.has_flashinfer",
|
||||
return_value=True,
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
|
||||
return_value=(False, None),
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsModular.is_supported_config",
|
||||
return_value=(False, None),
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe.FlashInferExperts.is_supported_config",
|
||||
return_value=(True, None),
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
|
||||
)
|
||||
def test_select_cuda_flashinfer_cutlass_backend(
|
||||
mock_is_supported_cutlass,
|
||||
mock_is_supported_trtllm_modular,
|
||||
mock_is_supported_trtllm_monolithic,
|
||||
mock_has_flashinfer,
|
||||
):
|
||||
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
|
||||
and FlashInfer CUTLASS is available."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(current_platform, "has_device_capability", return_value=True),
|
||||
):
|
||||
moe_config = make_dummy_moe_config()
|
||||
# Select FlashInfer CUTLASS explicitly
|
||||
moe_config.moe_backend = "flashinfer_cutlass"
|
||||
# CUTLASS requires EP and does not support DP
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.use_dp = False
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@skipif_not_cuda_rocm
|
||||
def test_select_lora_backend_prefers_triton():
|
||||
"""LoRA-enabled unquantized MoE should select Triton backend."""
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.is_lora_enabled = True
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@skipif_not_cuda_rocm
|
||||
def test_select_lora_explicit_non_triton_backend():
|
||||
"""LoRA should override explicit non-Triton backend to Triton."""
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.is_lora_enabled = True
|
||||
|
||||
# Use string from mapping in function map_unquantized_backend()
|
||||
moe_config.moe_backend = "flashinfer_cutlass"
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@skipif_not_cuda_rocm
|
||||
@pytest.mark.parametrize("is_lora_enabled", [False, True])
|
||||
def test_select_explicit_triton_backend(is_lora_enabled):
|
||||
"""Explicit triton backend selection should return Triton."""
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.is_lora_enabled = is_lora_enabled
|
||||
moe_config.moe_backend = "triton"
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@skipif_not_cuda_rocm
|
||||
@pytest.mark.parametrize("moe_backend", ["humming", "marlin"])
|
||||
def test_quantization_only_backend_falls_back_to_auto(moe_backend):
|
||||
"""A quantization-only --moe-backend also reaches the unquantized MoE
|
||||
layers of a quantized model (e.g. a BF16 MTP draft), which must use the
|
||||
auto-selected backend instead of failing."""
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_backend = moe_backend
|
||||
|
||||
assert select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
) == select_unquantized_moe_backend(moe_config=make_dummy_moe_config())
|
||||
@@ -1,211 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ZenTorch CPU fused MoE dispatch and forward."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
|
||||
from tests.kernels.moe.test_cpu_fused_moe import (
|
||||
UNALIGNED_INTERMEDIATE_DIM,
|
||||
_make_moe_config,
|
||||
_StubMoELayer,
|
||||
ref_fused_moe,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import (
|
||||
has_zentorch_op,
|
||||
is_zentorch_moe_config_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
biased_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUUnquantizedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_cpu() or not current_platform.is_zen_cpu():
|
||||
pytest.skip("skipping non-Zen CPU tests", allow_module_level=True)
|
||||
|
||||
if not has_zentorch_op(["zentorch_fused_moe"]):
|
||||
pytest.skip(
|
||||
"skipping tests: zentorch_fused_moe op not available",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
ZENTORCH_ACT = [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
]
|
||||
USE_BIAS = [False, True]
|
||||
|
||||
|
||||
def _make_layer_and_config(
|
||||
act: MoEActivation,
|
||||
*,
|
||||
hidden_size: int = 128,
|
||||
intermediate_size: int = 128,
|
||||
expert_num: int = 8,
|
||||
topk_num: int = 4,
|
||||
use_bias: bool = False,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
) -> tuple[_StubMoELayer, FusedMoEConfig]:
|
||||
up_dim = 2 * intermediate_size
|
||||
w13 = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w2 = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / (
|
||||
0.5 * intermediate_size**0.5
|
||||
)
|
||||
w13_bias = None
|
||||
w2_bias = None
|
||||
if use_bias:
|
||||
w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
|
||||
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
|
||||
layer = _StubMoELayer(
|
||||
w13,
|
||||
w2,
|
||||
act,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
moe_config = _make_moe_config(
|
||||
expert_num,
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
topk_num,
|
||||
dtype,
|
||||
act,
|
||||
use_bias,
|
||||
)
|
||||
return layer, moe_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("act", ZENTORCH_ACT)
|
||||
def test_zen_cpu_fused_moe_config_supported(act: MoEActivation):
|
||||
"""CPUUnquantizedExperts is selected when zentorch MoE is supported."""
|
||||
_, moe_config = _make_layer_and_config(act)
|
||||
assert is_zentorch_moe_config_supported(moe_config)
|
||||
|
||||
supported, reason = CPUUnquantizedExperts.is_supported_config(
|
||||
CPUUnquantizedExperts,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported, reason
|
||||
|
||||
|
||||
def test_zen_cpu_fused_moe_config_supported_unaligned_intermediate():
|
||||
"""Zentorch accepts shapes the grouped-gemm kernel rejects."""
|
||||
act = MoEActivation.SILU
|
||||
moe_config = _make_moe_config(
|
||||
expert_num=8,
|
||||
hidden_size=128,
|
||||
intermediate_size=UNALIGNED_INTERMEDIATE_DIM,
|
||||
topk_num=4,
|
||||
dtype=torch.bfloat16,
|
||||
act=act,
|
||||
)
|
||||
assert is_zentorch_moe_config_supported(moe_config)
|
||||
|
||||
supported, reason = CPUUnquantizedExperts.is_supported_config(
|
||||
CPUUnquantizedExperts,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("act", ZENTORCH_ACT)
|
||||
@pytest.mark.parametrize("use_bias", USE_BIAS)
|
||||
def test_zen_cpu_fused_moe_forward(
|
||||
default_vllm_config,
|
||||
act: MoEActivation,
|
||||
use_bias: bool,
|
||||
):
|
||||
"""The zentorch forward runs end-to-end and matches the reference."""
|
||||
set_random_seed(0)
|
||||
|
||||
expert_num = 8
|
||||
hidden_size = 128
|
||||
batch_size = 64
|
||||
dtype = torch.bfloat16
|
||||
topk_num = expert_num // 2
|
||||
|
||||
layer, moe_config = _make_layer_and_config(
|
||||
act,
|
||||
expert_num=expert_num,
|
||||
topk_num=topk_num,
|
||||
use_bias=use_bias,
|
||||
)
|
||||
|
||||
input = torch.randn((batch_size, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
router_logits = torch.randn((batch_size, expert_num), dtype=dtype)
|
||||
|
||||
quant_config = (
|
||||
biased_moe_quant_config(layer.w13_bias, layer.w2_bias)
|
||||
if use_bias
|
||||
else FusedMoEQuantConfig.make()
|
||||
)
|
||||
experts = CPUUnquantizedExperts(moe_config, quant_config)
|
||||
experts.process_weights_after_loading(layer)
|
||||
assert experts._use_zentorch is True
|
||||
|
||||
w13_before = layer.w13_weight.detach().clone()
|
||||
w2_before = layer.w2_weight.detach().clone()
|
||||
|
||||
topk_weight, topk_ids = _softmax_topk(router_logits, topk_num, False)
|
||||
|
||||
output = torch.empty_like(input)
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=input,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_weights=topk_weight,
|
||||
topk_ids=topk_ids,
|
||||
activation=act,
|
||||
global_num_experts=expert_num,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=torch.empty(0),
|
||||
workspace2=torch.empty(0),
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
# zentorch needs no prepacking; weights are consumed as loaded.
|
||||
torch.testing.assert_close(layer.w13_weight, w13_before)
|
||||
torch.testing.assert_close(layer.w2_weight, w2_before)
|
||||
|
||||
ref_output = ref_fused_moe(
|
||||
input,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
getattr(layer, "w13_bias", None),
|
||||
getattr(layer, "w2_bias", None),
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
act,
|
||||
)
|
||||
|
||||
atol, rtol = get_default_atol(output), get_default_rtol(output)
|
||||
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
|
||||
@@ -1,252 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Zen CPU INT8 MoE experts (ZenCPUExpertsInt8)."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.test_cpu_fused_moe import (
|
||||
_make_moe_config,
|
||||
quantize_per_channel,
|
||||
ref_fused_moe_int8,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import (
|
||||
_ZENTORCH_MOE_ACTIVATIONS,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
int8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ZenCPUExpertsInt8
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
|
||||
Int8MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
EXPERT_NUM = 8
|
||||
HIDDEN_SIZE = 128
|
||||
INTERMEDIATE_SIZE = 128
|
||||
BATCH_SIZE = 32
|
||||
# zentorch names its activations as strings; the experts API is enum-typed.
|
||||
ZENTORCH_ACT = [act for act in MoEActivation if act.value in _ZENTORCH_MOE_ACTIVATIONS]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_zentorch_fused_moe():
|
||||
"""Reference zentorch_fused_moe, registered when zentorch is absent."""
|
||||
if hasattr(torch.ops.zentorch, "zentorch_fused_moe"):
|
||||
yield None
|
||||
return
|
||||
|
||||
calls: dict[str, dict] = {}
|
||||
|
||||
def _fused_moe(
|
||||
output,
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_id,
|
||||
skip_weighted,
|
||||
act,
|
||||
w13_scales=None,
|
||||
w2_scales=None,
|
||||
zentorch_op_name="",
|
||||
):
|
||||
calls["fused_moe"] = {
|
||||
"w13": w13,
|
||||
"w2": w2,
|
||||
"w13_bias": w13_bias,
|
||||
"w2_bias": w2_bias,
|
||||
"topk_weights": topk_weights,
|
||||
"topk_id": topk_id,
|
||||
"skip_weighted": skip_weighted,
|
||||
"act": act,
|
||||
"w13_scales": w13_scales,
|
||||
"w2_scales": w2_scales,
|
||||
}
|
||||
output.copy_(
|
||||
ref_fused_moe_int8(
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_scales.float(),
|
||||
w2_scales.float(),
|
||||
None if w13_bias is None else w13_bias.float(),
|
||||
None if w2_bias is None else w2_bias.float(),
|
||||
topk_weights,
|
||||
topk_id,
|
||||
MoEActivation(act),
|
||||
)
|
||||
)
|
||||
|
||||
lib_def = torch.library.Library("zentorch", "DEF")
|
||||
lib_def.define(
|
||||
"zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, "
|
||||
"Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, "
|
||||
"Tensor topk_id, bool skip_weighted, str act, "
|
||||
"Tensor? w13_scales=None, Tensor? w2_scales=None, *, "
|
||||
"str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()"
|
||||
)
|
||||
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
|
||||
lib_impl.impl("zentorch_fused_moe", _fused_moe)
|
||||
|
||||
yield calls
|
||||
|
||||
lib_impl._destroy()
|
||||
lib_def._destroy()
|
||||
|
||||
|
||||
def _make_int8_moe_weights(use_bias: bool, dtype: torch.dtype = torch.bfloat16):
|
||||
up_dim = 2 * INTERMEDIATE_SIZE
|
||||
w13_fp = torch.randn((EXPERT_NUM, up_dim, HIDDEN_SIZE), dtype=dtype) / (
|
||||
0.5 * HIDDEN_SIZE**0.5
|
||||
)
|
||||
w2_fp = torch.randn((EXPERT_NUM, HIDDEN_SIZE, INTERMEDIATE_SIZE), dtype=dtype) / (
|
||||
0.5 * INTERMEDIATE_SIZE**0.5
|
||||
)
|
||||
w13, w13_scale = quantize_per_channel(w13_fp)
|
||||
w2, w2_scale = quantize_per_channel(w2_fp)
|
||||
w13_scale = w13_scale.to(torch.bfloat16).float()
|
||||
w2_scale = w2_scale.to(torch.bfloat16).float()
|
||||
|
||||
w13_bias = w2_bias = None
|
||||
if use_bias:
|
||||
w13_bias = torch.randn((EXPERT_NUM, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
|
||||
w2_bias = torch.randn((EXPERT_NUM, HIDDEN_SIZE), dtype=dtype) / (
|
||||
0.5 * HIDDEN_SIZE**0.5
|
||||
)
|
||||
return w13, w2, w13_scale, w2_scale, w13_bias, w2_bias
|
||||
|
||||
|
||||
def test_zen_int8_first_in_cpu_backend_priority():
|
||||
"""The Zen experts are preferred over the generic CPU int8 experts."""
|
||||
assert backend_to_kernel_cls(Int8MoeBackend.CPU)[0] is ZenCPUExpertsInt8
|
||||
|
||||
|
||||
def test_zen_int8_support_predicates(mock_zentorch_fused_moe, monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: False)
|
||||
assert not ZenCPUExpertsInt8._supports_current_device()
|
||||
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
|
||||
assert ZenCPUExpertsInt8._supports_current_device()
|
||||
assert ZenCPUExpertsInt8._supports_quant_scheme(
|
||||
kInt8StaticChannelSym, kInt8DynamicTokenSym
|
||||
)
|
||||
assert not ZenCPUExpertsInt8._supports_quant_scheme(kInt8StaticChannelSym, None)
|
||||
for act in ZENTORCH_ACT:
|
||||
assert ZenCPUExpertsInt8._supports_activation(act)
|
||||
assert not ZenCPUExpertsInt8._supports_activation(MoEActivation.RELU2)
|
||||
assert not ZenCPUExpertsInt8._supports_activation(
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE
|
||||
)
|
||||
assert not ZenCPUExpertsInt8._supports_no_act_and_mul()
|
||||
assert ZenCPUExpertsInt8._supports_parallel_config(
|
||||
FusedMoEParallelConfig.make_no_parallel()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("act", ZENTORCH_ACT)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_zen_int8_dispatch_contract(
|
||||
default_vllm_config,
|
||||
mock_zentorch_fused_moe,
|
||||
act: MoEActivation,
|
||||
use_bias: bool,
|
||||
):
|
||||
"""apply() hands zentorch the weights, scales and biases it expects."""
|
||||
if mock_zentorch_fused_moe is None:
|
||||
pytest.skip("real zentorch is installed; the recorded call is unavailable")
|
||||
set_random_seed(0)
|
||||
|
||||
topk_num = EXPERT_NUM // 2
|
||||
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = _make_int8_moe_weights(use_bias)
|
||||
moe_config = _make_moe_config(
|
||||
EXPERT_NUM,
|
||||
HIDDEN_SIZE,
|
||||
INTERMEDIATE_SIZE,
|
||||
topk_num,
|
||||
torch.bfloat16,
|
||||
act,
|
||||
use_bias,
|
||||
)
|
||||
# A layer carries per-channel scales as [E, N, 1].
|
||||
quant_config = int8_w8a8_moe_quant_config(
|
||||
w1_scale=w13_scale.unsqueeze(-1),
|
||||
w2_scale=w2_scale.unsqueeze(-1),
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
w1_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
per_act_token_quant=True,
|
||||
)
|
||||
|
||||
input = torch.randn((BATCH_SIZE, HIDDEN_SIZE), dtype=torch.bfloat16) / (
|
||||
0.5 * HIDDEN_SIZE**0.5
|
||||
)
|
||||
router_logits = torch.randn((BATCH_SIZE, EXPERT_NUM), dtype=torch.bfloat16)
|
||||
topk_weights, topk_ids = _softmax_topk(router_logits, topk_num, False)
|
||||
|
||||
experts = ZenCPUExpertsInt8(moe_config, quant_config)
|
||||
# zentorch_fused_moe takes no expert_map.
|
||||
assert not experts.supports_expert_map()
|
||||
output = torch.empty_like(input)
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=input,
|
||||
w1=w13,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=act,
|
||||
global_num_experts=EXPERT_NUM,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=torch.empty(0),
|
||||
workspace2=torch.empty(0),
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
call = mock_zentorch_fused_moe["fused_moe"]
|
||||
assert call["w13_scales"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE)
|
||||
assert call["w2_scales"].shape == (EXPERT_NUM, HIDDEN_SIZE)
|
||||
# f32 scales keep the layer on zentorch's fast routed MoE executor.
|
||||
assert call["w13_scales"].dtype == torch.float32
|
||||
assert call["w2_scales"].dtype == torch.float32
|
||||
assert call["act"] == act.value.lower()
|
||||
assert call["skip_weighted"] is False
|
||||
assert call["topk_weights"].dtype == torch.float32
|
||||
assert call["topk_id"].dtype == torch.int32
|
||||
if use_bias:
|
||||
assert call["w13_bias"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE)
|
||||
assert call["w2_bias"].shape == (EXPERT_NUM, HIDDEN_SIZE)
|
||||
else:
|
||||
assert call["w13_bias"] is None and call["w2_bias"] is None
|
||||
|
||||
ref = ref_fused_moe_int8(
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
None if w13_bias is None else w13_bias.float(),
|
||||
None if w2_bias is None else w2_bias.float(),
|
||||
topk_weights.to(torch.float32),
|
||||
topk_ids.to(torch.int32),
|
||||
act,
|
||||
)
|
||||
torch.testing.assert_close(output, ref)
|
||||
@@ -1,658 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from tests.kernels.quant_utils import per_block_cast_to_int8
|
||||
from tests.kernels.quantization.nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
NaiveBatchedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
|
||||
TritonExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
fused_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
|
||||
BatchedPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
ref_nvfp4_quant,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import per_block_cast_to_fp8
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
|
||||
"""Fold weights to adjacent locations for Triton MoE / SwiGLU kernel layout."""
|
||||
shape = w.shape
|
||||
n = shape[-1]
|
||||
first = w[..., : n // 2]
|
||||
second = w[..., n // 2 :]
|
||||
stacked = torch.stack((first, second), dim=-1)
|
||||
return stacked.reshape(shape)
|
||||
|
||||
|
||||
def make_dummy_moe_config(
|
||||
num_experts: int = 1,
|
||||
num_local_experts: int | None = None,
|
||||
experts_per_token: int = 1,
|
||||
hidden_dim: int = 1,
|
||||
intermediate_size: int = 1,
|
||||
in_dtype: torch.dtype = torch.bfloat16,
|
||||
max_num_tokens: int = 512,
|
||||
activation: MoEActivation = MoEActivation.SILU,
|
||||
) -> FusedMoEConfig:
|
||||
"""This is a dummy config for the mk constructor interface
|
||||
as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN
|
||||
do not actually use this config.
|
||||
|
||||
CUTLASSFp8 needs to set some params for workshapes.
|
||||
"""
|
||||
return FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=experts_per_token,
|
||||
hidden_dim=hidden_dim,
|
||||
intermediate_size=intermediate_size,
|
||||
num_local_experts=num_local_experts
|
||||
if num_local_experts is not None
|
||||
else num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=activation,
|
||||
in_dtype=in_dtype,
|
||||
device=DEVICE,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=max_num_tokens,
|
||||
)
|
||||
|
||||
|
||||
def triton_moe(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
a1_scale: torch.Tensor | None = None,
|
||||
a2_scale: torch.Tensor | None = None,
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
per_act_token_quant=False,
|
||||
block_shape: list[int] | None = None,
|
||||
) -> torch.Tensor:
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
|
||||
return fused_experts(a, w1, w2, topk_weight, topk_ids, quant_config=quant_config)
|
||||
|
||||
|
||||
def batched_moe(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
a1_scale: torch.Tensor | None = None,
|
||||
a2_scale: torch.Tensor | None = None,
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
block_shape: list[int] | None = None,
|
||||
moe_config: FusedMoEConfig | None = None,
|
||||
) -> torch.Tensor:
|
||||
max_num_tokens = round_up(a.shape[0], 64)
|
||||
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
|
||||
if moe_config is None:
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
fused_experts = FusedMoEKernel(
|
||||
BatchedPrepareAndFinalize(
|
||||
max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0
|
||||
),
|
||||
BatchedTritonExperts(
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=1,
|
||||
quant_config=quant_config,
|
||||
moe_config=moe_config,
|
||||
),
|
||||
)
|
||||
|
||||
return fused_experts.apply(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
global_num_experts=w1.shape[0],
|
||||
activation=moe_config.activation,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
|
||||
def naive_batched_moe(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
a1_scale: torch.Tensor | None = None,
|
||||
a2_scale: torch.Tensor | None = None,
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
block_shape: list[int] | None = None,
|
||||
) -> torch.Tensor:
|
||||
max_num_tokens = round_up(a.shape[0], 64)
|
||||
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
fused_experts = FusedMoEKernel(
|
||||
BatchedPrepareAndFinalize(
|
||||
max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0
|
||||
),
|
||||
NaiveBatchedExperts(
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=1,
|
||||
quant_config=quant_config,
|
||||
moe_config=moe_config,
|
||||
),
|
||||
)
|
||||
|
||||
return fused_experts.apply(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
global_num_experts=w1.shape[0],
|
||||
activation=moe_config.activation,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
|
||||
def chunk_scales(
|
||||
scales: torch.Tensor | None, start: int, end: int
|
||||
) -> torch.Tensor | None:
|
||||
if scales is not None:
|
||||
if scales.numel() == 1:
|
||||
return scales
|
||||
else:
|
||||
return scales[start:end]
|
||||
return None
|
||||
|
||||
|
||||
def make_quantized_test_activations(
|
||||
E: int,
|
||||
m: int,
|
||||
k: int,
|
||||
in_dtype: torch.dtype,
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
block_shape: list[int] | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
a = torch.randn((E, m, k), device=DEVICE, dtype=in_dtype) / 10
|
||||
a_q = a
|
||||
a_scale = None
|
||||
|
||||
if quant_dtype is not None:
|
||||
assert quant_dtype == torch.float8_e4m3fn or quant_dtype == torch.int8, (
|
||||
"only fp8/int8 supported"
|
||||
)
|
||||
a_q = torch.zeros_like(a, dtype=quant_dtype)
|
||||
a_scale_l = [None] * E
|
||||
for e in range(E):
|
||||
a_q[e], a_scale_l[e] = moe_kernel_quantize_input(
|
||||
a[e], None, quant_dtype, per_act_token_quant, block_shape
|
||||
)
|
||||
a_scale = torch.stack(a_scale_l)
|
||||
|
||||
if not per_act_token_quant and block_shape is None:
|
||||
a_scale = a_scale.view(E, 1, 1)
|
||||
|
||||
return a, a_q, a_scale
|
||||
|
||||
|
||||
def moe_quantize_weights_2d(
|
||||
w: torch.Tensor,
|
||||
w_s: torch.Tensor | None,
|
||||
quant_dtype: torch.dtype | str | None,
|
||||
per_token_quant: bool,
|
||||
block_shape: list[int] | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
|
||||
is_fp8_dtype = is_fp8(quant_dtype)
|
||||
assert is_fp8_dtype or quant_dtype == torch.int8 or quant_dtype == "nvfp4", (
|
||||
"only fp8/int8/nvfp4 supported"
|
||||
)
|
||||
|
||||
w_gs = None
|
||||
|
||||
if block_shape is not None:
|
||||
assert not per_token_quant
|
||||
if quant_dtype == torch.int8:
|
||||
w, w_s = per_block_cast_to_int8(w, block_shape)
|
||||
elif is_fp8_dtype:
|
||||
w, w_s = per_block_cast_to_fp8(w, block_shape)
|
||||
elif quant_dtype == "nvfp4":
|
||||
raise RuntimeError("blocked quantization not supported for nvfp4")
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
|
||||
else:
|
||||
if quant_dtype == torch.int8:
|
||||
w, w_s = ops.scaled_int8_quant(
|
||||
w, w_s, use_per_token_if_dynamic=per_token_quant
|
||||
)
|
||||
elif is_fp8_dtype:
|
||||
w, w_s = ops.scaled_fp8_quant(
|
||||
w, w_s, use_per_token_if_dynamic=per_token_quant
|
||||
)
|
||||
elif quant_dtype == "nvfp4":
|
||||
assert not per_token_quant
|
||||
w_amax = torch.abs(w).max().to(torch.float32)
|
||||
w_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w_amax
|
||||
if current_platform.is_rocm():
|
||||
w, w_s = _scaled_fp4_quant_emulated(w, w_gs)
|
||||
else:
|
||||
w, w_s = ops.scaled_fp4_quant(w, w_gs)
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
|
||||
|
||||
return w, w_s, w_gs
|
||||
|
||||
|
||||
def _pack_e2m1_fp4(fp4_values: torch.Tensor) -> torch.Tensor:
|
||||
assert fp4_values.shape[-1] % 2 == 0
|
||||
|
||||
abs_values = fp4_values.abs()
|
||||
codes = torch.empty_like(abs_values, dtype=torch.uint8)
|
||||
for code, value in enumerate((0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)):
|
||||
codes[abs_values == value] = code
|
||||
codes = codes | ((fp4_values < 0).to(torch.uint8) << 3)
|
||||
return codes[..., 0::2] | (codes[..., 1::2] << 4)
|
||||
|
||||
|
||||
def _scaled_fp4_quant_emulated(
|
||||
w: torch.Tensor, w_gs: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
fp4_values, w_s = ref_nvfp4_quant(w, w_gs, block_size=16)
|
||||
return _pack_e2m1_fp4(fp4_values), w_s.to(torch.float8_e4m3fn)
|
||||
|
||||
|
||||
def moe_quantize_weights(
|
||||
w: torch.Tensor,
|
||||
w_s: torch.Tensor | None,
|
||||
quant_dtype: torch.dtype | str | None,
|
||||
per_token_quant: bool,
|
||||
block_shape: list[int] | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
|
||||
assert w.dim() == 3
|
||||
e, rows, cols = w.shape
|
||||
w_l = [None] * e
|
||||
w_s_l = [None] * e
|
||||
w_gs_l = [None] * e
|
||||
for idx in range(e):
|
||||
w_l[idx], w_s_l[idx], w_gs_l[idx] = moe_quantize_weights_2d(
|
||||
w[idx], None, quant_dtype, per_token_quant, block_shape
|
||||
)
|
||||
|
||||
w = torch.stack(w_l)
|
||||
w_s = torch.stack(w_s_l)
|
||||
w_gs = torch.stack(w_gs_l) if e > 0 and w_gs_l[0] is not None else None
|
||||
|
||||
if w_s.ndim == 2:
|
||||
assert w_s.shape[-1] == 1
|
||||
w_s = w_s.view(-1, 1, 1)
|
||||
|
||||
if block_shape is not None:
|
||||
block_n, block_k = block_shape
|
||||
n_tiles = (rows + block_n - 1) // block_n
|
||||
k_tiles = (cols + block_k - 1) // block_k
|
||||
assert w_s.shape == (e, n_tiles, k_tiles)
|
||||
|
||||
return w, w_s, w_gs
|
||||
|
||||
|
||||
def make_test_weight(
|
||||
e: int,
|
||||
rows: int,
|
||||
cols: int,
|
||||
in_dtype: torch.dtype = torch.bfloat16,
|
||||
quant_dtype: torch.dtype | str | None = None,
|
||||
block_shape: list[int] | None = None,
|
||||
per_out_ch_quant: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
|
||||
w_16 = torch.randn((e, rows, cols), device=DEVICE, dtype=in_dtype) / 15
|
||||
|
||||
if quant_dtype is not None:
|
||||
w, w_s, w_gs = moe_quantize_weights(
|
||||
w_16, None, quant_dtype, per_out_ch_quant, block_shape
|
||||
)
|
||||
else:
|
||||
w = w_16
|
||||
w_s = None
|
||||
w_gs = None
|
||||
|
||||
return w_16, w, w_s, w_gs
|
||||
|
||||
|
||||
def make_test_weights(
|
||||
e: int,
|
||||
n: int,
|
||||
k: int,
|
||||
in_dtype: torch.dtype = torch.bfloat16,
|
||||
quant_dtype: torch.dtype | str | None = None,
|
||||
block_shape: list[int] | None = None,
|
||||
per_out_ch_quant: bool = False,
|
||||
make_gate: bool = True,
|
||||
) -> tuple[
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None],
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None],
|
||||
]:
|
||||
return (
|
||||
make_test_weight(
|
||||
e,
|
||||
(2 if make_gate else 1) * n,
|
||||
k,
|
||||
in_dtype,
|
||||
quant_dtype,
|
||||
block_shape,
|
||||
per_out_ch_quant,
|
||||
),
|
||||
make_test_weight(e, k, n, in_dtype, quant_dtype, block_shape, per_out_ch_quant),
|
||||
)
|
||||
|
||||
|
||||
def per_token_cast_to_fp8(
|
||||
x: torch.Tensor, block_size: int = 128
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert x.dim() == 2
|
||||
m, n = x.shape
|
||||
pad_size = (block_size - (n % block_size)) % block_size
|
||||
x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x
|
||||
x_view = x.view(m, -1, block_size)
|
||||
x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4)
|
||||
fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn)
|
||||
return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1)
|
||||
|
||||
|
||||
def make_test_quant_config(
|
||||
e: int,
|
||||
n: int,
|
||||
k: int,
|
||||
in_dtype: torch.dtype,
|
||||
quant_dtype: torch.dtype | str | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
block_shape: list[int] | None = None,
|
||||
make_gate: bool = True,
|
||||
is_scale_swizzled: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]:
|
||||
(_, w1, w1_s, w1_gs), (_, w2, w2_s, w2_gs) = make_test_weights(
|
||||
e,
|
||||
n,
|
||||
k,
|
||||
in_dtype,
|
||||
quant_dtype,
|
||||
per_out_ch_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
make_gate=make_gate,
|
||||
)
|
||||
|
||||
# Hacky/trivial scales for nvfp4.
|
||||
a1_gscale: torch.Tensor | None = None
|
||||
a2_gscale: torch.Tensor | None = None
|
||||
if quant_dtype == "nvfp4":
|
||||
a1_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
|
||||
a2_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
|
||||
a1_scale = a1_gscale
|
||||
a2_scale = a2_gscale
|
||||
else:
|
||||
a1_scale = None
|
||||
a2_scale = None
|
||||
|
||||
return (
|
||||
w1,
|
||||
w2,
|
||||
FusedMoEQuantConfig.make(
|
||||
quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
a1_gscale=a1_gscale,
|
||||
a2_gscale=a2_gscale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
# TODO: make sure this is handled properly
|
||||
g1_alphas=(1 / w1_gs) if w1_gs is not None else None,
|
||||
g2_alphas=(1 / w2_gs) if w2_gs is not None else None,
|
||||
is_scale_swizzled=is_scale_swizzled,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def fused_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
score: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool = False,
|
||||
quant_config: FusedMoEQuantConfig | None = None,
|
||||
global_num_experts: int = -1,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
topk_weights, topk_ids, _ = fused_topk(
|
||||
hidden_states, score.float(), topk, renormalize
|
||||
)
|
||||
return fused_experts(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=expert_map,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
|
||||
class BaselineMM(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
b: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
):
|
||||
super().__init__()
|
||||
self.b = torch.nn.Parameter(b.to(dtype=torch.float32))
|
||||
self.out_dtype = out_dtype
|
||||
|
||||
def forward(self, a: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
return torch.mm(a.to(dtype=torch.float32), self.b).to(self.out_dtype), None
|
||||
|
||||
|
||||
class BaselineSiluAndMul(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
return torch.nn.functional.silu(x[..., :d]) * x[..., d:]
|
||||
|
||||
|
||||
class TestMLP(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
):
|
||||
super().__init__()
|
||||
self.gate_up_proj = BaselineMM(w1, out_dtype)
|
||||
self.down_proj = BaselineMM(w2, out_dtype)
|
||||
self.act_fn = BaselineSiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
x, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(x)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
def modular_triton_fused_moe(
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> FusedMoEKernel:
|
||||
return FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
TritonExperts(moe_config, quant_config),
|
||||
)
|
||||
|
||||
|
||||
def check_accuracy(a, b, atol, rtol, percent):
|
||||
"""Allow a mismatch percentage of 1 - percent."""
|
||||
if torch.any(torch.isnan(a)):
|
||||
raise Exception("NaN in reference output")
|
||||
if torch.any(torch.isnan(b)):
|
||||
raise Exception("NaN in actual output")
|
||||
if torch.any(torch.isinf(a)):
|
||||
raise Exception("Inf in reference output")
|
||||
if torch.any(torch.isinf(b)):
|
||||
raise Exception("Inf in actual output")
|
||||
assert a.shape == b.shape, f"Shape mismatch: {a.shape} vs {b.shape}"
|
||||
|
||||
left = torch.abs(a - b)
|
||||
right = atol + rtol * torch.abs(b)
|
||||
count = torch.sum(left > right)
|
||||
mismatch_percent = count / a.numel()
|
||||
if mismatch_percent > 1 - percent:
|
||||
raise Exception(
|
||||
f"Mismatch percentage is {mismatch_percent:.4f} for rtol {rtol} "
|
||||
f"(threshold: {1 - percent:.4f})"
|
||||
)
|
||||
|
||||
|
||||
def check_deferred_moe_finalize(
|
||||
moe_config: FusedMoEConfig,
|
||||
run: Callable[[], torch.Tensor | UnfinalizedMoEOutput],
|
||||
router_weights: torch.Tensor | None = None,
|
||||
chunked: bool = False,
|
||||
) -> None:
|
||||
"""Check a kernel that defers its finalize against the finalize it skips.
|
||||
|
||||
``run`` calls the kernel on fixed inputs, first as built and then with
|
||||
``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must
|
||||
report truthfully. A deferred output reduced by ``moe_unpermute``, the
|
||||
TRT-LLM finalize kernel, must give the kernel's own finalized output bit for
|
||||
bit, and modular experts hand the router's weights back as-is. A call the
|
||||
experts split across kernel launches finalizes instead.
|
||||
"""
|
||||
# The deferred output views the router's buffer, so compare with a copy.
|
||||
expected_weights = None if router_weights is None else router_weights.clone()
|
||||
finalized = run()
|
||||
assert isinstance(finalized, torch.Tensor)
|
||||
moe_config.defer_moe_finalize()
|
||||
output = run()
|
||||
assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked
|
||||
if chunked:
|
||||
torch.testing.assert_close(output, finalized, atol=0, rtol=0)
|
||||
return
|
||||
|
||||
assert isinstance(output, UnfinalizedMoEOutput)
|
||||
if expected_weights is not None:
|
||||
torch.testing.assert_close(
|
||||
output.expert_weights, expected_weights, atol=0, rtol=0
|
||||
)
|
||||
reference = torch.empty_like(finalized)
|
||||
moe_unpermute(
|
||||
reference,
|
||||
output.gemm2_permuted,
|
||||
output.expert_weights.float(),
|
||||
output.expanded_idx_to_permuted_idx,
|
||||
# The kernel reads its valid-row count through this pointer either way.
|
||||
expert_first_token_offset=output.gemm2_permuted.new_full(
|
||||
(1,), output.gemm2_permuted.shape[0], dtype=torch.int64
|
||||
),
|
||||
)
|
||||
torch.testing.assert_close(reference, finalized, atol=0, rtol=0)
|
||||
|
||||
|
||||
def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8):
|
||||
"""Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs.
|
||||
|
||||
triton_kernels 3.8 returns layout instances; earlier versions return a
|
||||
(layout, opts) tuple.
|
||||
"""
|
||||
from triton_kernels.tensor_details import layout
|
||||
|
||||
from vllm.utils.import_utils import get_triton_kernels_version
|
||||
|
||||
if get_triton_kernels_version() == "3.8":
|
||||
w = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis)
|
||||
s = layout.make_default_matmul_mxfp4_w_scale_layout(
|
||||
mx_axis=mx_axis, num_warps=num_warps
|
||||
)
|
||||
return w, {}, s, {}
|
||||
|
||||
w, w_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis)
|
||||
s, s_opts = layout.make_default_matmul_mxfp4_w_scale_layout(
|
||||
mx_axis=mx_axis, num_warps=num_warps
|
||||
)
|
||||
return w, w_opts, s, s_opts
|
||||
Reference in New Issue
Block a user