fused_moe: add VLLM_TRITON_USE_TD tensor-descriptor path (#42436)

Signed-off-by: Artur Fierka <artur.fierka@intel.com>
Signed-off-by: Lena Onyshchenko <162571002+oonyshch@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Lena Onyshchenko <162571002+oonyshch@users.noreply.github.com>
This commit is contained in:
Artur Fierka
2026-07-29 13:15:10 +08:00
committed by GitHub
co-authored by Claude Lena Onyshchenko
parent 6f91edf96d
commit 6f00a1ae3b
4 changed files with 179 additions and 21 deletions
@@ -7,7 +7,9 @@ from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
@@ -39,6 +41,8 @@ from vllm.utils.math_utils import cdiv
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
logger = init_logger(__name__)
@triton.jit
def _count_expert_num_tokens(
@@ -585,3 +589,80 @@ def enable_swap_ab(BLOCK_SIZE_M: int, BLOCK_SIZE_N: int) -> bool:
and BLOCK_SIZE_M < 64
and BLOCK_SIZE_N >= 64
)
def moe_use_td_hw_supported() -> bool:
"""Whether the current device can run the TD (gather) path of
``fused_moe_kernel`` (ignores the ``VLLM_TRITON_USE_TD`` override).
The A-load uses ``tensor_descriptor.gather``, which lowers to the PTX
``tile::gather4`` instruction. That instruction is part of the
``tcgen05``/Tensor Memory (TMEM) family introduced with Blackwell and has
no Hopper (sm90) equivalent -- ptxas rejects it there ("Feature
'.tile::gather4 ...' requires .target sm_100 or higher"). Unlike
``scatter4``, ``gather4`` is supported across the whole sm100+ range
including consumer Blackwell (sm120/sm121): see triton-lang/triton#8498,
which enables ``gather4`` on sm120/sm121 while leaving ``scatter4``
unsupported there. So this gates on a blanket ``has_device_capability(100)``
rather than the sm100 *family* check used for the scatter store path.
"""
if current_platform.is_xpu():
return True
if current_platform.is_cuda():
return current_platform.has_device_capability(100)
return False
def resolve_moe_use_td() -> bool:
"""Tri-state resolver for ``VLLM_TRITON_USE_TD``.
Unset auto-selects the TD path on XPU only, mirroring the attention
dispatcher in ``triton_attn.py``. ``1``/``0`` force it on/off regardless
of hardware; forcing ``1`` where it cannot compile (see
``moe_use_td_hw_supported``) fails at ptxas. Blackwell CUDA (sm100+) can
compile it but is opt-in only, pending validation.
"""
override = envs.VLLM_TRITON_USE_TD
if override is None:
return current_platform.is_xpu()
return override
_warned_moe_use_td_ineffective = False
def warn_if_moe_use_td_ineffective(
active_backend: str, is_quantized: bool = False
) -> None:
"""One-shot warning when ``VLLM_TRITON_USE_TD`` is set but ignored.
Fires when the user set the env explicitly and either (a) the active
MoE backend is not the fused Triton kernel, or (b) the model is
quantized (the TD path falls back to the pointer path under any
quantization).
"""
global _warned_moe_use_td_ineffective
if _warned_moe_use_td_ineffective:
return
if envs.VLLM_TRITON_USE_TD is None:
return
is_triton = active_backend.upper() == "TRITON"
if is_triton and not is_quantized:
return
if not is_triton:
reason = (
f"the active MoE backend is {active_backend!r}; pass "
"`--moe-backend triton` to enable the tensor-descriptor path"
)
else:
reason = (
"the model uses quantized MoE weights; the TD path is "
"currently restricted to non-quantized weights and falls "
"back to the pointer path"
)
logger.warning(
"VLLM_TRITON_USE_TD is set to %s but %s.",
envs.VLLM_TRITON_USE_TD,
reason,
)
_warned_moe_use_td_ineffective = True