fused_moe: add VLLM_TRITON_USE_TD tensor-descriptor path (#42436)
Signed-off-by: Artur Fierka <artur.fierka@intel.com> Signed-off-by: Lena Onyshchenko <162571002+oonyshch@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Lena Onyshchenko <162571002+oonyshch@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Lena Onyshchenko
parent
6f91edf96d
commit
6f00a1ae3b
@@ -7,7 +7,9 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
@@ -39,6 +41,8 @@ from vllm.utils.math_utils import cdiv
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _count_expert_num_tokens(
|
||||
@@ -585,3 +589,80 @@ def enable_swap_ab(BLOCK_SIZE_M: int, BLOCK_SIZE_N: int) -> bool:
|
||||
and BLOCK_SIZE_M < 64
|
||||
and BLOCK_SIZE_N >= 64
|
||||
)
|
||||
|
||||
|
||||
def moe_use_td_hw_supported() -> bool:
|
||||
"""Whether the current device can run the TD (gather) path of
|
||||
``fused_moe_kernel`` (ignores the ``VLLM_TRITON_USE_TD`` override).
|
||||
|
||||
The A-load uses ``tensor_descriptor.gather``, which lowers to the PTX
|
||||
``tile::gather4`` instruction. That instruction is part of the
|
||||
``tcgen05``/Tensor Memory (TMEM) family introduced with Blackwell and has
|
||||
no Hopper (sm90) equivalent -- ptxas rejects it there ("Feature
|
||||
'.tile::gather4 ...' requires .target sm_100 or higher"). Unlike
|
||||
``scatter4``, ``gather4`` is supported across the whole sm100+ range
|
||||
including consumer Blackwell (sm120/sm121): see triton-lang/triton#8498,
|
||||
which enables ``gather4`` on sm120/sm121 while leaving ``scatter4``
|
||||
unsupported there. So this gates on a blanket ``has_device_capability(100)``
|
||||
rather than the sm100 *family* check used for the scatter store path.
|
||||
"""
|
||||
if current_platform.is_xpu():
|
||||
return True
|
||||
if current_platform.is_cuda():
|
||||
return current_platform.has_device_capability(100)
|
||||
return False
|
||||
|
||||
|
||||
def resolve_moe_use_td() -> bool:
|
||||
"""Tri-state resolver for ``VLLM_TRITON_USE_TD``.
|
||||
|
||||
Unset auto-selects the TD path on XPU only, mirroring the attention
|
||||
dispatcher in ``triton_attn.py``. ``1``/``0`` force it on/off regardless
|
||||
of hardware; forcing ``1`` where it cannot compile (see
|
||||
``moe_use_td_hw_supported``) fails at ptxas. Blackwell CUDA (sm100+) can
|
||||
compile it but is opt-in only, pending validation.
|
||||
"""
|
||||
override = envs.VLLM_TRITON_USE_TD
|
||||
if override is None:
|
||||
return current_platform.is_xpu()
|
||||
return override
|
||||
|
||||
|
||||
_warned_moe_use_td_ineffective = False
|
||||
|
||||
|
||||
def warn_if_moe_use_td_ineffective(
|
||||
active_backend: str, is_quantized: bool = False
|
||||
) -> None:
|
||||
"""One-shot warning when ``VLLM_TRITON_USE_TD`` is set but ignored.
|
||||
|
||||
Fires when the user set the env explicitly and either (a) the active
|
||||
MoE backend is not the fused Triton kernel, or (b) the model is
|
||||
quantized (the TD path falls back to the pointer path under any
|
||||
quantization).
|
||||
"""
|
||||
global _warned_moe_use_td_ineffective
|
||||
if _warned_moe_use_td_ineffective:
|
||||
return
|
||||
if envs.VLLM_TRITON_USE_TD is None:
|
||||
return
|
||||
is_triton = active_backend.upper() == "TRITON"
|
||||
if is_triton and not is_quantized:
|
||||
return
|
||||
if not is_triton:
|
||||
reason = (
|
||||
f"the active MoE backend is {active_backend!r}; pass "
|
||||
"`--moe-backend triton` to enable the tensor-descriptor path"
|
||||
)
|
||||
else:
|
||||
reason = (
|
||||
"the model uses quantized MoE weights; the TD path is "
|
||||
"currently restricted to non-quantized weights and falls "
|
||||
"back to the pointer path"
|
||||
)
|
||||
logger.warning(
|
||||
"VLLM_TRITON_USE_TD is set to %s but %s.",
|
||||
envs.VLLM_TRITON_USE_TD,
|
||||
reason,
|
||||
)
|
||||
_warned_moe_use_td_ineffective = True
|
||||
|
||||
Reference in New Issue
Block a user