[Kernel] TD operand loads for batched MoE GEMM (moe_mmk) on XPU (#46340)
Signed-off-by: oonyshch <xonyshch@gmail.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
d30b1ecd1b
commit
0111002323
@@ -351,3 +351,187 @@ def test_fused_moe_batched_experts(
|
||||
torch.testing.assert_close(batched_output, baseline_output, atol=3e-2, rtol=2e-2)
|
||||
|
||||
torch.testing.assert_close(triton_output, batched_output, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
# USE_TD in moe_mmk
|
||||
|
||||
# K % 8 == 0 (bf16, 16-byte alignment)
|
||||
_TD_SHAPES = [
|
||||
(4, 1, 128, 256),
|
||||
(8, 64, 512, 512),
|
||||
(4, 256, 1024, 2048),
|
||||
]
|
||||
|
||||
|
||||
def _td_supported() -> bool:
|
||||
return current_platform.is_xpu() or (
|
||||
current_platform.is_cuda() and current_platform.has_device_capability(90)
|
||||
)
|
||||
|
||||
|
||||
def _run_td(A, B, num_expert_tokens, use_td: bool):
|
||||
out = torch.zeros(
|
||||
A.shape[0],
|
||||
A.shape[1],
|
||||
B.shape[1],
|
||||
dtype=torch.bfloat16,
|
||||
device=A.device,
|
||||
)
|
||||
import triton
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
batched_triton_kernel,
|
||||
)
|
||||
|
||||
if use_td:
|
||||
from vllm.triton_utils.allocation import set_triton_allocator
|
||||
|
||||
set_triton_allocator(A.device)
|
||||
|
||||
E = A.shape[0]
|
||||
max_tokens = A.shape[1]
|
||||
K = A.shape[2]
|
||||
N = B.shape[1]
|
||||
BM = BN = BK = 64
|
||||
grid = (E, triton.cdiv(max_tokens, BM) * triton.cdiv(N, BN))
|
||||
batched_triton_kernel[grid](
|
||||
A,
|
||||
B,
|
||||
out,
|
||||
num_expert_tokens,
|
||||
tl.bfloat16,
|
||||
max_tokens,
|
||||
K,
|
||||
N,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
A.stride(0),
|
||||
A.stride(1),
|
||||
A.stride(2),
|
||||
B.stride(0),
|
||||
B.stride(2),
|
||||
B.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
out.stride(2),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
BLOCK_M=BM,
|
||||
BLOCK_N=BN,
|
||||
BLOCK_K=BK,
|
||||
USE_TD=use_td,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts,max_tokens_per_expert,K,N", _TD_SHAPES)
|
||||
def test_batched_mm_td_matches_plain(num_experts, max_tokens_per_expert, K, N):
|
||||
if not _td_supported():
|
||||
pytest.skip("TD requires XPU or CUDA sm_90+")
|
||||
set_random_seed(42)
|
||||
device = current_platform.device_type
|
||||
A = (
|
||||
torch.randn(
|
||||
num_experts, max_tokens_per_expert, K, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
/ 10
|
||||
)
|
||||
B = torch.randn(num_experts, N, K, device=device, dtype=torch.bfloat16)
|
||||
num_expert_tokens = torch.randint(
|
||||
1,
|
||||
max_tokens_per_expert + 1,
|
||||
size=(num_experts,),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
out_plain = _run_td(A, B, num_expert_tokens, False)
|
||||
out_td = _run_td(A, B, num_expert_tokens, True)
|
||||
|
||||
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
|
||||
|
||||
|
||||
def test_batched_mm_td_zero_expert_tokens():
|
||||
if not _td_supported():
|
||||
pytest.skip("TD requires XPU or CUDA sm_90+")
|
||||
set_random_seed(42)
|
||||
device = current_platform.device_type
|
||||
E, M, K, N = 8, 32, 256, 256
|
||||
A = torch.randn(E, M, K, device=device, dtype=torch.bfloat16) / 10
|
||||
B = torch.randn(E, N, K, device=device, dtype=torch.bfloat16)
|
||||
num_expert_tokens = torch.zeros(E, device=device, dtype=torch.int32)
|
||||
num_expert_tokens[::2] = M
|
||||
|
||||
out_plain = _run_td(A, B, num_expert_tokens, False)
|
||||
out_td = _run_td(A, B, num_expert_tokens, True)
|
||||
|
||||
for e in range(E):
|
||||
if num_expert_tokens[e].item() == 0:
|
||||
assert out_plain[e].abs().max().item() == 0.0
|
||||
assert out_td[e].abs().max().item() == 0.0
|
||||
|
||||
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
|
||||
|
||||
|
||||
# BatchedTritonExperts device enablement (XPU)
|
||||
def test_batched_triton_experts_supports_current_device():
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
)
|
||||
|
||||
if current_platform.is_xpu() or current_platform.is_cuda_alike():
|
||||
assert BatchedTritonExperts._supports_current_device()
|
||||
else:
|
||||
pytest.skip("No GPU device available")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
|
||||
def test_batched_experts_end_to_end(m, n, k, e, topk):
|
||||
"""End-to-end BatchedTritonExperts via the reference (no-comms)
|
||||
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
|
||||
the device-enablement path."""
|
||||
if not (current_platform.is_xpu() or current_platform.is_cuda_alike()):
|
||||
pytest.skip("No GPU device available")
|
||||
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
set_random_seed(7)
|
||||
device = current_platform.device_type
|
||||
init_workspace_manager(torch.device(f"{device}:0"))
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
|
||||
# w1 is the fused gate+up projection [E, 2N, K]; w2 is [E, K, N].
|
||||
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
topk_weight, topk_ids, _ = fused_topk(a, score, topk, False)
|
||||
|
||||
baseline_output = torch_experts(a, w1, w2, topk_weight, topk_ids)
|
||||
triton_output = batched_moe(a, w1, w2, topk_weight, topk_ids)
|
||||
|
||||
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def test_batched_triton_backend_mapping():
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
map_unquantized_backend,
|
||||
)
|
||||
|
||||
assert (
|
||||
map_unquantized_backend("batched_triton")
|
||||
== UnquantizedMoeBackend.BATCHED_TRITON
|
||||
)
|
||||
assert map_unquantized_backend("triton") == UnquantizedMoeBackend.TRITON
|
||||
|
||||
@@ -122,6 +122,7 @@ class IrOpPriorityConfig:
|
||||
MoEBackend = Literal[
|
||||
"auto",
|
||||
"triton",
|
||||
"batched_triton",
|
||||
"deep_gemm",
|
||||
"deep_gemm_mega_moe",
|
||||
"cutlass",
|
||||
@@ -192,6 +193,8 @@ class KernelConfig:
|
||||
|
||||
- "auto": Automatically select the best backend based on model and hardware
|
||||
- "triton": Use Triton-based fused MoE kernels
|
||||
- "batched_triton": Use batched Triton experts (moe_mmk) on the batched
|
||||
activation format ([E_local, max_num_tokens, K])
|
||||
- "deep_gemm": Use DeepGEMM kernels (FP8 block-quantized only)
|
||||
- "deep_gemm_mega_moe": Use DeepGEMM mega MoE kernels
|
||||
- "cutlass": Use vLLM CUTLASS kernels
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
BatchedPrepareAndFinalize,
|
||||
make_moe_prepare_and_finalize_naive_dp_ep,
|
||||
make_moe_prepare_and_finalize_no_dp_ep,
|
||||
)
|
||||
@@ -140,6 +141,16 @@ def maybe_make_prepare_finalize(
|
||||
if not allow_new_interface:
|
||||
return None
|
||||
|
||||
# Opt-in XPU batched path: reorganize tokens into E x T x K locally
|
||||
# (no all-to-all) so BatchedTritonExperts (moe_mmk TD) can run.
|
||||
if current_platform.is_xpu() and moe.moe_backend == "batched_triton":
|
||||
return BatchedPrepareAndFinalize(
|
||||
max_num_tokens=moe.max_num_tokens,
|
||||
num_local_experts=moe.num_local_experts,
|
||||
num_dispatchers=1,
|
||||
rank=moe.moe_parallel_config.ep_rank,
|
||||
)
|
||||
|
||||
# For DP/TP case, fall back to naive P/F.
|
||||
if moe.moe_parallel_config.dp_size > 1:
|
||||
logger.info_once(
|
||||
|
||||
@@ -32,7 +32,15 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.triton_utils import tl, triton, use_tensor_descriptor
|
||||
from vllm.triton_utils.allocation import set_triton_allocator
|
||||
|
||||
|
||||
def _is_capturing_or_compiling() -> bool:
|
||||
# torch.cuda.is_current_stream_capturing() is unavailable on non-CUDA (XPU) torch.
|
||||
return torch.compiler.is_compiling() or (
|
||||
current_platform.is_cuda_alike() and torch.cuda.is_current_stream_capturing()
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -71,9 +79,32 @@ def moe_mmk(
|
||||
use_w8a8: tl.constexpr,
|
||||
use_w8a16: tl.constexpr,
|
||||
per_act_token_quant: tl.constexpr,
|
||||
# TD: a_base_ptr/b_base_ptr are the expert/CTA-offset bases of A[M,K]/B[N,K].
|
||||
a_base_ptr=None,
|
||||
b_base_ptr=None,
|
||||
M=0,
|
||||
N=0,
|
||||
stride_am: tl.int64 = 0,
|
||||
stride_bn: tl.int64 = 0,
|
||||
USE_TD: tl.constexpr = False,
|
||||
):
|
||||
offs_k = tl.arange(0, BLOCK_K)
|
||||
|
||||
if USE_TD:
|
||||
# make_tensor_descriptor requires the last (K) stride to be a
|
||||
# compile-time 1; the launcher only enables USE_TD for K-contiguous A/B.
|
||||
a_desc = tl.make_tensor_descriptor(
|
||||
a_base_ptr,
|
||||
shape=[M, K],
|
||||
strides=[stride_am, 1],
|
||||
block_shape=[BLOCK_M, BLOCK_K],
|
||||
)
|
||||
b_desc = tl.make_tensor_descriptor(
|
||||
b_base_ptr,
|
||||
shape=[N, K],
|
||||
strides=[stride_bn, 1],
|
||||
block_shape=[BLOCK_N, BLOCK_K],
|
||||
)
|
||||
if use_w8a16:
|
||||
b_scale_ptrs = (
|
||||
b_scale_ptr + expert_id * stride_bse + offs_n[None, :] * stride_bsn
|
||||
@@ -110,12 +141,17 @@ def moe_mmk(
|
||||
for k in range(0, tl.cdiv(K, BLOCK_K)):
|
||||
# Load the next block of A and B, generate a mask by checking the
|
||||
# K dimension.
|
||||
a = tl.load(
|
||||
a_ptrs,
|
||||
mask=mask_m[:, None] & (offs_k[None, :] < K - k * BLOCK_K),
|
||||
other=0.0,
|
||||
)
|
||||
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_K, other=0.0)
|
||||
if USE_TD:
|
||||
# B is [N, K]; tile is [BLOCK_N, BLOCK_K], transposed for dot.
|
||||
a = a_desc.load([0, k * BLOCK_K])
|
||||
b = tl.trans(b_desc.load([0, k * BLOCK_K]))
|
||||
else:
|
||||
a = tl.load(
|
||||
a_ptrs,
|
||||
mask=mask_m[:, None] & (offs_k[None, :] < K - k * BLOCK_K),
|
||||
other=0.0,
|
||||
)
|
||||
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_K, other=0.0)
|
||||
# We accumulate along the K dimension.
|
||||
if use_w8a16:
|
||||
accumulator = tl.dot(a, b.to(compute_type), acc=accumulator)
|
||||
@@ -193,6 +229,7 @@ def expert_triton_kernel(
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
USE_TD: tl.constexpr = False,
|
||||
):
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_n = tl.arange(0, BLOCK_N) % N
|
||||
@@ -238,6 +275,13 @@ def expert_triton_kernel(
|
||||
use_fp8_w8a8,
|
||||
use_int8_w8a16,
|
||||
per_act_token_quant,
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
M,
|
||||
N,
|
||||
stride_am,
|
||||
stride_bn,
|
||||
USE_TD,
|
||||
)
|
||||
|
||||
# store in C
|
||||
@@ -292,6 +336,7 @@ def batched_triton_kernel(
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
USE_TD: tl.constexpr = False,
|
||||
):
|
||||
expert_id = tl.program_id(axis=0)
|
||||
e_num_tokens = tl.load(expert_num_tokens + expert_id)
|
||||
@@ -372,6 +417,7 @@ def batched_triton_kernel(
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
BLOCK_K,
|
||||
USE_TD,
|
||||
)
|
||||
|
||||
|
||||
@@ -444,6 +490,18 @@ def invoke_moe_batched_triton_kernel(
|
||||
stride_asm = 0
|
||||
stride_ask = 0
|
||||
|
||||
use_td = (
|
||||
use_tensor_descriptor()
|
||||
and A.stride(2) == 1
|
||||
and B.stride(2) == 1
|
||||
and (K * A.element_size()) % 16 == 0
|
||||
and (BLOCK_M & (BLOCK_M - 1)) == 0
|
||||
and (BLOCK_N & (BLOCK_N - 1)) == 0
|
||||
and (BLOCK_K & (BLOCK_K - 1)) == 0
|
||||
)
|
||||
if use_td:
|
||||
set_triton_allocator(A.device)
|
||||
|
||||
batched_triton_kernel[grid](
|
||||
A,
|
||||
B,
|
||||
@@ -485,6 +543,7 @@ def invoke_moe_batched_triton_kernel(
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_N=BLOCK_N,
|
||||
BLOCK_K=BLOCK_K,
|
||||
USE_TD=use_td,
|
||||
)
|
||||
|
||||
|
||||
@@ -616,10 +675,7 @@ class NaiveBatchedExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
for expert in range(num_local_experts):
|
||||
# Indexing expert_num_tokens doesn't work w/cudagraphs or inductor
|
||||
if (
|
||||
torch.compiler.is_compiling()
|
||||
or torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
if _is_capturing_or_compiling():
|
||||
num = hidden_states.shape[1]
|
||||
else:
|
||||
num = int(expert_num_tokens[expert].item())
|
||||
@@ -659,7 +715,7 @@ def batched_moe_kernel_quantize_input(
|
||||
per_act_token_quant: bool,
|
||||
block_shape: list[int] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing():
|
||||
if _is_capturing_or_compiling():
|
||||
# Note: this does a bunch of extra work because expert_num_tokens is
|
||||
# ignored but it does support torch.compile + cudagraphs.
|
||||
hidden_dim = A.size(-1)
|
||||
@@ -745,7 +801,7 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cuda_alike()
|
||||
return current_platform.is_cuda_alike() or current_platform.is_xpu()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
|
||||
@@ -151,6 +151,7 @@ def map_unquantized_backend(runner_backend: MoEBackend) -> UnquantizedMoeBackend
|
||||
"""Map user's MoEBackend to UnquantizedMoeBackend."""
|
||||
mapping = {
|
||||
"triton": UnquantizedMoeBackend.TRITON,
|
||||
"batched_triton": UnquantizedMoeBackend.BATCHED_TRITON,
|
||||
"flashinfer_trtllm": UnquantizedMoeBackend.FLASHINFER_TRTLLM,
|
||||
"flashinfer_cutlass": UnquantizedMoeBackend.FLASHINFER_CUTLASS,
|
||||
"aiter": UnquantizedMoeBackend.AITER,
|
||||
@@ -233,6 +234,7 @@ def select_unquantized_moe_backend(
|
||||
activation_format = (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
if moe_config.moe_parallel_config.use_batched_activation_format
|
||||
or moe_config.moe_backend == "batched_triton"
|
||||
else mk.FusedMoEActivationFormat.Standard
|
||||
)
|
||||
|
||||
|
||||
@@ -112,15 +112,28 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
|
||||
a1_scale = normalize_scales_shape(quant_config.a1_scale)
|
||||
|
||||
for expert_id in range(first_expert, last_expert):
|
||||
topks = torch.any(topk_ids == expert_id, dim=1).flatten()
|
||||
rows = torch.count_nonzero(topks.flatten())
|
||||
if rows == 0:
|
||||
continue
|
||||
idx = expert_id - first_expert
|
||||
tokens_per_expert[idx] = rows
|
||||
rhs = a1[: topks.numel()][topks]
|
||||
if quant_config.quant_dtype is not None:
|
||||
if quant_config.quant_dtype is None:
|
||||
# Vectorized dispatch: compute the [E_local, T] hit mask and
|
||||
# per-expert slot offsets in one shot (single nonzero sync instead
|
||||
# of a launch-bound per-expert Python loop).
|
||||
local_ids = torch.arange(first_expert, last_expert, device=a1.device).view(
|
||||
-1, 1, 1
|
||||
)
|
||||
hits = (topk_ids.unsqueeze(0) == local_ids).any(dim=2) # [E_local, T]
|
||||
tokens_per_expert[:num_local_experts] = hits.sum(dim=1).to(torch.int32)
|
||||
# slot of each token within its expert batch, preserving token order
|
||||
slots = hits.to(torch.int32).cumsum(dim=1) - 1 # [E_local, T]
|
||||
e_idx, t_idx = hits.nonzero(as_tuple=True)
|
||||
b_a1[e_idx, slots[e_idx, t_idx]] = a1[t_idx].to(b_type)
|
||||
else:
|
||||
for expert_id in range(first_expert, last_expert):
|
||||
topks = torch.any(topk_ids == expert_id, dim=1).flatten()
|
||||
rows = torch.count_nonzero(topks.flatten())
|
||||
if rows == 0:
|
||||
continue
|
||||
idx = expert_id - first_expert
|
||||
tokens_per_expert[idx] = rows
|
||||
rhs = a1[: topks.numel()][topks]
|
||||
if a1_scale is not None:
|
||||
if quant_config.is_per_act_token:
|
||||
rhs_a1_scale = a1_scale[: topks.numel()][topks]
|
||||
@@ -140,8 +153,6 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
b_a1_scale[idx, :rows] = b_s[:rows]
|
||||
else:
|
||||
b_a1_scale[idx, : b_s.shape[0]] = b_s
|
||||
else:
|
||||
b_a1[idx, :rows, :] = rhs
|
||||
|
||||
assert b_a1_scale is None or b_a1_scale.ndim == 3
|
||||
|
||||
|
||||
@@ -164,13 +164,20 @@ class TopKWeightAndReduceNaiveBatched(mk.TopKWeightAndReduce):
|
||||
first_expert = num_local_experts * self.rank
|
||||
last_expert = first_expert + num_local_experts
|
||||
|
||||
for expert_id in range(first_expert, last_expert):
|
||||
matching_tokens = topk_ids == expert_id
|
||||
topks = torch.any(matching_tokens, dim=1).flatten()
|
||||
rows = torch.count_nonzero(topks)
|
||||
rhs = fused_expert_output[expert_id - first_expert, :rows, :]
|
||||
if not apply_router_weight_on_input:
|
||||
rhs.mul_(topk_weights[matching_tokens].view(rhs.size(0), 1))
|
||||
output[topks] = output[topks] + rhs
|
||||
# Vectorized weighted scatter-add (single nonzero sync instead of a
|
||||
# per-expert loop; mirrors the dispatch path).
|
||||
local_ids = torch.arange(
|
||||
first_expert, last_expert, device=topk_ids.device
|
||||
).view(-1, 1, 1)
|
||||
matching = topk_ids.unsqueeze(0) == local_ids # [E_local, T, topk]
|
||||
hits = matching.any(dim=2) # [E_local, T]
|
||||
slots = hits.to(torch.int32).cumsum(dim=1) - 1 # [E_local, T]
|
||||
e_idx, t_idx = hits.nonzero(as_tuple=True)
|
||||
gathered = fused_expert_output[e_idx, slots[e_idx, t_idx], :]
|
||||
if not apply_router_weight_on_input:
|
||||
# weight of token t at expert e: the topk weight on the matching slot
|
||||
weights = (matching * topk_weights.unsqueeze(0)).sum(dim=2) # [E_local, T]
|
||||
gathered = gathered * weights[e_idx, t_idx].unsqueeze(1).to(gathered.dtype)
|
||||
output.index_add_(0, t_idx, gathered.to(output.dtype))
|
||||
|
||||
return output
|
||||
|
||||
@@ -17,7 +17,17 @@ else:
|
||||
tl = TritonLanguagePlaceholder()
|
||||
tldevice = TritonLanguagePlaceholder()
|
||||
|
||||
from vllm.triton_utils.tensor_descriptor import use_tensor_descriptor
|
||||
|
||||
LOG2E = 1.4426950408889634
|
||||
LOGE2 = 0.6931471805599453
|
||||
|
||||
__all__ = ["HAS_TRITON", "triton", "tl", "tldevice", "LOG2E", "LOGE2"]
|
||||
__all__ = [
|
||||
"HAS_TRITON",
|
||||
"triton",
|
||||
"tl",
|
||||
"tldevice",
|
||||
"LOG2E",
|
||||
"LOGE2",
|
||||
"use_tensor_descriptor",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
def use_tensor_descriptor(override: bool | None = None) -> bool:
|
||||
"""Tri-state VLLM_TRITON_USE_TD: unset=auto (on for XPU), 1/0=force on/off."""
|
||||
from vllm import envs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if override is None:
|
||||
override = envs.VLLM_TRITON_USE_TD
|
||||
if override is not None:
|
||||
return override
|
||||
return current_platform.is_xpu()
|
||||
Reference in New Issue
Block a user