[Feat][XPU] VLLM_BATCH_INVARIANT support for Dense/MoE models (#55881)
Signed-off-by: Tony Lin <tony.lin@intel.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -1104,7 +1104,10 @@ def enable_batch_invariant_mode():
|
||||
|
||||
_fp16_block_size_n = 128
|
||||
|
||||
# Softmax, log_softmax, mean, and bmm are already batch-invariant on XPU
|
||||
# Native mean can change reduction order with the row count on XPU too.
|
||||
_batch_invariant_LIB.impl("aten::mean.dim", mean_batch_invariant, key)
|
||||
|
||||
# Softmax, log_softmax, and bmm are already batch-invariant on XPU
|
||||
# (oneDNN backend produces bitwise-identical results regardless of batch
|
||||
# context). Only register these overrides on CUDA where they are needed.
|
||||
if not current_platform.is_xpu():
|
||||
@@ -1113,7 +1116,6 @@ def enable_batch_invariant_mode():
|
||||
)
|
||||
_batch_invariant_LIB.impl("aten::softmax", softmax_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::_softmax", softmax_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::mean.dim", mean_batch_invariant, key)
|
||||
# torch 2.12+ registers a built-in Triton bmm kernel for CUDA
|
||||
# (torch._native.ops.bmm_outer_product), so we need allow_override
|
||||
# to replace it at the dispatcher level.
|
||||
|
||||
@@ -94,7 +94,13 @@ def _get_priority_backends(moe_config: FusedMoEConfig) -> list[UnquantizedMoeBac
|
||||
_move_to_back(_AVAILABLE_BACKENDS, UnquantizedMoeBackend.FLASHINFER_CUTLASS)
|
||||
|
||||
elif current_platform.is_xpu():
|
||||
_AVAILABLE_BACKENDS = [UnquantizedMoeBackend.XPU]
|
||||
# XPUExperts does not support batch invariance; TRITON is kept as a
|
||||
# fallback so VLLM_BATCH_INVARIANT=1 still resolves to a valid
|
||||
# (batch-invariant) backend on XPU.
|
||||
_AVAILABLE_BACKENDS = [
|
||||
UnquantizedMoeBackend.XPU,
|
||||
UnquantizedMoeBackend.TRITON,
|
||||
]
|
||||
elif current_platform.is_cpu():
|
||||
_AVAILABLE_BACKENDS = [UnquantizedMoeBackend.CPU]
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
@@ -119,6 +119,14 @@ class RMSNorm(CustomOp):
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if envs.VLLM_BATCH_INVARIANT and residual is not None:
|
||||
assert self.variance_size_override is None, (
|
||||
"Batch invariance is not supported for variance_size_override"
|
||||
)
|
||||
weight = self.weight.data if self.pass_weight_add else None
|
||||
return ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
|
||||
x, residual, weight, self.variance_epsilon
|
||||
)
|
||||
return self.forward_cuda(x, residual)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
@@ -171,6 +179,15 @@ class GemmaRMSNorm(CustomOp):
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
if residual is None:
|
||||
return self.forward_cuda(x, residual)
|
||||
return ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
|
||||
x,
|
||||
residual,
|
||||
self.weight.float() + 1.0,
|
||||
self.variance_epsilon,
|
||||
)
|
||||
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_gemma_rms_norm
|
||||
|
||||
# Fall back to the native path if the fused gemma kernels are not
|
||||
|
||||
Reference in New Issue
Block a user