[Feat][XPU] VLLM_BATCH_INVARIANT support for Dense/MoE models (#55881)
Signed-off-by: Tony Lin <tony.lin@intel.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -94,7 +94,13 @@ def _get_priority_backends(moe_config: FusedMoEConfig) -> list[UnquantizedMoeBac
|
||||
_move_to_back(_AVAILABLE_BACKENDS, UnquantizedMoeBackend.FLASHINFER_CUTLASS)
|
||||
|
||||
elif current_platform.is_xpu():
|
||||
_AVAILABLE_BACKENDS = [UnquantizedMoeBackend.XPU]
|
||||
# XPUExperts does not support batch invariance; TRITON is kept as a
|
||||
# fallback so VLLM_BATCH_INVARIANT=1 still resolves to a valid
|
||||
# (batch-invariant) backend on XPU.
|
||||
_AVAILABLE_BACKENDS = [
|
||||
UnquantizedMoeBackend.XPU,
|
||||
UnquantizedMoeBackend.TRITON,
|
||||
]
|
||||
elif current_platform.is_cpu():
|
||||
_AVAILABLE_BACKENDS = [UnquantizedMoeBackend.CPU]
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
@@ -119,6 +119,14 @@ class RMSNorm(CustomOp):
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if envs.VLLM_BATCH_INVARIANT and residual is not None:
|
||||
assert self.variance_size_override is None, (
|
||||
"Batch invariance is not supported for variance_size_override"
|
||||
)
|
||||
weight = self.weight.data if self.pass_weight_add else None
|
||||
return ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
|
||||
x, residual, weight, self.variance_epsilon
|
||||
)
|
||||
return self.forward_cuda(x, residual)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
@@ -171,6 +179,15 @@ class GemmaRMSNorm(CustomOp):
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
if residual is None:
|
||||
return self.forward_cuda(x, residual)
|
||||
return ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
|
||||
x,
|
||||
residual,
|
||||
self.weight.float() + 1.0,
|
||||
self.variance_epsilon,
|
||||
)
|
||||
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_gemma_rms_norm
|
||||
|
||||
# Fall back to the native path if the fused gemma kernels are not
|
||||
|
||||
Reference in New Issue
Block a user