[Feat][XPU] VLLM_BATCH_INVARIANT support for Dense/MoE models (#55881)

Signed-off-by: Tony Lin <tony.lin@intel.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Tony Lin
2026-09-22 11:06:54 -04:00
committed by GitHub
co-authored by Flora Feng
parent 1ea7c63f4a
commit e4340e41c9
10 changed files with 619 additions and 8 deletions
+17
View File
@@ -119,6 +119,14 @@ class RMSNorm(CustomOp):
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if envs.VLLM_BATCH_INVARIANT and residual is not None:
assert self.variance_size_override is None, (
"Batch invariance is not supported for variance_size_override"
)
weight = self.weight.data if self.pass_weight_add else None
return ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
x, residual, weight, self.variance_epsilon
)
return self.forward_cuda(x, residual)
def extra_repr(self) -> str:
@@ -171,6 +179,15 @@ class GemmaRMSNorm(CustomOp):
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if envs.VLLM_BATCH_INVARIANT:
if residual is None:
return self.forward_cuda(x, residual)
return ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
x,
residual,
self.weight.float() + 1.0,
self.variance_epsilon,
)
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_gemma_rms_norm
# Fall back to the native path if the fused gemma kernels are not