[Feat][XPU] VLLM_BATCH_INVARIANT support for Dense/MoE models (#55881)
Signed-off-by: Tony Lin <tony.lin@intel.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -119,6 +119,14 @@ class RMSNorm(CustomOp):
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if envs.VLLM_BATCH_INVARIANT and residual is not None:
|
||||
assert self.variance_size_override is None, (
|
||||
"Batch invariance is not supported for variance_size_override"
|
||||
)
|
||||
weight = self.weight.data if self.pass_weight_add else None
|
||||
return ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
|
||||
x, residual, weight, self.variance_epsilon
|
||||
)
|
||||
return self.forward_cuda(x, residual)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
@@ -171,6 +179,15 @@ class GemmaRMSNorm(CustomOp):
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
if residual is None:
|
||||
return self.forward_cuda(x, residual)
|
||||
return ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
|
||||
x,
|
||||
residual,
|
||||
self.weight.float() + 1.0,
|
||||
self.variance_epsilon,
|
||||
)
|
||||
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_gemma_rms_norm
|
||||
|
||||
# Fall back to the native path if the fused gemma kernels are not
|
||||
|
||||
Reference in New Issue
Block a user