[Feat][XPU] VLLM_BATCH_INVARIANT support for Dense/MoE models (#55881)

Signed-off-by: Tony Lin <tony.lin@intel.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Tony Lin
2026-09-22 11:06:54 -04:00
committed by GitHub
co-authored by Flora Feng
parent 1ea7c63f4a
commit e4340e41c9
10 changed files with 619 additions and 8 deletions
@@ -6,6 +6,7 @@ import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup
import vllm.envs as envs
from vllm.logger import init_logger
from .base_device_communicator import DeviceCommunicatorBase
@@ -45,7 +46,25 @@ class XpuCommunicator(DeviceCommunicatorBase):
self.all2all_manager = AgRsAll2AllManager(self.cpu_group)
logger.info("Using AgRs manager on XPU device.")
def _fixed_rank_sum(self, input_: torch.Tensor) -> torch.Tensor:
flat_input = input_.reshape(-1)
gathered = torch.empty(
(self.world_size, flat_input.numel()),
dtype=input_.dtype,
device=input_.device,
)
dist.all_gather_single(gathered.view(-1), flat_input, group=self.device_group)
output = gathered[0].clone()
for rank in range(1, self.world_size):
output.add_(gathered[rank])
return output.view(input_.shape)
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
if envs.VLLM_BATCH_INVARIANT:
# XCCL changes its reduction order with message size. Gather first
# and accumulate in rank order, independent of the token batch.
return self._fixed_rank_sum(input_)
output = input_.clone()
dist.all_reduce(output, group=self.device_group)
return output
@@ -69,7 +88,12 @@ class XpuCommunicator(DeviceCommunicatorBase):
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
)
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
if envs.VLLM_BATCH_INVARIANT:
reduced = self._fixed_rank_sum(input_tensor)
start = self.rank_in_group * chunk_size
output.copy_(reduced.narrow(0, start, chunk_size))
else:
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
# Reshape before returning
return output.movedim(0, dim).contiguous()
@@ -99,7 +123,15 @@ class XpuCommunicator(DeviceCommunicatorBase):
output = torch.empty(
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
)
if sizes is None or sizes.count(sizes[0]) == len(sizes):
if envs.VLLM_BATCH_INVARIANT:
reduced = self._fixed_rank_sum(input_tensor)
start = (
self.rank_in_group * chunk_size
if sizes is None
else sum(sizes[: self.rank_in_group])
)
output.copy_(reduced.narrow(0, start, chunk_size))
elif sizes is None or sizes.count(sizes[0]) == len(sizes):
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
elif torch.xpu.device_count() < self.world_size:
# uneven reduce_scatter is not supported when the
@@ -191,7 +223,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
(self.world_size,) + input_size, dtype=input_.dtype, device=input_.device
)
# All-gather.
dist.all_gather_into_tensor(output_tensor, input_, group=self.device_group)
dist.all_gather_single(output_tensor, input_, group=self.device_group)
if self.rank_in_group == dst:
# Reshape
output_tensor = output_tensor.movedim(0, dim)