[Feature] support sequence parallelism using compilation pass (#16155)
Signed-off-by: cascade812 <cascade812@outlook.com> Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
ed7a29d9f8
commit
690fe019f0
@@ -113,6 +113,38 @@ def all_reduce_fake(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
|
||||
return torch.empty_like(tensor)
|
||||
|
||||
|
||||
def reduce_scatter(tensor: torch.Tensor, dim: int, world_size: int,
|
||||
group_name: str) -> torch.Tensor:
|
||||
assert group_name in _groups, f"Group {group_name} is not found."
|
||||
group = _groups[group_name]()
|
||||
if group is None:
|
||||
raise ValueError(f"Group {group_name} is destroyed.")
|
||||
return group.reduce_scatter(tensor, dim)
|
||||
|
||||
|
||||
def reduce_scatter_fake(tensor: torch.Tensor, dim: int, world_size: int,
|
||||
group_name: str) -> torch.Tensor:
|
||||
new_shape = list(tensor.shape)
|
||||
new_shape[dim] = tensor.shape[dim] // world_size
|
||||
return torch.empty(new_shape, dtype=tensor.dtype, device=tensor.device)
|
||||
|
||||
|
||||
def all_gather(tensor: torch.Tensor, dim: int, world_size: int,
|
||||
group_name: str) -> torch.Tensor:
|
||||
assert group_name in _groups, f"Group {group_name} is not found."
|
||||
group = _groups[group_name]()
|
||||
if group is None:
|
||||
raise ValueError(f"Group {group_name} is destroyed.")
|
||||
return group.all_gather(tensor, dim)
|
||||
|
||||
|
||||
def all_gather_fake(tensor: torch.Tensor, dim: int, world_size: int,
|
||||
group_name: str) -> torch.Tensor:
|
||||
new_shape = list(tensor.shape)
|
||||
new_shape[dim] = tensor.shape[dim] * world_size
|
||||
return torch.empty(new_shape, dtype=tensor.dtype, device=tensor.device)
|
||||
|
||||
|
||||
if supports_custom_op():
|
||||
from vllm.platforms import current_platform
|
||||
direct_register_custom_op(
|
||||
@@ -123,6 +155,20 @@ if supports_custom_op():
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="reduce_scatter",
|
||||
op_func=reduce_scatter,
|
||||
mutates_args=[],
|
||||
fake_impl=reduce_scatter_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="all_gather",
|
||||
op_func=all_gather,
|
||||
mutates_args=[],
|
||||
fake_impl=all_gather_fake,
|
||||
)
|
||||
|
||||
|
||||
class GroupCoordinator:
|
||||
"""
|
||||
@@ -322,6 +368,18 @@ class GroupCoordinator:
|
||||
|
||||
return self.device_communicator.all_gather(input_, dim)
|
||||
|
||||
def reduce_scatter(self,
|
||||
input_: torch.Tensor,
|
||||
dim: int = -1) -> torch.Tensor:
|
||||
world_size = self.world_size
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if world_size == 1:
|
||||
return input_
|
||||
assert -input_.dim() <= dim < input_.dim(), (
|
||||
f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
|
||||
|
||||
return self.device_communicator.reduce_scatter(input_, dim)
|
||||
|
||||
def gather(self,
|
||||
input_: torch.Tensor,
|
||||
dst: int = 0,
|
||||
|
||||
Reference in New Issue
Block a user