[compile] Add patched_fused_scaled_matmul_reduce_scatter (#26604)
Signed-off-by: angelayi <yiangela7@gmail.com>
This commit is contained in:
@@ -37,6 +37,8 @@ from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
import torch.distributed
|
||||
import torch.distributed._functional_collectives as funcol
|
||||
import torch.distributed._symmetric_memory
|
||||
from torch.distributed import Backend, ProcessGroup
|
||||
from typing_extensions import deprecated
|
||||
|
||||
@@ -159,6 +161,90 @@ def all_gather_fake(
|
||||
return torch.empty(new_shape, dtype=tensor.dtype, device=tensor.device)
|
||||
|
||||
|
||||
def patched_fused_scaled_matmul_reduce_scatter_fake(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
A_scale: torch.Tensor,
|
||||
B_scale: torch.Tensor,
|
||||
reduce_op: str,
|
||||
orig_scatter_dim: int,
|
||||
scatter_dim_after_maybe_reshape: int,
|
||||
group_name: str,
|
||||
output_shape: list[int],
|
||||
bias: torch.Tensor | None = None,
|
||||
result_scale: torch.Tensor | None = None,
|
||||
out_dtype: torch.dtype | None = None,
|
||||
use_fast_accum: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# Copied from
|
||||
# https://github.com/pytorch/pytorch/blob/50c338c2da905062449e4d9ac807832d1b5cd90e/torch/distributed/_symmetric_memory/__init__.py#L1189
|
||||
if A_scale.numel() > 1:
|
||||
if A_scale.shape[:-1] != A.shape[:-1]:
|
||||
raise ValueError(
|
||||
"For row-wise scaling, the leading dims of A_scale "
|
||||
"must match the leading dims of A "
|
||||
f"(A shape: {A.shape}, A_scale shape: {A_scale.shape})"
|
||||
)
|
||||
A_scale = A_scale.flatten(0, -2).contiguous()
|
||||
elif A_scale.numel() != 1:
|
||||
raise ValueError(
|
||||
"Invalid A_scale shape "
|
||||
f"(A shape: {A.shape}, A_scale shape: {A_scale.shape})"
|
||||
)
|
||||
|
||||
C = torch._scaled_mm(
|
||||
A.flatten(0, -2).contiguous(),
|
||||
B,
|
||||
A_scale,
|
||||
B_scale,
|
||||
bias,
|
||||
result_scale,
|
||||
out_dtype,
|
||||
use_fast_accum,
|
||||
)
|
||||
C = C.view(*output_shape[:-1], B.shape[1])
|
||||
res = funcol.reduce_scatter_tensor(
|
||||
C,
|
||||
reduce_op,
|
||||
orig_scatter_dim, # need original scatter dim for 3D+ output tensor here
|
||||
group_name,
|
||||
)
|
||||
res = funcol.wait_tensor(res)
|
||||
return res
|
||||
|
||||
|
||||
def patched_fused_scaled_matmul_reduce_scatter(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
A_scale: torch.Tensor,
|
||||
B_scale: torch.Tensor,
|
||||
reduce_op: str,
|
||||
orig_scatter_dim: int,
|
||||
scatter_dim_after_maybe_reshape: int,
|
||||
group_name: str,
|
||||
output_shape: list[int],
|
||||
bias: torch.Tensor | None = None,
|
||||
result_scale: torch.Tensor | None = None,
|
||||
out_dtype: torch.dtype | None = None,
|
||||
use_fast_accum: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.symm_mem.fused_scaled_matmul_reduce_scatter(
|
||||
A,
|
||||
B,
|
||||
A_scale,
|
||||
B_scale,
|
||||
reduce_op,
|
||||
orig_scatter_dim,
|
||||
scatter_dim_after_maybe_reshape,
|
||||
group_name,
|
||||
output_shape,
|
||||
bias,
|
||||
result_scale,
|
||||
out_dtype,
|
||||
use_fast_accum,
|
||||
)
|
||||
|
||||
|
||||
if supports_custom_op():
|
||||
direct_register_custom_op(
|
||||
op_name="all_reduce",
|
||||
@@ -178,6 +264,15 @@ if supports_custom_op():
|
||||
fake_impl=all_gather_fake,
|
||||
)
|
||||
|
||||
# TODO: Remove this once the pytorch fix
|
||||
# (https://github.com/pytorch/pytorch/pull/165086) gets released,
|
||||
# in either 2.9.1 or 2.10
|
||||
direct_register_custom_op(
|
||||
op_name="patched_fused_scaled_matmul_reduce_scatter",
|
||||
op_func=patched_fused_scaled_matmul_reduce_scatter,
|
||||
fake_impl=patched_fused_scaled_matmul_reduce_scatter_fake,
|
||||
)
|
||||
|
||||
|
||||
class GroupCoordinator:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user