[ROCm][DSV4] Enable Tilelang MHC replacing torch/triton mhc (#43679)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -3,8 +3,12 @@
|
||||
import torch
|
||||
|
||||
# this import will also register the custom ops
|
||||
# import vllm.model_executor.kernels.mhc # noqa: F401
|
||||
import vllm.model_executor.kernels.mhc as mhc_kernels
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.utils.import_utils import has_tilelang
|
||||
|
||||
HAS_TILELANG = has_tilelang()
|
||||
|
||||
|
||||
# --8<-- [start:mhc_pre]
|
||||
@@ -85,6 +89,52 @@ class MHCPreOp(CustomOp):
|
||||
# sinkhorn_repeat,
|
||||
# )
|
||||
# else:
|
||||
if HAS_TILELANG:
|
||||
return torch.ops.vllm.mhc_pre_tilelang(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
else:
|
||||
return self.forward_native(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return mhc_kernels.mhc_pre_torch(
|
||||
residual,
|
||||
fn,
|
||||
@@ -97,9 +147,6 @@ class MHCPreOp(CustomOp):
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
raise NotImplementedError("Native implementation of mhc_pre is not available")
|
||||
|
||||
|
||||
# --8<-- [start:mhc_post]
|
||||
@CustomOp.register("mhc_post")
|
||||
@@ -147,6 +194,20 @@ class MHCPostOp(CustomOp):
|
||||
# comb_res_mix,
|
||||
# )
|
||||
# else:
|
||||
if HAS_TILELANG:
|
||||
return torch.ops.vllm.mhc_post_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix
|
||||
)
|
||||
else:
|
||||
return self.forward_native(x, residual, post_layer_mix, comb_res_mix)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return mhc_kernels.mhc_post_torch(
|
||||
x,
|
||||
residual,
|
||||
@@ -154,9 +215,6 @@ class MHCPostOp(CustomOp):
|
||||
comb_res_mix,
|
||||
)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
raise NotImplementedError("Native implementation of mhc_post is not available")
|
||||
|
||||
|
||||
# --8<-- [start:hc_head]
|
||||
@CustomOp.register("hc_head")
|
||||
@@ -220,17 +278,32 @@ class HCHeadOp(CustomOp):
|
||||
out = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=hidden_states.device
|
||||
)
|
||||
torch.ops.vllm.hc_head_triton(
|
||||
hs_flat,
|
||||
hc_fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_norm_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
if HAS_TILELANG:
|
||||
torch.ops.vllm.hc_head_fused_kernel_tilelang(
|
||||
hs_flat,
|
||||
hc_fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_norm_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
else:
|
||||
torch.ops.vllm.hc_head_triton(
|
||||
hs_flat,
|
||||
hc_fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_norm_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
return out.view(*outer_shape, hidden_size)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
@@ -290,9 +363,42 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
def forward_hip(self, *args, **kwargs):
|
||||
raise NotImplementedError(
|
||||
"Hip implementation of mhc_fused_post_pre is not available"
|
||||
def forward_hip(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.mhc_fused_post_pre_tilelang(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
tile_n,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
|
||||
Reference in New Issue
Block a user