[ROCm][DSV4] Use aiter mHC pre/post as the default ROCm path (#43950)
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Signed-off-by: Fangzhou-Ai <fangzhouai@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Cursor
Claude
parent
77a9c5ae28
commit
ed41aa270a
@@ -5,6 +5,7 @@ import torch
|
||||
# this import will also register the custom ops
|
||||
# import vllm.model_executor.kernels.mhc # noqa: F401
|
||||
import vllm.model_executor.kernels.mhc as mhc_kernels
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_tilelang
|
||||
@@ -25,6 +26,7 @@ def _has_tilelang_mhc() -> bool:
|
||||
|
||||
|
||||
HAS_TILELANG_MHC = _has_tilelang_mhc()
|
||||
HAS_AITER_MHC = is_aiter_found_and_supported()
|
||||
|
||||
|
||||
# --8<-- [start:mhc_pre]
|
||||
@@ -87,25 +89,20 @@ class MHCPreOp(CustomOp):
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# TODO: Reenable aiter after we are at the aiter
|
||||
# version that has this bugfix
|
||||
# https://github.com/ROCm/aiter/commit/b639cb63bcac4672dce33a731fad042a65cb3649
|
||||
# It has accuracy problem at large number of tokens.
|
||||
# hidden_size = residual.shape[-1]
|
||||
# if hidden_size % 256 == 0:
|
||||
# return torch.ops.vllm.mhc_pre_aiter(
|
||||
# residual,
|
||||
# fn,
|
||||
# hc_scale,
|
||||
# hc_base,
|
||||
# rms_eps,
|
||||
# hc_pre_eps,
|
||||
# hc_sinkhorn_eps,
|
||||
# hc_post_mult_value,
|
||||
# sinkhorn_repeat,
|
||||
# )
|
||||
# else:
|
||||
if HAS_TILELANG_MHC:
|
||||
hidden_size = residual.shape[-1]
|
||||
if HAS_AITER_MHC and hidden_size % 256 == 0:
|
||||
return torch.ops.vllm.mhc_pre_aiter(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
elif HAS_TILELANG_MHC:
|
||||
return torch.ops.vllm.mhc_pre_tilelang(
|
||||
residual,
|
||||
fn,
|
||||
@@ -227,19 +224,14 @@ class MHCPostOp(CustomOp):
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# TODO: Reenable aiter after we are at the aiter
|
||||
# version that has this bugfix
|
||||
# https://github.com/ROCm/aiter/commit/b639cb63bcac4672dce33a731fad042a65cb3649
|
||||
# It has accuracy problem at large number of tokens.
|
||||
# hidden_size = residual.shape[-1]
|
||||
# if hidden_size % 256 == 0:
|
||||
# return torch.ops.vllm.mhc_post_aiter(
|
||||
# x,
|
||||
# residual,
|
||||
# post_layer_mix,
|
||||
# comb_res_mix,
|
||||
# )
|
||||
# else:
|
||||
hidden_size = residual.shape[-1]
|
||||
if HAS_AITER_MHC and hidden_size % 256 == 0:
|
||||
return torch.ops.vllm.mhc_post_aiter(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
)
|
||||
if HAS_TILELANG_MHC:
|
||||
return torch.ops.vllm.mhc_post_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import (
|
||||
HAS_AITER_MHC,
|
||||
HAS_TILELANG_MHC,
|
||||
HCHeadOp,
|
||||
MHCFusedPostPreOp,
|
||||
@@ -303,7 +304,9 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
self.mhc_pre = MHCPreOp()
|
||||
self.mhc_post = MHCPostOp()
|
||||
self.mhc_fused_post_pre = MHCFusedPostPreOp()
|
||||
self.has_tilelang = HAS_TILELANG_MHC
|
||||
self.use_fused_mhc = HAS_TILELANG_MHC and not (
|
||||
HAS_AITER_MHC and self.hidden_size % 256 == 0
|
||||
)
|
||||
|
||||
def hc_pre(
|
||||
self,
|
||||
@@ -425,7 +428,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
|
||||
]:
|
||||
if not self.has_tilelang:
|
||||
if not self.use_fused_mhc:
|
||||
return self._forward_unfused_post_pre(
|
||||
x, positions, input_ids, post_mix, res_mix, residual
|
||||
)
|
||||
@@ -513,7 +516,6 @@ class DeepseekV4Model(nn.Module):
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_op = HCHeadOp()
|
||||
self.has_tilelang = HAS_TILELANG_MHC
|
||||
# Pre-hc_head residual stream buffer for the MTP draft. Stable
|
||||
# address (outside the cudagraph pool) so the copy_ in forward()
|
||||
# refreshes it correctly across captured shapes.
|
||||
@@ -580,7 +582,7 @@ class DeepseekV4Model(nn.Module):
|
||||
res_mix,
|
||||
residual,
|
||||
)
|
||||
if layer is not None and self.has_tilelang:
|
||||
if layer is not None and layer.use_fused_mhc:
|
||||
hidden_states = layer.hc_post(hidden_states, residual, post_mix, res_mix)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
|
||||
@@ -28,7 +28,7 @@ from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_ma
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import HAS_TILELANG_MHC, HCHeadOp
|
||||
from vllm.model_executor.layers.mhc import HCHeadOp
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
@@ -123,7 +123,6 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.hc_head_op = HCHeadOp()
|
||||
self.has_tilelang = HAS_TILELANG_MHC
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -156,7 +155,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
hidden_states, residual, post_mix, res_mix = self.mtp_block(
|
||||
positions=positions, x=hidden_states, input_ids=None
|
||||
)
|
||||
if self.has_tilelang:
|
||||
if self.mtp_block.use_fused_mhc:
|
||||
hidden_states = self.mtp_block.hc_post(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user