[ROCm][DSV4] Enable Tilelang MHC replacing torch/triton mhc (#43679)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
TJian
2026-05-28 07:05:28 +00:00
committed by GitHub
parent e1814f822d
commit 0ba46d4b11
13 changed files with 715 additions and 98 deletions
+11 -7
View File
@@ -54,6 +54,7 @@ from vllm.models.deepseek_v4.attention import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.import_utils import has_tilelang
class DeepseekV4MLP(nn.Module):
@@ -473,6 +474,7 @@ class DeepseekV4DecoderLayer(nn.Module):
self.mhc_pre = MHCPreOp()
self.mhc_post = MHCPostOp()
self.mhc_fused_post_pre = MHCFusedPostPreOp()
self.has_tilelang = has_tilelang()
def hc_pre(
self,
@@ -503,7 +505,7 @@ class DeepseekV4DecoderLayer(nn.Module):
):
return self.mhc_post(x, residual, post, comb)
def _forward_cuda(
def _forward_fused_post_pre(
self,
x: torch.Tensor,
positions: torch.Tensor,
@@ -555,7 +557,7 @@ class DeepseekV4DecoderLayer(nn.Module):
x = self.ffn(x, input_ids)
return x, residual, post_mix, res_mix
def _forward_rocm(
def _forward_unfused_post_pre(
self,
x: torch.Tensor,
positions: torch.Tensor,
@@ -594,12 +596,13 @@ class DeepseekV4DecoderLayer(nn.Module):
) -> tuple[
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
]:
if current_platform.is_rocm():
return self._forward_rocm(
if not self.has_tilelang:
return self._forward_unfused_post_pre(
x, positions, input_ids, post_mix, res_mix, residual
)
return self._forward_cuda(x, positions, input_ids, post_mix, res_mix, residual)
return self._forward_fused_post_pre(
x, positions, input_ids, post_mix, res_mix, residual
)
@support_torch_compile
@@ -682,6 +685,7 @@ class DeepseekV4Model(nn.Module):
requires_grad=False,
)
self.hc_head_op = HCHeadOp()
self.has_tilelang = has_tilelang()
# Pre-hc_head residual stream buffer for the MTP draft. Stable
# address (outside the cudagraph pool) so the copy_ in forward()
# refreshes it correctly across captured shapes.
@@ -748,7 +752,7 @@ class DeepseekV4Model(nn.Module):
res_mix,
residual,
)
if layer is not None and current_platform.is_cuda():
if layer is not None and self.has_tilelang:
hidden_states = layer.hc_post(hidden_states, residual, post_mix, res_mix)
if not get_pp_group().is_last_rank:
+3 -1
View File
@@ -39,6 +39,7 @@ from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weigh
from vllm.model_executor.models.utils import maybe_prefix
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.import_utils import has_tilelang
from .model import DeepseekV4DecoderLayer
@@ -118,6 +119,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
)
self.hc_head_op = HCHeadOp()
self.has_tilelang = has_tilelang()
def forward(
self,
@@ -144,7 +146,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
hidden_states, residual, post_mix, res_mix = self.mtp_block(
positions=positions, x=hidden_states, input_ids=None
)
if current_platform.is_cuda():
if self.has_tilelang:
hidden_states = self.mtp_block.hc_post(
hidden_states, residual, post_mix, res_mix
)