[ROCm][DSV4] Enable Tilelang MHC replacing torch/triton mhc (#43679)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -54,6 +54,7 @@ from vllm.models.deepseek_v4.attention import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.import_utils import has_tilelang
|
||||
|
||||
|
||||
class DeepseekV4MLP(nn.Module):
|
||||
@@ -473,6 +474,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
self.mhc_pre = MHCPreOp()
|
||||
self.mhc_post = MHCPostOp()
|
||||
self.mhc_fused_post_pre = MHCFusedPostPreOp()
|
||||
self.has_tilelang = has_tilelang()
|
||||
|
||||
def hc_pre(
|
||||
self,
|
||||
@@ -503,7 +505,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
):
|
||||
return self.mhc_post(x, residual, post, comb)
|
||||
|
||||
def _forward_cuda(
|
||||
def _forward_fused_post_pre(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
@@ -555,7 +557,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
x = self.ffn(x, input_ids)
|
||||
return x, residual, post_mix, res_mix
|
||||
|
||||
def _forward_rocm(
|
||||
def _forward_unfused_post_pre(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
@@ -594,12 +596,13 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None
|
||||
]:
|
||||
if current_platform.is_rocm():
|
||||
return self._forward_rocm(
|
||||
if not self.has_tilelang:
|
||||
return self._forward_unfused_post_pre(
|
||||
x, positions, input_ids, post_mix, res_mix, residual
|
||||
)
|
||||
|
||||
return self._forward_cuda(x, positions, input_ids, post_mix, res_mix, residual)
|
||||
return self._forward_fused_post_pre(
|
||||
x, positions, input_ids, post_mix, res_mix, residual
|
||||
)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
@@ -682,6 +685,7 @@ class DeepseekV4Model(nn.Module):
|
||||
requires_grad=False,
|
||||
)
|
||||
self.hc_head_op = HCHeadOp()
|
||||
self.has_tilelang = has_tilelang()
|
||||
# Pre-hc_head residual stream buffer for the MTP draft. Stable
|
||||
# address (outside the cudagraph pool) so the copy_ in forward()
|
||||
# refreshes it correctly across captured shapes.
|
||||
@@ -748,7 +752,7 @@ class DeepseekV4Model(nn.Module):
|
||||
res_mix,
|
||||
residual,
|
||||
)
|
||||
if layer is not None and current_platform.is_cuda():
|
||||
if layer is not None and self.has_tilelang:
|
||||
hidden_states = layer.hc_post(hidden_states, residual, post_mix, res_mix)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weigh
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.import_utils import has_tilelang
|
||||
|
||||
from .model import DeepseekV4DecoderLayer
|
||||
|
||||
@@ -118,6 +119,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
)
|
||||
|
||||
self.hc_head_op = HCHeadOp()
|
||||
self.has_tilelang = has_tilelang()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -144,7 +146,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
hidden_states, residual, post_mix, res_mix = self.mtp_block(
|
||||
positions=positions, x=hidden_states, input_ids=None
|
||||
)
|
||||
if current_platform.is_cuda():
|
||||
if self.has_tilelang:
|
||||
hidden_states = self.mtp_block.hc_post(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user