[Perf][DSV4.1] Fuse TP all-reduce with mHC input preparation (#57643)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -17,6 +17,7 @@ from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.forward_context import (
|
||||
get_forward_context,
|
||||
@@ -88,10 +89,11 @@ from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
from ..common.engram import EngramLayout, NgramHashState
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
|
||||
from .engram import Engram, gather_engram_hashes
|
||||
from .ops.mega_mhc import mhc_shifted_post_pre
|
||||
from .ops.mhc import (
|
||||
MHC_OVERLAP_MAX_TOKENS,
|
||||
mhc_pre_delayed_overlap,
|
||||
mhc_shifted_post_pre,
|
||||
supports_mhc_all_reduce,
|
||||
supports_mhc_overlap,
|
||||
)
|
||||
|
||||
@@ -107,6 +109,7 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
use_sequence_parallel: bool = False,
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
config = vllm_config.model_config.hf_config
|
||||
n_routed_experts = config.n_routed_experts
|
||||
@@ -125,6 +128,7 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
|
||||
n_routed_experts=n_routed_experts,
|
||||
n_activated_experts=n_activated_experts,
|
||||
num_hash_layers=0,
|
||||
reduce_results=reduce_results,
|
||||
image_sentinel_lo=IMAGE_SENTINEL_BASE_ID,
|
||||
)
|
||||
|
||||
@@ -199,12 +203,14 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
candidate_block_buffer: torch.Tensor | None = None,
|
||||
engram_layout: EngramLayout | None = None,
|
||||
mhc_stream: torch.cuda.Stream | None = None,
|
||||
fuse_mhc_all_reduce: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.mhc_stream = mhc_stream
|
||||
self.fuse_mhc_all_reduce = fuse_mhc_all_reduce
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
|
||||
self.engram: Engram | None = None
|
||||
@@ -228,12 +234,13 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
aux_stream_list=aux_stream_list,
|
||||
candidate_block_buffer=candidate_block_buffer,
|
||||
)
|
||||
if self.use_sequence_parallel:
|
||||
if self.use_sequence_parallel or fuse_mhc_all_reduce:
|
||||
self.attn.wo_b.reduce_results = False
|
||||
self.ffn = DeepseekV4MoE(
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.ffn",
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
reduce_results=not fuse_mhc_all_reduce,
|
||||
)
|
||||
|
||||
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
@@ -430,6 +437,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
# and this block's pre, on the full hc stream, so the mix
|
||||
# coefficients see the injected stream. The injection also keeps
|
||||
# the post out of the pre-norm GEMM's fused prologue.
|
||||
if self.fuse_mhc_all_reduce:
|
||||
x = tensor_model_parallel_all_reduce(x)
|
||||
previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix)
|
||||
if capture_previous_aux:
|
||||
previous_aux = previous_post.mean(dim=1)
|
||||
@@ -473,6 +482,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
capture_aux=capture_previous_aux,
|
||||
stream=mhc_stream,
|
||||
reduce_results=self.fuse_mhc_all_reduce,
|
||||
)
|
||||
if capture_previous_aux:
|
||||
previous_aux = aux
|
||||
@@ -503,6 +513,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
norm_weight=self.ffn_norm.weight,
|
||||
norm_eps=self.ffn_norm.variance_epsilon,
|
||||
stream=mhc_stream,
|
||||
reduce_results=self.fuse_mhc_all_reduce,
|
||||
)
|
||||
x = self.ffn(x, input_ids, mega_gate_metadata)
|
||||
if mhc_stream is not None:
|
||||
@@ -540,6 +551,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
|
||||
# Keep mHC independent of the streams used inside attention.
|
||||
mhc_stream = torch.cuda.Stream() if supports_mhc_overlap(vllm_config) else None
|
||||
self.fuse_mhc_all_reduce = mhc_stream is not None and supports_mhc_all_reduce(
|
||||
vllm_config
|
||||
)
|
||||
|
||||
# Reserved topk indices buffer for all Indexer layers to reuse.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
@@ -585,6 +599,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
candidate_block_buffer=self.candidate_block_buffer,
|
||||
engram_layout=self.engram_layout,
|
||||
mhc_stream=mhc_stream,
|
||||
fuse_mhc_all_reduce=self.fuse_mhc_all_reduce,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
@@ -794,6 +809,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
aux_hidden_by_layer[idx] = previous_aux
|
||||
if layer is not None:
|
||||
# The last layer has no successor to fold its post into.
|
||||
if self.fuse_mhc_all_reduce:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states = mhc_post_tilelang(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user