[Perf][DSv4.1] Exchange packed FP8 Engram rows across DP and feed wkv MXFP8 (#59923)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-10-09 15:30:18 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 82e89e9944
commit efaca9547d
6 changed files with 505 additions and 31 deletions
+21 -9
View File
@@ -100,6 +100,7 @@ from vllm.v1.worker.ubatching import dbo_current_ubatch_id
from ..common.engram import (
Engram,
EngramBatch,
EngramLayout,
NgramHashState,
can_share_engram_tables,
@@ -777,6 +778,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
),
prefix=f"{prefix}.layers",
)
self.engram_batch = EngramBatch.create(
[
layer.engram
for layer in islice(self.layers, self.start_layer, self.end_layer)
if getattr(layer, "engram", None) is not None
],
vllm_config.scheduler_config.max_num_batched_tokens,
)
if self.fuse_mhc_all_reduce:
# A MoE's top-k finalize folds into the next layer's first mHC
# boundary, which the last local layer lacks and an engram layer
@@ -971,15 +980,18 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
if engram_hashes is not None:
# Gather all Engram rows before entering the decoder layers.
# One gather feeds every layer sharing the DP-split table.
gathered_hashes = gather_engram_hashes(
engram_hashes, dp_shared_memory=self.engram_dp_shared_memory
)
for layer in islice(self.layers, self.start_layer, self.end_layer):
engram = getattr(layer, "engram", None)
if engram is not None:
engram.prepare_embeddings(
gathered_hashes[:, engram.layer_hash_index]
)
if self.engram_batch is not None:
self.engram_batch.prepare_embeddings(engram_hashes)
else:
gathered_hashes = gather_engram_hashes(
engram_hashes, dp_shared_memory=self.engram_dp_shared_memory
)
for layer in islice(self.layers, self.start_layer, self.end_layer):
engram = getattr(layer, "engram", None)
if engram is not None:
engram.prepare_embeddings(
gathered_hashes[:, engram.layer_hash_index]
)
full_num_tokens = positions.shape[0]
if self.use_sequence_parallel: