[Perf][DSv4.1] Exchange packed FP8 Engram rows across DP and feed wkv MXFP8 (#59923)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
82e89e9944
commit
efaca9547d
@@ -39,9 +39,16 @@ from vllm.distributed.parallel_state import (
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
convert_swizzled_to_linear,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram,
|
||||
EngramBatch,
|
||||
EngramLayout,
|
||||
ParallelEngramEmbedding,
|
||||
engram_head_shard_rank,
|
||||
@@ -207,10 +214,75 @@ def _worker(rank: int, tp_size: int, port: int) -> None:
|
||||
n_heads,
|
||||
sequence_parallel,
|
||||
)
|
||||
if tp_size == 1:
|
||||
_check_fp8_batch(vllm_config)
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def _check_fp8_batch(vllm_config):
|
||||
"""MXFP8 staging matches the tables across peer/all-to-all slots, idle
|
||||
replicas and graph replays."""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
layers = []
|
||||
for index in range(2):
|
||||
weight, scale = _full_table(sum(HEAD_SIZES))
|
||||
weight = (weight.float() * (1 - 2 * index)).to(torch.float8_e4m3fn)
|
||||
table = ParallelEngramEmbedding(
|
||||
sum(HEAD_SIZES), DIM, HEAD_SIZES, cpu_offload=True
|
||||
)
|
||||
table.weight.weight_loader(table.weight, weight)
|
||||
table.weight_scale_inv.weight_loader(table.weight_scale_inv, scale)
|
||||
staged = torch.empty(0, device="cuda")
|
||||
reference = (weight.cuda(), scale.cuda())
|
||||
layers.append(SimpleNamespace(embed_tokens=table, staged_rows=staged))
|
||||
layers[-1].layer_hash_index, layers[-1].reference = index, reference
|
||||
batch = EngramBatch(layers, 257)
|
||||
|
||||
def check(ids):
|
||||
tokens = ids.shape[0]
|
||||
for layer in layers:
|
||||
x = batch.wkv_input(layer, tokens)
|
||||
sf = convert_swizzled_to_linear(x.scale, tokens, x.data.shape[1], BLOCK)
|
||||
actual = dequant_mxfp8_to_bf16(x.data, sf).unflatten(-1, (-1, DIM))
|
||||
weight, scale = layer.reference
|
||||
expected = _reference(weight, scale, ids[:, layer.layer_hash_index])
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
for counts in ((7, 0, 5, 1), (127,) * 4, (128,) * 4, (257, 0, 129, 1)):
|
||||
tokens = counts[dp_rank]
|
||||
seed = 700 + 10 * dp_rank
|
||||
ids = torch.stack([_make_ids(HEAD_SIZES, tokens, seed + i) for i in (0, 1)], 1)
|
||||
with set_forward_context(
|
||||
None,
|
||||
vllm_config,
|
||||
num_tokens=tokens,
|
||||
num_tokens_across_dp=torch.tensor(counts, dtype=torch.int32),
|
||||
):
|
||||
batch.prepare_embeddings(ids)
|
||||
check(ids)
|
||||
for capture in ("full", "breakable") if len(set(counts)) == 1 else ():
|
||||
warmup = torch.cuda.Stream()
|
||||
warmup.wait_stream(torch.cuda.current_stream())
|
||||
if capture == "full":
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=warmup):
|
||||
batch.prepare_embeddings(ids)
|
||||
else:
|
||||
graph = BreakableCUDAGraphCapture()
|
||||
with torch.cuda.stream(warmup), graph:
|
||||
batch.prepare_embeddings(ids)
|
||||
torch.cuda.current_stream().wait_stream(warmup)
|
||||
for step in (2, 3):
|
||||
replay_ids = _make_ids(HEAD_SIZES, tokens, seed + step)
|
||||
ids.copy_(torch.stack([replay_ids] * 2, 1))
|
||||
graph.replay()
|
||||
check(ids)
|
||||
del graph
|
||||
|
||||
|
||||
def _check_shared_storage_lifetime_and_failures(failure):
|
||||
"""One peer's failure must roll back every mapping/registration, without hangs."""
|
||||
import gc
|
||||
@@ -516,6 +588,7 @@ def _check_dummy_hash_model_forward(
|
||||
model.use_native_mega_moe = False
|
||||
model.use_sequence_parallel = False
|
||||
model.fuse_mhc_all_reduce = False
|
||||
model.engram_batch = None
|
||||
model.engram_hash = state
|
||||
model.engram_swa_prefix = "swa"
|
||||
model.engram_dp_shared_memory = dp_shared_memory
|
||||
|
||||
@@ -7,6 +7,9 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram,
|
||||
@@ -714,8 +717,9 @@ def test_engram_lookup_reuses_jit_across_token_shapes():
|
||||
)
|
||||
@pytest.mark.parametrize("cpu_offload", [False, True])
|
||||
@pytest.mark.parametrize("background", [False, True])
|
||||
@pytest.mark.parametrize("packed", [False, True])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 256])
|
||||
def test_engram_lookup_matches_torch(cpu_offload, background, num_tokens):
|
||||
def test_engram_lookup_matches_torch(cpu_offload, background, packed, num_tokens):
|
||||
"""The fused gather must be bit-exact with the torch dequant path it
|
||||
replaces, from HBM and from pinned host memory alike, and must contribute
|
||||
zeros for rows another TP rank owns."""
|
||||
@@ -729,8 +733,13 @@ def test_engram_lookup_matches_torch(cpu_offload, background, num_tokens):
|
||||
layer.vocab_start_idx,
|
||||
layer.vocab_end_idx,
|
||||
)
|
||||
out = torch.empty(num_tokens, cols, layer.dim, dtype=torch.bfloat16, device="cuda")
|
||||
width = layer.dim + layer.dim // 32 if packed else layer.dim
|
||||
dtype = torch.uint8 if packed else torch.bfloat16
|
||||
out = torch.empty(num_tokens, cols, width, dtype=dtype, device="cuda")
|
||||
layer.lookup(ids, out, background=background)
|
||||
if packed:
|
||||
values = out[..., : layer.dim].view(torch.float8_e4m3fn)
|
||||
out = dequant_mxfp8_to_bf16(values, out[..., layer.dim :])
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user