[Perf][DSV4.1] Overlap mHC coefficients for small TP batches (#57603)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -266,6 +266,77 @@ def test_deepseek_v41_mhc_pre_delayed(
|
||||
torch.testing.assert_close(actual[2], expected[2], atol=1.6e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not HAS_TILELANG_MHC or not current_platform.is_device_capability_family(100),
|
||||
reason="SM100 TileLang mHC required",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 8, 9, 32, 128, 256])
|
||||
@pytest.mark.parametrize("entry", ["broadcast", "identity", "carried"])
|
||||
@pytest.mark.parametrize("hidden_size,hc_mult", [(4096, 2), (5120, 4), (7168, 4)])
|
||||
def test_deepseek_v41_mhc_overlap_preserves_outputs_on_replay(
|
||||
num_tokens, entry, hidden_size, hc_mult
|
||||
):
|
||||
"""The input is usable before joining; coefficients stay exact on graph replay."""
|
||||
from vllm.models.deepseek_v41.nvidia.ops.mhc import mhc_pre_delayed_overlap
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
if not is_deep_gemm_supported():
|
||||
pytest.skip("DeepGEMM required")
|
||||
set_random_seed(42)
|
||||
mix_size = hc_mult * (hc_mult + 2)
|
||||
residual = torch.randn(
|
||||
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
fn = torch.randn(mix_size, hc_mult, hidden_size, device=DEVICE) * 0.02
|
||||
x = None
|
||||
pre_mix = None
|
||||
if entry == "broadcast":
|
||||
x = residual[:, 0].contiguous()
|
||||
residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
|
||||
fn = fn.sum(1)
|
||||
else:
|
||||
fn = fn.flatten(1)
|
||||
if entry == "carried":
|
||||
pre_mix = torch.rand(num_tokens, hc_mult, device=DEVICE)
|
||||
scale = torch.tensor([0.5, 0.25, 1.0], device=DEVICE)
|
||||
base = torch.randn(mix_size, device=DEVICE)
|
||||
weight = torch.empty(hidden_size, dtype=torch.bfloat16, device=DEVICE).uniform_(
|
||||
0.5, 1.5
|
||||
)
|
||||
args = (residual, fn, scale, base, 1e-20, 1e-6, 1e-6, 2.0, 20)
|
||||
kwargs = dict(pre_mix=pre_mix, x=x, norm_weight=weight, norm_eps=1e-20)
|
||||
side = torch.cuda.Stream()
|
||||
|
||||
def run():
|
||||
outputs = mhc_pre_delayed_overlap(*args, **kwargs, stream=side)
|
||||
consumed_input = outputs[2].clone()
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
return (*outputs, consumed_input)
|
||||
|
||||
def check(actual):
|
||||
expected = mhc_pre_delayed_tilelang(*args, **kwargs)
|
||||
for a, b in zip(actual[:4], expected, strict=True):
|
||||
torch.testing.assert_close(a, b, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual[4], expected[2], atol=0, rtol=0)
|
||||
|
||||
check(run())
|
||||
if num_tokens == 0:
|
||||
return
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = run()
|
||||
for _ in range(5):
|
||||
if x is not None:
|
||||
x.normal_()
|
||||
residual.copy_(x.unsqueeze(1).expand_as(residual))
|
||||
else:
|
||||
residual.normal_()
|
||||
if pre_mix is not None:
|
||||
pre_mix.uniform_()
|
||||
graph.replay()
|
||||
check(captured)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_delayed_ref(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -433,10 +504,16 @@ def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"])
|
||||
@pytest.mark.parametrize(
|
||||
"mhc_mode", ["disabled", "overlap", "piecewise", "large_batch"]
|
||||
)
|
||||
def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
entry, monkeypatch, default_vllm_config
|
||||
entry, mhc_mode, monkeypatch, default_vllm_config
|
||||
):
|
||||
"""Preserve carried pre-mixes and Engram ordering at every decoder entry."""
|
||||
from vllm.models.deepseek_v41.nvidia.ops.mhc import MHC_OVERLAP_MAX_TOKENS
|
||||
|
||||
num_tokens = MHC_OVERLAP_MAX_TOKENS + 1 if mhc_mode == "large_batch" else 3
|
||||
set_random_seed(0)
|
||||
decoder = DeepseekV41DecoderLayer.__new__(DeepseekV41DecoderLayer)
|
||||
nn.Module.__init__(decoder)
|
||||
@@ -446,6 +523,29 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
decoder.rms_norm_eps = 1e-20
|
||||
decoder.hc_post_alpha = 2.0
|
||||
decoder.use_sequence_parallel = False
|
||||
decoder.mhc_stream = None
|
||||
if mhc_mode != "disabled":
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
if (
|
||||
not current_platform.is_device_capability_family(100)
|
||||
or not is_deep_gemm_supported()
|
||||
):
|
||||
pytest.skip("SM100 DeepGEMM required for overlap")
|
||||
decoder.mhc_stream = torch.cuda.Stream()
|
||||
if mhc_mode in ("piecewise", "large_batch"):
|
||||
|
||||
def unexpected_overlap(*args, **kwargs):
|
||||
pytest.fail("unsupported execution must retain native mHC")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_overlap",
|
||||
unexpected_overlap,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v41.nvidia.ops.mhc.mhc_pre_delayed_overlap",
|
||||
unexpected_overlap,
|
||||
)
|
||||
decoder.engram = None
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
@@ -461,18 +561,18 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
decoder.hc_attn_scale = decoder.hc_ffn_scale = torch.ones(3)
|
||||
decoder.hc_attn_base = torch.randn(24)
|
||||
decoder.hc_ffn_base = torch.randn(24)
|
||||
x = torch.randn(3, 5120, dtype=torch.bfloat16)
|
||||
positions = torch.arange(3)
|
||||
x = torch.randn(num_tokens, 5120, dtype=torch.bfloat16)
|
||||
positions = torch.arange(num_tokens)
|
||||
kwargs = {}
|
||||
if entry != "broadcast":
|
||||
kwargs["pre_mix"] = torch.rand(3, 4)
|
||||
kwargs["pre_mix"] = torch.rand(num_tokens, 4)
|
||||
if entry == "pipeline":
|
||||
x = torch.randn(3, 4, 5120, dtype=torch.bfloat16)
|
||||
x = torch.randn(num_tokens, 4, 5120, dtype=torch.bfloat16)
|
||||
if entry in ("residual", "engram"):
|
||||
kwargs.update(
|
||||
residual=torch.randn(3, 4, 5120, dtype=torch.bfloat16),
|
||||
post_mix=torch.rand(3, 4, 1),
|
||||
res_mix=torch.rand(3, 4, 4),
|
||||
residual=torch.randn(num_tokens, 4, 5120, dtype=torch.bfloat16),
|
||||
post_mix=torch.rand(num_tokens, 4, 1),
|
||||
res_mix=torch.rand(num_tokens, 4, 4),
|
||||
)
|
||||
if entry == "engram":
|
||||
|
||||
@@ -483,15 +583,22 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
return residual + 0.125
|
||||
|
||||
decoder.engram = FakeEngram()
|
||||
kwargs["engram_hashes"] = torch.zeros(3, 1, 1, dtype=torch.int32)
|
||||
kwargs["engram_hashes"] = torch.zeros(num_tokens, 1, 1, dtype=torch.int32)
|
||||
|
||||
actual = decoder(x, positions, None, **kwargs)
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.forward_context import set_forward_context
|
||||
|
||||
mode = CUDAGraphMode.PIECEWISE if mhc_mode == "piecewise" else CUDAGraphMode.NONE
|
||||
with set_forward_context(None, default_vllm_config, cudagraph_runtime_mode=mode):
|
||||
actual = decoder(x, positions, None, **kwargs)
|
||||
|
||||
def reference(*args, norm_weight, norm_eps, **kwargs):
|
||||
post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs)
|
||||
return post, res, decoder.attn_norm(collapsed), pre
|
||||
|
||||
def fused_reference(x, residual, post_mix, res_mix, *args, capture_aux=False, **kw):
|
||||
def fused_reference(
|
||||
x, residual, post_mix, res_mix, *args, capture_aux=False, stream=None, **kw
|
||||
):
|
||||
residual = mhc_post_torch(x, residual, post_mix, res_mix)
|
||||
aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0)
|
||||
return residual, *reference(residual, *args, **kw), aux
|
||||
@@ -500,6 +607,7 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
"vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_tilelang",
|
||||
reference,
|
||||
)
|
||||
decoder.mhc_stream = None
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v41.nvidia.model.mhc_shifted_post_pre",
|
||||
fused_reference,
|
||||
@@ -530,6 +638,7 @@ def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_conf
|
||||
decoder.rms_norm_eps = 1e-6
|
||||
decoder.hc_post_alpha = 2.0
|
||||
decoder.use_sequence_parallel = False
|
||||
decoder.mhc_stream = None
|
||||
decoder.engram = None
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
@@ -772,7 +881,8 @@ def hc_head_ref(
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [4096, 7168])
|
||||
@pytest.mark.parametrize("hc_mult", [4])
|
||||
def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult):
|
||||
@pytest.mark.parametrize("fused_norm", [False, True])
|
||||
def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult, fused_norm):
|
||||
torch.set_default_device(DEVICE)
|
||||
set_random_seed(0)
|
||||
|
||||
@@ -802,6 +912,12 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult):
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
norm_kwargs = {}
|
||||
if fused_norm:
|
||||
# Non-shifted mHC must still use its freshly computed pre-mix.
|
||||
weight = torch.ones(hidden_size, dtype=torch.bfloat16)
|
||||
norm_kwargs = dict(norm_weight=weight, norm_eps=rms_eps)
|
||||
ref = (*ref[:2], F.rms_norm(ref[2], (hidden_size,), weight, rms_eps))
|
||||
out = torch.ops.vllm.mhc_pre_tilelang(
|
||||
residual,
|
||||
fn,
|
||||
@@ -812,6 +928,7 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult):
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
**norm_kwargs,
|
||||
)
|
||||
|
||||
for actual, expected in zip(out, ref, strict=True):
|
||||
|
||||
Reference in New Issue
Block a user