[Model] Support DeepSeek-V4.1-Flash (#56214)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jee Jee Li
andyluo7
OpenAI Codex
zjy0516
Claude Opus 5
parent
e6821ceac9
commit
e77daef89e
@@ -159,13 +159,12 @@ def _ref_ue8m0_quant_block(x_f32: torch.Tensor) -> tuple[torch.Tensor, torch.Ten
|
||||
absmax -> 2^ceil(log2(absmax / fp8_max)) -> clamp(x / scale) -> fp8
|
||||
|
||||
Args:
|
||||
x_f32: [..., quant_group_size] float32 — one or more 128-element blocks.
|
||||
x_f32: [..., quant_group_size] float32 quantization blocks.
|
||||
|
||||
Returns:
|
||||
x_fp8: same shape, float8_e4m3fn
|
||||
scales: [...] float32, one scale per block
|
||||
"""
|
||||
assert x_f32.shape[-1] == QUANT_GROUP_SIZE
|
||||
absmax = x_f32.abs().amax(dim=-1, keepdim=True).clamp(min=EPS)
|
||||
scale_raw = absmax * (1.0 / FP8_MAX)
|
||||
scale = torch.exp2(torch.ceil(torch.log2(scale_raw)))
|
||||
@@ -253,6 +252,56 @@ def reference_inv_rope_fp8_quant(
|
||||
# =========================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128, 4096])
|
||||
@pytest.mark.parametrize("quant_group_size", [32, 128])
|
||||
@torch.inference_mode()
|
||||
def test_packed_quantization_preserves_rotary_dimensions(num_tokens, quant_group_size):
|
||||
"""RoPE may span multiple scale blocks; pack every scale and padding row."""
|
||||
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
|
||||
fused_inv_rope_fp8_quant as quantize,
|
||||
)
|
||||
|
||||
torch.manual_seed(42)
|
||||
groups, heads_per_group = 2, 2
|
||||
x = torch.randn(
|
||||
num_tokens,
|
||||
groups * heads_per_group,
|
||||
HEAD_DIM,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
cache = make_cos_sin_cache(4096)
|
||||
positions = torch.randint(4096, (num_tokens,), device="cuda")
|
||||
q, packed_scales = quantize(
|
||||
x,
|
||||
positions,
|
||||
cache,
|
||||
groups,
|
||||
heads_per_group,
|
||||
quant_group_size=quant_group_size,
|
||||
tma_aligned_scales=True,
|
||||
)
|
||||
scale_bytes = torch.stack(
|
||||
[(packed_scales >> (8 * i)) & 0xFF for i in range(4)], dim=-1
|
||||
).flatten(-2)
|
||||
scales = torch.exp2(scale_bytes.float() - 127)
|
||||
rotated = reference_inv_rope(x.float(), positions, cache)
|
||||
ref_q, ref_scales = _ref_ue8m0_quant_block(
|
||||
rotated.reshape(num_tokens, groups, -1, quant_group_size)
|
||||
)
|
||||
torch.testing.assert_close(scales, ref_scales, rtol=0, atol=0)
|
||||
actual = q.float().unflatten(-1, (-1, quant_group_size)) * scales.unsqueeze(-1)
|
||||
expected = ref_q.float() * ref_scales.unsqueeze(-1)
|
||||
assert (actual - expected).norm() / expected.norm() < 1e-3
|
||||
|
||||
aligned_tokens = (num_tokens + 3) // 4 * 4
|
||||
full_scales = packed_scales.transpose(0, 1).as_strided(
|
||||
(groups, aligned_tokens, packed_scales.shape[-1]),
|
||||
(packed_scales.stride(1), 1, packed_scales.stride(2)),
|
||||
)
|
||||
assert torch.count_nonzero(full_scales[:, num_tokens:]) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads,n_groups",
|
||||
@@ -260,8 +309,9 @@ def reference_inv_rope_fp8_quant(
|
||||
ids=["H64_G8", "H32_G4", "H128_G8"],
|
||||
)
|
||||
@pytest.mark.parametrize("seed", [0, 42])
|
||||
@pytest.mark.parametrize("quantize", [True, False], ids=["fp8", "bf16"])
|
||||
@torch.inference_mode()
|
||||
def test_correctness(num_tokens, num_heads, n_groups, seed):
|
||||
def test_correctness(num_tokens, num_heads, n_groups, seed, quantize):
|
||||
"""Compare fused kernel against reference for FP8 values and scales."""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
@@ -280,6 +330,26 @@ def test_correctness(num_tokens, num_heads, n_groups, seed):
|
||||
max_pos, ROPE_DIM, dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
if not quantize:
|
||||
ref_output = reference_inv_rope(o, positions, cos_sin_cache).view(
|
||||
num_tokens, n_groups, -1
|
||||
)
|
||||
fused_output, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
quantize=False,
|
||||
)
|
||||
d = heads_per_group * HEAD_DIM
|
||||
assert fused_output.shape == (num_tokens, n_groups, d)
|
||||
assert fused_scale.numel() == 0
|
||||
assert fused_output.dtype == torch.bfloat16
|
||||
assert fused_output.stride() == (d, num_tokens * d, 1)
|
||||
torch.testing.assert_close(fused_output, ref_output, rtol=1e-2, atol=1e-2)
|
||||
return
|
||||
|
||||
# Reference
|
||||
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
@@ -726,6 +796,7 @@ def test_einsum_end_to_end(num_tokens, num_heads, n_groups):
|
||||
This catches stride/layout bugs that only manifest when the einsum
|
||||
kernel actually consumes the quantized activations.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_einsum,
|
||||
is_deep_gemm_supported,
|
||||
@@ -793,23 +864,23 @@ def test_einsum_end_to_end(num_tokens, num_heads, n_groups):
|
||||
)
|
||||
|
||||
# -- FUSED path --
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
wo_a = torch.nn.Module()
|
||||
wo_a.register_buffer("weight", w_fp8)
|
||||
wo_a.register_buffer("weight_scale", w_scale_t)
|
||||
z_fused = deep_gemm_fp8_o_proj(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
z_fused = torch.empty(
|
||||
num_tokens, n_groups, o_lora_rank, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
fp8_einsum(
|
||||
"bhr,hdr->bhd",
|
||||
(fused_fp8, fused_scale),
|
||||
(w_fp8, w_scale_t),
|
||||
z_fused,
|
||||
recipe=recipe,
|
||||
)
|
||||
wo_a,
|
||||
torch.nn.Identity(),
|
||||
n_groups=n_groups,
|
||||
heads_per_group=heads_per_group,
|
||||
nope_dim=NOPE_DIM,
|
||||
rope_dim=ROPE_DIM,
|
||||
o_lora_rank=o_lora_rank,
|
||||
einsum_recipe=recipe,
|
||||
tma_aligned_scales=True,
|
||||
).view_as(z_ref)
|
||||
|
||||
# -- Checks --
|
||||
# Einsum output: Triton and CUDA both rotate in fp32 now, so diffs
|
||||
|
||||
Reference in New Issue
Block a user