[Model] Support DeepSeek-V4.1-Flash (#56214)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-11 02:11:19 -07:00
committed by GitHub
co-authored by Jee Jee Li andyluo7 OpenAI Codex zjy0516 Claude Opus 5
parent e6821ceac9
commit e77daef89e
49 changed files with 3996 additions and 138 deletions
+88 -17
View File
@@ -159,13 +159,12 @@ def _ref_ue8m0_quant_block(x_f32: torch.Tensor) -> tuple[torch.Tensor, torch.Ten
absmax -> 2^ceil(log2(absmax / fp8_max)) -> clamp(x / scale) -> fp8
Args:
x_f32: [..., quant_group_size] float32 — one or more 128-element blocks.
x_f32: [..., quant_group_size] float32 quantization blocks.
Returns:
x_fp8: same shape, float8_e4m3fn
scales: [...] float32, one scale per block
"""
assert x_f32.shape[-1] == QUANT_GROUP_SIZE
absmax = x_f32.abs().amax(dim=-1, keepdim=True).clamp(min=EPS)
scale_raw = absmax * (1.0 / FP8_MAX)
scale = torch.exp2(torch.ceil(torch.log2(scale_raw)))
@@ -253,6 +252,56 @@ def reference_inv_rope_fp8_quant(
# =========================================================================
@pytest.mark.parametrize("num_tokens", [1, 7, 128, 4096])
@pytest.mark.parametrize("quant_group_size", [32, 128])
@torch.inference_mode()
def test_packed_quantization_preserves_rotary_dimensions(num_tokens, quant_group_size):
"""RoPE may span multiple scale blocks; pack every scale and padding row."""
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
fused_inv_rope_fp8_quant as quantize,
)
torch.manual_seed(42)
groups, heads_per_group = 2, 2
x = torch.randn(
num_tokens,
groups * heads_per_group,
HEAD_DIM,
device="cuda",
dtype=torch.bfloat16,
)
cache = make_cos_sin_cache(4096)
positions = torch.randint(4096, (num_tokens,), device="cuda")
q, packed_scales = quantize(
x,
positions,
cache,
groups,
heads_per_group,
quant_group_size=quant_group_size,
tma_aligned_scales=True,
)
scale_bytes = torch.stack(
[(packed_scales >> (8 * i)) & 0xFF for i in range(4)], dim=-1
).flatten(-2)
scales = torch.exp2(scale_bytes.float() - 127)
rotated = reference_inv_rope(x.float(), positions, cache)
ref_q, ref_scales = _ref_ue8m0_quant_block(
rotated.reshape(num_tokens, groups, -1, quant_group_size)
)
torch.testing.assert_close(scales, ref_scales, rtol=0, atol=0)
actual = q.float().unflatten(-1, (-1, quant_group_size)) * scales.unsqueeze(-1)
expected = ref_q.float() * ref_scales.unsqueeze(-1)
assert (actual - expected).norm() / expected.norm() < 1e-3
aligned_tokens = (num_tokens + 3) // 4 * 4
full_scales = packed_scales.transpose(0, 1).as_strided(
(groups, aligned_tokens, packed_scales.shape[-1]),
(packed_scales.stride(1), 1, packed_scales.stride(2)),
)
assert torch.count_nonzero(full_scales[:, num_tokens:]) == 0
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
@pytest.mark.parametrize(
"num_heads,n_groups",
@@ -260,8 +309,9 @@ def reference_inv_rope_fp8_quant(
ids=["H64_G8", "H32_G4", "H128_G8"],
)
@pytest.mark.parametrize("seed", [0, 42])
@pytest.mark.parametrize("quantize", [True, False], ids=["fp8", "bf16"])
@torch.inference_mode()
def test_correctness(num_tokens, num_heads, n_groups, seed):
def test_correctness(num_tokens, num_heads, n_groups, seed, quantize):
"""Compare fused kernel against reference for FP8 values and scales."""
torch.manual_seed(seed)
@@ -280,6 +330,26 @@ def test_correctness(num_tokens, num_heads, n_groups, seed):
max_pos, ROPE_DIM, dtype=torch.float32, device=device
)
if not quantize:
ref_output = reference_inv_rope(o, positions, cos_sin_cache).view(
num_tokens, n_groups, -1
)
fused_output, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
quantize=False,
)
d = heads_per_group * HEAD_DIM
assert fused_output.shape == (num_tokens, n_groups, d)
assert fused_scale.numel() == 0
assert fused_output.dtype == torch.bfloat16
assert fused_output.stride() == (d, num_tokens * d, 1)
torch.testing.assert_close(fused_output, ref_output, rtol=1e-2, atol=1e-2)
return
# Reference
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
o.clone(),
@@ -726,6 +796,7 @@ def test_einsum_end_to_end(num_tokens, num_heads, n_groups):
This catches stride/layout bugs that only manifest when the einsum
kernel actually consumes the quantized activations.
"""
from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj
from vllm.utils.deep_gemm import (
fp8_einsum,
is_deep_gemm_supported,
@@ -793,23 +864,23 @@ def test_einsum_end_to_end(num_tokens, num_heads, n_groups):
)
# -- FUSED path --
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
wo_a = torch.nn.Module()
wo_a.register_buffer("weight", w_fp8)
wo_a.register_buffer("weight_scale", w_scale_t)
z_fused = deep_gemm_fp8_o_proj(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
z_fused = torch.empty(
num_tokens, n_groups, o_lora_rank, device=device, dtype=torch.bfloat16
)
fp8_einsum(
"bhr,hdr->bhd",
(fused_fp8, fused_scale),
(w_fp8, w_scale_t),
z_fused,
recipe=recipe,
)
wo_a,
torch.nn.Identity(),
n_groups=n_groups,
heads_per_group=heads_per_group,
nope_dim=NOPE_DIM,
rope_dim=ROPE_DIM,
o_lora_rank=o_lora_rank,
einsum_recipe=recipe,
tma_aligned_scales=True,
).view_as(z_ref)
# -- Checks --
# Einsum output: Triton and CUDA both rotate in fp32 now, so diffs