From 19d61b1273a56e0ebd8a9160bffb2e75dccad82b Mon Sep 17 00:00:00 2001 From: kiroxu <148877251+BabyDrangoner@users.noreply.github.com> Date: Thu, 13 Aug 2026 02:27:50 +0800 Subject: [PATCH] [Bugfix] Initialize DeepGemmQuantScaleFMT oracle lazily; bound QuantFP8 UE8M0 packed path to group_size 128 (#51359) Signed-off-by: BabyDrangoner <148877251+BabyDrangoner@users.noreply.github.com> Signed-off-by: kiroxu <148877251+BabyDrangoner@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../kernels/quantization/test_fp8_quant_group.py | 15 +++++++++++++++ .../layers/quantization/input_quant_fp8.py | 2 ++ vllm/utils/deep_gemm.py | 15 +++++++++++++-- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/kernels/quantization/test_fp8_quant_group.py b/tests/kernels/quantization/test_fp8_quant_group.py index 113afb3c10..977b64e2b1 100644 --- a/tests/kernels/quantization/test_fp8_quant_group.py +++ b/tests/kernels/quantization/test_fp8_quant_group.py @@ -73,6 +73,21 @@ def test_quantfp8_group_functionality( if is_divisible: x_quant_cuda, scales_cuda = quant_op.forward_cuda(x.clone()) assert x_quant_cuda.shape == x.shape + + if scales_cuda.dtype == torch.int32: + # DeepGEMM UE8M0 fast path (group_size == 128 on Blackwell with + # the UE8M0 oracle): scales are packed 4-per-int32 in a + # TMA-aligned layout. Unpack and compare against native. + assert use_ue8m0 and group_size == 128 + packed_groups = (expected_num_groups + 3) // 4 + assert scales_cuda.shape == (batch_size, packed_groups) + unpacked = ( + scales_cuda.contiguous() + .view(torch.uint8) + .reshape(batch_size, -1)[:, :expected_num_groups] + ) + scales_cuda = torch.pow(2.0, unpacked.float() - 127) + assert scales_cuda.shape == (batch_size, expected_num_groups) # Verify CUDA/native consistency diff --git a/vllm/model_executor/layers/quantization/input_quant_fp8.py b/vllm/model_executor/layers/quantization/input_quant_fp8.py index 2eb34630aa..2d9cf624ab 100644 --- a/vllm/model_executor/layers/quantization/input_quant_fp8.py +++ b/vllm/model_executor/layers/quantization/input_quant_fp8.py @@ -94,6 +94,8 @@ class QuantFP8(CustomOp): self.is_group_quant and self.use_ue8m0 and self.use_deep_gemm_supported + and not self.static + and self.group_size == 128 and (DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0) ): return fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm( diff --git a/vllm/utils/deep_gemm.py b/vllm/utils/deep_gemm.py index 0a1644bcc5..502f046ff9 100644 --- a/vllm/utils/deep_gemm.py +++ b/vllm/utils/deep_gemm.py @@ -84,9 +84,20 @@ class DeepGemmQuantScaleFMT(Enum): @classmethod def from_oracle(cls) -> "DeepGemmQuantScaleFMT": - """Return the pre-initialized oracle decision""" + """Return the oracle decision, initializing it on first use. + + The cache is normally populated by ``_lazy_init()`` (e.g. during + engine startup), but standalone consumers such as ``QuantFP8`` with an + explicit ``use_ue8m0=True`` can reach this before any DeepGEMM kernel + wrapper has run. Resolve the DeepGEMM symbols and initialize the + decision here instead of asserting; without DeepGEMM this yields + FLOAT32, matching ``is_deep_gemm_e8m0_used()``. + """ cached = getattr(cls, "_oracle_cache", None) - assert cached is not None, "DeepGemmQuantScaleFMT oracle cache not initialized" + if cached is None: + _lazy_init() + cls.init_oracle_cache() + cached = cls._oracle_cache # type: ignore[attr-defined] return cached