[Bugfix] Initialize DeepGemmQuantScaleFMT oracle lazily; bound QuantFP8 UE8M0 packed path to group_size 128 (#51359)

Signed-off-by: BabyDrangoner <148877251+BabyDrangoner@users.noreply.github.com>
Signed-off-by: kiroxu <148877251+BabyDrangoner@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
kiroxu
2026-08-12 14:27:50 -04:00
committed by GitHub
co-authored by Copilot Wentao Ye mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d20a031d33
commit 19d61b1273
3 changed files with 30 additions and 2 deletions
@@ -73,6 +73,21 @@ def test_quantfp8_group_functionality(
if is_divisible:
x_quant_cuda, scales_cuda = quant_op.forward_cuda(x.clone())
assert x_quant_cuda.shape == x.shape
if scales_cuda.dtype == torch.int32:
# DeepGEMM UE8M0 fast path (group_size == 128 on Blackwell with
# the UE8M0 oracle): scales are packed 4-per-int32 in a
# TMA-aligned layout. Unpack and compare against native.
assert use_ue8m0 and group_size == 128
packed_groups = (expected_num_groups + 3) // 4
assert scales_cuda.shape == (batch_size, packed_groups)
unpacked = (
scales_cuda.contiguous()
.view(torch.uint8)
.reshape(batch_size, -1)[:, :expected_num_groups]
)
scales_cuda = torch.pow(2.0, unpacked.float() - 127)
assert scales_cuda.shape == (batch_size, expected_num_groups)
# Verify CUDA/native consistency
@@ -94,6 +94,8 @@ class QuantFP8(CustomOp):
self.is_group_quant
and self.use_ue8m0
and self.use_deep_gemm_supported
and not self.static
and self.group_size == 128
and (DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0)
):
return fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm(
+13 -2
View File
@@ -84,9 +84,20 @@ class DeepGemmQuantScaleFMT(Enum):
@classmethod
def from_oracle(cls) -> "DeepGemmQuantScaleFMT":
"""Return the pre-initialized oracle decision"""
"""Return the oracle decision, initializing it on first use.
The cache is normally populated by ``_lazy_init()`` (e.g. during
engine startup), but standalone consumers such as ``QuantFP8`` with an
explicit ``use_ue8m0=True`` can reach this before any DeepGEMM kernel
wrapper has run. Resolve the DeepGEMM symbols and initialize the
decision here instead of asserting; without DeepGEMM this yields
FLOAT32, matching ``is_deep_gemm_e8m0_used()``.
"""
cached = getattr(cls, "_oracle_cache", None)
assert cached is not None, "DeepGemmQuantScaleFMT oracle cache not initialized"
if cached is None:
_lazy_init()
cls.init_oracle_cache()
cached = cls._oracle_cache # type: ignore[attr-defined]
return cached