[Bugfix] Initialize DeepGemmQuantScaleFMT oracle lazily; bound QuantFP8 UE8M0 packed path to group_size 128 (#51359)
Signed-off-by: BabyDrangoner <148877251+BabyDrangoner@users.noreply.github.com> Signed-off-by: kiroxu <148877251+BabyDrangoner@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Copilot
Wentao Ye
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d20a031d33
commit
19d61b1273
@@ -73,6 +73,21 @@ def test_quantfp8_group_functionality(
|
||||
if is_divisible:
|
||||
x_quant_cuda, scales_cuda = quant_op.forward_cuda(x.clone())
|
||||
assert x_quant_cuda.shape == x.shape
|
||||
|
||||
if scales_cuda.dtype == torch.int32:
|
||||
# DeepGEMM UE8M0 fast path (group_size == 128 on Blackwell with
|
||||
# the UE8M0 oracle): scales are packed 4-per-int32 in a
|
||||
# TMA-aligned layout. Unpack and compare against native.
|
||||
assert use_ue8m0 and group_size == 128
|
||||
packed_groups = (expected_num_groups + 3) // 4
|
||||
assert scales_cuda.shape == (batch_size, packed_groups)
|
||||
unpacked = (
|
||||
scales_cuda.contiguous()
|
||||
.view(torch.uint8)
|
||||
.reshape(batch_size, -1)[:, :expected_num_groups]
|
||||
)
|
||||
scales_cuda = torch.pow(2.0, unpacked.float() - 127)
|
||||
|
||||
assert scales_cuda.shape == (batch_size, expected_num_groups)
|
||||
|
||||
# Verify CUDA/native consistency
|
||||
|
||||
@@ -94,6 +94,8 @@ class QuantFP8(CustomOp):
|
||||
self.is_group_quant
|
||||
and self.use_ue8m0
|
||||
and self.use_deep_gemm_supported
|
||||
and not self.static
|
||||
and self.group_size == 128
|
||||
and (DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0)
|
||||
):
|
||||
return fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
|
||||
+13
-2
@@ -84,9 +84,20 @@ class DeepGemmQuantScaleFMT(Enum):
|
||||
|
||||
@classmethod
|
||||
def from_oracle(cls) -> "DeepGemmQuantScaleFMT":
|
||||
"""Return the pre-initialized oracle decision"""
|
||||
"""Return the oracle decision, initializing it on first use.
|
||||
|
||||
The cache is normally populated by ``_lazy_init()`` (e.g. during
|
||||
engine startup), but standalone consumers such as ``QuantFP8`` with an
|
||||
explicit ``use_ue8m0=True`` can reach this before any DeepGEMM kernel
|
||||
wrapper has run. Resolve the DeepGEMM symbols and initialize the
|
||||
decision here instead of asserting; without DeepGEMM this yields
|
||||
FLOAT32, matching ``is_deep_gemm_e8m0_used()``.
|
||||
"""
|
||||
cached = getattr(cls, "_oracle_cache", None)
|
||||
assert cached is not None, "DeepGemmQuantScaleFMT oracle cache not initialized"
|
||||
if cached is None:
|
||||
_lazy_init()
|
||||
cls.init_oracle_cache()
|
||||
cached = cls._oracle_cache # type: ignore[attr-defined]
|
||||
return cached
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user