[ROCm][Perf] Fuse SWA q/kv RMSNorm and q FP8 group quant for DeepSeek-V4 (#53540)
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
@@ -24,7 +24,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm._aiter_ops # noqa: F401 - ensure ops are registered
|
||||
from tests.kernels.utils import bf16_ulp_distance
|
||||
from tests.kernels.utils import bf16_ulp_distance, fp8_ulp_distance
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -531,6 +531,81 @@ def test_rocm_aiter_rmsnorm_with_add_fp8_group_quant_residual_accuracy():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [1, 32])
|
||||
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_vs_sequential(m):
|
||||
"""Fused q/kv RMSNorm + q group-quant matches rms_norm -> group_fp8_quant.
|
||||
|
||||
M=1 covers the single-token decode case. M=0 is guarded by the caller
|
||||
(the ROCm DeepSeek-V4 attention checks num_tokens > 0) and the op has
|
||||
no zero-row guard itself, so it is intentionally not tested here.
|
||||
"""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
nq, nk, group_size = 1536, 576, 128
|
||||
q = torch.randn(m, nq, dtype=torch.bfloat16)
|
||||
kv = torch.randn(m, nk, dtype=torch.bfloat16)
|
||||
q_weight = torch.ones(nq, dtype=torch.bfloat16)
|
||||
kv_weight = torch.ones(nk, dtype=torch.bfloat16)
|
||||
eps = 1e-5
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
|
||||
fused_q, fused_scales, fused_kv = rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
|
||||
q, q_weight, eps, kv, kv_weight, eps, group_size=group_size
|
||||
)
|
||||
_assert_quant_shapes(fused_q, fused_scales, m, nq, fp8_dtype, nq // group_size)
|
||||
assert fused_kv.shape == (m, nk)
|
||||
assert fused_kv.dtype == kv.dtype
|
||||
|
||||
# q: dequantized comparison against the sequential composition.
|
||||
normed_q = rocm_aiter_ops.rms_norm(q, q_weight, eps)
|
||||
ref_q, ref_scales = rocm_aiter_ops.group_fp8_quant(normed_q, group_size)
|
||||
_assert_rel_error_quality(
|
||||
_dequantize_grouped(fused_q, fused_scales, group_size),
|
||||
_dequantize_grouped(ref_q, ref_scales, group_size),
|
||||
label="mla_dual_rmsnorm_group_quant",
|
||||
mean_limit=0.05,
|
||||
preferred_rel=0.05,
|
||||
max_rel=0.5,
|
||||
max_fail_rate=0.01,
|
||||
)
|
||||
|
||||
# kv: both sides are aiter RMSNorm kernels; allow one bf16 ULP.
|
||||
ref_kv = rocm_aiter_ops.rms_norm(kv, kv_weight, eps)
|
||||
max_ulp = int(bf16_ulp_distance(fused_kv, ref_kv).max().item())
|
||||
assert max_ulp <= 1, f"fused kv: max ULP {max_ulp} > 1"
|
||||
|
||||
|
||||
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view():
|
||||
"""Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match
|
||||
contiguous inputs."""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
m, nq, nk, group_size = 16, 1536, 576, 128
|
||||
qr_kv = torch.randn(m, nq + nk, dtype=torch.bfloat16)
|
||||
q_view, kv_view = qr_kv.split([nq, nk], dim=-1)
|
||||
assert not q_view.is_contiguous()
|
||||
q_weight, kv_weight = torch.ones(nq + nk, dtype=torch.bfloat16).split([nq, nk])
|
||||
eps = 1e-5
|
||||
|
||||
def run(q, kv):
|
||||
return rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
|
||||
q, q_weight, eps, kv, kv_weight, eps, group_size=group_size
|
||||
)
|
||||
|
||||
out_strided = run(q_view, kv_view)
|
||||
out_contig = run(q_view.contiguous(), kv_view.contiguous())
|
||||
|
||||
q_ulp = int(fp8_ulp_distance(out_strided[0], out_contig[0]).max().item())
|
||||
kv_ulp = int(bf16_ulp_distance(out_strided[2], out_contig[2]).max().item())
|
||||
assert q_ulp <= 1, f"strided q: max fp8 ULP {q_ulp} > 1"
|
||||
assert kv_ulp <= 1, f"strided kv: max bf16 ULP {kv_ulp} > 1"
|
||||
torch.testing.assert_close(out_strided[1], out_contig[1])
|
||||
|
||||
|
||||
# -- End-to-end inference chain test ---------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -1455,6 +1455,70 @@ def _fused_mla_dual_rms_norm_per_token_quant_fake(
|
||||
return q_out, q_scale, kv_normed
|
||||
|
||||
|
||||
def _fused_mla_dual_rms_norm_group_quant_impl(
|
||||
q: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
kv_weight: torch.Tensor,
|
||||
q_epsilon: float,
|
||||
kv_epsilon: float,
|
||||
group_size: int,
|
||||
transpose_scale: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Fused MLA q/kv RMSNorm (+ FP8 per-1x128 group quant on q) via AITER.
|
||||
|
||||
Backs the ``fused_mla_dual_rms_norm_group_quant`` custom op used by the
|
||||
DeepSeek-V4 ROCm attention path when the q latent is group-quantized
|
||||
(``(M, N/128)`` scales) for the block-scaled FP8 wq_b GEMMs. The q latent
|
||||
is quantized directly from the fp32 norm accumulator, skipping the bf16
|
||||
intermediate the standalone quant path would re-read; the kv latent is
|
||||
RMS-normed to bf16 and consumed by the fused insert kernel unchanged.
|
||||
"""
|
||||
from aiter.ops.fused_qk_rmsnorm_group_quant import (
|
||||
fused_qk_rmsnorm_group_quant,
|
||||
)
|
||||
|
||||
mq, nq = q.shape
|
||||
q_out = torch.empty((mq, nq), dtype=FP8_DTYPE, device=q.device)
|
||||
q_scale = torch.empty((mq, nq // group_size), dtype=torch.float32, device=q.device)
|
||||
kv_normed = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
|
||||
|
||||
# q -> RMSNorm + FP8 group quant (q slot); kv -> RMSNorm only (k slot).
|
||||
# `split` views are accepted directly (unit inner stride); the kernel
|
||||
# handles strided inputs, matching the aiter op-test usage.
|
||||
fused_qk_rmsnorm_group_quant(
|
||||
q_out_quantized=q_out,
|
||||
q_out_scale=q_scale,
|
||||
q=q,
|
||||
q_weight=q_weight,
|
||||
q_epsilon=q_epsilon,
|
||||
k_out=kv_normed,
|
||||
k=kv,
|
||||
k_weight=kv_weight,
|
||||
k_epsilon=kv_epsilon,
|
||||
group_size=group_size,
|
||||
transpose_scale=transpose_scale,
|
||||
)
|
||||
return q_out, q_scale, kv_normed
|
||||
|
||||
|
||||
def _fused_mla_dual_rms_norm_group_quant_fake(
|
||||
q: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
kv_weight: torch.Tensor,
|
||||
q_epsilon: float,
|
||||
kv_epsilon: float,
|
||||
group_size: int,
|
||||
transpose_scale: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
mq, nq = q.shape
|
||||
q_out = torch.empty((mq, nq), dtype=FP8_DTYPE, device=q.device)
|
||||
q_scale = torch.empty((mq, nq // group_size), dtype=torch.float32, device=q.device)
|
||||
kv_normed = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
|
||||
return q_out, q_scale, kv_normed
|
||||
|
||||
|
||||
def _rocm_aiter_gemm_a8wfp4_impl(
|
||||
x: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
@@ -2277,6 +2341,13 @@ class rocm_aiter_ops:
|
||||
fake_impl=_fused_mla_dual_rms_norm_per_token_quant_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_mla_dual_rms_norm_group_quant",
|
||||
op_func=_fused_mla_dual_rms_norm_group_quant_impl,
|
||||
mutates_args=[],
|
||||
fake_impl=_fused_mla_dual_rms_norm_group_quant_fake,
|
||||
)
|
||||
|
||||
_OPS_REGISTERED = True
|
||||
|
||||
@staticmethod
|
||||
@@ -2340,6 +2411,29 @@ class rocm_aiter_ops:
|
||||
def get_fused_mla_dual_rms_norm_per_token_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.fused_mla_dual_rms_norm_per_token_quant.default
|
||||
|
||||
@staticmethod
|
||||
def fused_qk_rmsnorm_group_quant(
|
||||
q: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
q_epsilon: float,
|
||||
kv: torch.Tensor,
|
||||
kv_weight: torch.Tensor,
|
||||
kv_epsilon: float,
|
||||
group_size: int = 128,
|
||||
transpose_scale: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Fused q/kv RMSNorm + per-1x128 FP8 group quant on q via AITER."""
|
||||
return torch.ops.vllm.fused_mla_dual_rms_norm_group_quant(
|
||||
q,
|
||||
q_weight,
|
||||
kv,
|
||||
kv_weight,
|
||||
q_epsilon,
|
||||
kv_epsilon,
|
||||
group_size,
|
||||
transpose_scale,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def rms_norm(
|
||||
x: torch.Tensor,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
from dataclasses import dataclass
|
||||
from typing import cast
|
||||
|
||||
@@ -60,6 +61,36 @@ def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
|
||||
return indptr
|
||||
|
||||
|
||||
def apply_pre_quantized_block_scaled_mm(
|
||||
linear: torch.nn.Module,
|
||||
x_fp8: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Block-scaled fp8 GEMM on pre-quantized activations.
|
||||
|
||||
The fused q/kv norm kernel writes fp8 qr + per-1x128 scales; this
|
||||
drives the linear's block-scaled GEMM directly with them, bypassing
|
||||
apply_weights which would re-quantize the fp8 input. Only valid for
|
||||
the wq_b-style column/replicated linears: their output is the local
|
||||
TP shard, so no all-reduce is needed.
|
||||
"""
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel import (
|
||||
FP8BlockParams,
|
||||
)
|
||||
|
||||
params = FP8BlockParams.from_layer(linear)
|
||||
weight_scale = (
|
||||
params.weight_scale
|
||||
if params.weight_scale_inv is None
|
||||
else params.weight_scale_inv
|
||||
)
|
||||
kernel = linear.quant_method.fp8_linear
|
||||
out = kernel.apply_block_scaled_mm(
|
||||
A=x_fp8, B=params.weight, As=x_scale, Bs=weight_scale
|
||||
)
|
||||
return out.to(dtype=kernel.config.out_dtype)
|
||||
|
||||
|
||||
# ROCm sparse prefill keeps this dense combine local so AMD-specific SWA changes
|
||||
# do not touch the shared DeepSeek V4 cache utilities.
|
||||
_SPARSE_PREFILL_TOPK_ALIGNMENT = 128
|
||||
@@ -621,6 +652,71 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
indexer_weights, _ = indexer.weights_proj(hidden_states)
|
||||
return qr_kv, kv_score, indexer_kv_score, indexer_weights
|
||||
|
||||
@functools.cached_property
|
||||
def _wq_b_uses_aiter_block_scaled(self) -> bool:
|
||||
"""True when both wq_b GEMMs run the aiter block-scaled fp8 kernel.
|
||||
|
||||
Cached: the linear kernels and the aiter env gates are fixed once
|
||||
the model is built, so this is evaluated at the first forward
|
||||
only.
|
||||
|
||||
The fused norm+quant path is only valid if the quant and GEMM it
|
||||
replaces are exactly the aiter ones; otherwise fall back to the
|
||||
shared path.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
Fp8BlockScaledMMLinearKernel,
|
||||
)
|
||||
|
||||
if not rocm_aiter_ops.is_linear_fp8_enabled():
|
||||
return False
|
||||
|
||||
linears = [self.wq_b]
|
||||
if self.indexer is not None:
|
||||
linears.append(self.indexer.wq_b)
|
||||
for linear in linears:
|
||||
kernel = getattr(getattr(linear, "quant_method", None), "fp8_linear", None)
|
||||
if not isinstance(kernel, Fp8BlockScaledMMLinearKernel):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _split_qkv_and_norm(
|
||||
self, qr_kv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
|
||||
"""Fuse q/kv RMSNorm + per-1x128 fp8 q quant into one aiter kernel.
|
||||
|
||||
The shared path norms q and kv in one triton kernel and the wq_b
|
||||
linears then re-read the bf16 qr to quantize it. The aiter kernel
|
||||
computes both RMSNorms (fp32 accumulate) and the fp8 group quant
|
||||
in a single pass, writing fp8 qr + group scales directly; both
|
||||
wq_b GEMMs (attention and indexer) then consume that pair and
|
||||
skip their own input quant. kv stays bf16: the fused insert
|
||||
kernel RoPE/quantizes it itself. Falls back to the shared path
|
||||
when the aiter linear path is not active.
|
||||
"""
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
if not (
|
||||
qr.dim() == 2
|
||||
and qr.shape[0] > 0
|
||||
and self.q_lora_rank % 128 == 0
|
||||
and self._wq_b_uses_aiter_block_scaled
|
||||
):
|
||||
return super()._split_qkv_and_norm(qr_kv)
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
|
||||
q=qr,
|
||||
q_weight=self.q_norm.weight.data,
|
||||
q_epsilon=self.eps,
|
||||
kv=kv,
|
||||
kv_weight=self.kv_norm.weight.data,
|
||||
kv_epsilon=self.eps,
|
||||
group_size=128,
|
||||
transpose_scale=False,
|
||||
)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
|
||||
z = rocm_inv_rope_einsum(
|
||||
|
||||
@@ -373,19 +373,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
qr_kv, kv_score, indexer_kv_score, indexer_weights = (
|
||||
self._run_parallel_input_projections(hidden_states)
|
||||
)
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
qr, kv = fused_q_kv_rmsnorm(
|
||||
qr,
|
||||
kv,
|
||||
self.q_norm.weight.data,
|
||||
self.kv_norm.weight.data,
|
||||
self.eps,
|
||||
)
|
||||
qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
|
||||
|
||||
self._prepare_and_attn_fn(
|
||||
hidden_states,
|
||||
qr,
|
||||
kv,
|
||||
qr_scale,
|
||||
kv_score,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
@@ -397,12 +391,32 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
|
||||
return self._o_proj(o, positions)
|
||||
|
||||
def _split_qkv_and_norm(
|
||||
self, qr_kv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
|
||||
"""Split the fused q-lora / kv projection and RMSNorm both halves.
|
||||
|
||||
``qr_scale`` is None on the shared path. The ROCm subclass returns
|
||||
a pre-quantized fp8 ``qr`` together with its per-1x128 fp32 scales,
|
||||
which the downstream wq_b projections consume directly.
|
||||
"""
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
qr, kv = fused_q_kv_rmsnorm(
|
||||
qr,
|
||||
kv,
|
||||
self.q_norm.weight.data,
|
||||
self.kv_norm.weight.data,
|
||||
self.eps,
|
||||
)
|
||||
return qr, None, kv
|
||||
|
||||
@eager_break_during_capture
|
||||
def _prepare_and_attn_eager(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
qr_scale: torch.Tensor | None,
|
||||
kv_score: torch.Tensor,
|
||||
indexer_kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
@@ -418,6 +432,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
hidden_states,
|
||||
qr,
|
||||
kv,
|
||||
qr_scale,
|
||||
kv_score,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
@@ -430,6 +445,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
qr_scale: torch.Tensor | None,
|
||||
kv_score: torch.Tensor,
|
||||
indexer_kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
@@ -446,7 +462,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
aux_streams = self.aux_stream_list
|
||||
|
||||
def project_query_and_cache_kv() -> torch.Tensor:
|
||||
q = self.wq_b(qr).view(-1, self.n_local_heads, self.head_dim)
|
||||
q = self._wq_b_proj(qr, qr_scale).view(
|
||||
-1, self.n_local_heads, self.head_dim
|
||||
)
|
||||
return self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
||||
|
||||
index_q: torch.Tensor | None = None
|
||||
@@ -468,6 +486,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
indexer_weights,
|
||||
positions,
|
||||
self.indexer_rotary_emb,
|
||||
qr_scale,
|
||||
),
|
||||
lambda: compressor(kv_score, positions, self.rotary_emb),
|
||||
],
|
||||
@@ -507,6 +526,24 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
qr_kv, _ = self.fused_wqa_wkv(hidden_states)
|
||||
return qr_kv
|
||||
|
||||
def _wq_b_proj(
|
||||
self, qr: torch.Tensor, qr_scale: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
"""Project the q-lora input to the query heads.
|
||||
|
||||
``qr_scale`` is only set by the ROCm fused path, where ``qr`` is
|
||||
already fp8-quantized with per-1x128 scales and must bypass the
|
||||
linear's internal re-quantization (apply_weights would quantize
|
||||
the fp8 input again).
|
||||
"""
|
||||
if qr_scale is None:
|
||||
return self.wq_b(qr)
|
||||
from vllm.models.deepseek_v4.amd.rocm import (
|
||||
apply_pre_quantized_block_scaled_mm,
|
||||
)
|
||||
|
||||
return apply_pre_quantized_block_scaled_mm(self.wq_b, qr, qr_scale)
|
||||
|
||||
def _run_parallel_input_projections(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[
|
||||
@@ -881,6 +918,7 @@ class DeepseekV4Indexer(nn.Module):
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rotary_emb: nn.Module,
|
||||
qr_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
|
||||
compressor = self.compressor
|
||||
|
||||
@@ -911,8 +949,7 @@ class DeepseekV4Indexer(nn.Module):
|
||||
return None, None, None
|
||||
|
||||
def wq_b_and_q_quant():
|
||||
# ReplicatedLinear returns (output, bias); bias is None.
|
||||
q, _ = self.wq_b(qr)
|
||||
q = self._wq_b_proj(qr, qr_scale)
|
||||
q = q.view(-1, self.n_head, self.head_dim)
|
||||
return fused_indexer_q_rope_quant(
|
||||
positions,
|
||||
@@ -938,3 +975,22 @@ class DeepseekV4Indexer(nn.Module):
|
||||
else:
|
||||
q, q_scale = q_quant, None
|
||||
return q, q_scale, weights
|
||||
|
||||
def _wq_b_proj(
|
||||
self, qr: torch.Tensor, qr_scale: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
"""Project the q-lora input to the indexer query heads.
|
||||
|
||||
Same bypass as ``DeepseekV4Attention._wq_b_proj``: on ROCm the fused
|
||||
norm path hands over pre-quantized fp8 ``qr`` with per-1x128 scales,
|
||||
so the linear's internal re-quantization must be skipped.
|
||||
"""
|
||||
if qr_scale is None:
|
||||
# ReplicatedLinear returns (output, bias); bias is None.
|
||||
q, _ = self.wq_b(qr)
|
||||
return q
|
||||
from vllm.models.deepseek_v4.amd.rocm import (
|
||||
apply_pre_quantized_block_scaled_mm,
|
||||
)
|
||||
|
||||
return apply_pre_quantized_block_scaled_mm(self.wq_b, qr, qr_scale)
|
||||
|
||||
Reference in New Issue
Block a user