[DeepSeek] Use torch.mm for bf16xbf16->fp32 gemm (#41300)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-04-30 16:28:57 -07:00
committed by GitHub
parent 71725f6730
commit 9c61864bf8
8 changed files with 9 additions and 91 deletions
@@ -17,7 +17,6 @@ from vllm.model_executor.layers.linear import (
ReplicatedLinear,
)
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.layers.utils import cublas_gemm_bf16_bf16_fp32
from vllm.utils.deep_gemm import fp8_einsum
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.ops.deepseek_v4_ops import (
@@ -349,8 +348,10 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
compressor = self.compressor
def compressor_kv_score() -> torch.Tensor:
return cublas_gemm_bf16_bf16_fp32(
hidden_states, compressor.fused_wkv_wgate.weight
return torch.mm(
hidden_states,
compressor.fused_wkv_wgate.weight.T,
out_dtype=torch.float32,
)
aux_fns[0] = compressor_kv_score
@@ -364,8 +365,10 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
return weights
def indexer_compressor_kv_score() -> torch.Tensor:
return cublas_gemm_bf16_bf16_fp32(
hidden_states, indexer.compressor.fused_wkv_wgate.weight
return torch.mm(
hidden_states,
indexer.compressor.fused_wkv_wgate.weight.T,
out_dtype=torch.float32,
)
aux_fns[1] = indexer_weights_proj
@@ -105,7 +105,7 @@ class GateLinear(ReplicatedLinear):
# Tier 2: cuBLAS bf16→fp32
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
output = ops.router_gemm_bf16_fp32(x, self.weight)
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
return output, None
# Tier 3: F.linear (ReplicatedLinear)
-7
View File
@@ -304,13 +304,6 @@ def cpu_unquantized_gemm(
return layer.cpu_linear(x, weight, bias)
def cublas_gemm_bf16_bf16_fp32(
x: torch.Tensor,
weight: torch.Tensor,
):
return ops.router_gemm_bf16_fp32(x, weight)
def dispatch_unquantized_gemm() -> Callable[..., torch.Tensor]:
if current_platform.is_rocm():
return rocm_unquantized_gemm