[DeepSeek] Use torch.mm for bf16xbf16->fp32 gemm (#41300)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -2449,23 +2449,6 @@ def moe_wna16_gemm(
|
||||
)
|
||||
|
||||
|
||||
def router_gemm_bf16_fp32(input: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
||||
"""bf16 x bf16 -> fp32 GEMM via cuBLAS. weight shape: (N, K)."""
|
||||
return torch.ops._moe_C.router_gemm_bf16_fp32(input, weight)
|
||||
|
||||
|
||||
if hasattr(torch.ops, "_moe_C") and hasattr(torch.ops._moe_C, "router_gemm_bf16_fp32"):
|
||||
|
||||
@register_fake("_moe_C::router_gemm_bf16_fp32")
|
||||
def router_gemm_bf16_fp32_fake(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
input.shape[0], weight.shape[0], dtype=torch.float32, device=input.device
|
||||
)
|
||||
|
||||
|
||||
def dsv3_router_gemm(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
|
||||
@@ -17,7 +17,6 @@ from vllm.model_executor.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.layers.utils import cublas_gemm_bf16_bf16_fp32
|
||||
from vllm.utils.deep_gemm import fp8_einsum
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.ops.deepseek_v4_ops import (
|
||||
@@ -349,8 +348,10 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
compressor = self.compressor
|
||||
|
||||
def compressor_kv_score() -> torch.Tensor:
|
||||
return cublas_gemm_bf16_bf16_fp32(
|
||||
hidden_states, compressor.fused_wkv_wgate.weight
|
||||
return torch.mm(
|
||||
hidden_states,
|
||||
compressor.fused_wkv_wgate.weight.T,
|
||||
out_dtype=torch.float32,
|
||||
)
|
||||
|
||||
aux_fns[0] = compressor_kv_score
|
||||
@@ -364,8 +365,10 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
return weights
|
||||
|
||||
def indexer_compressor_kv_score() -> torch.Tensor:
|
||||
return cublas_gemm_bf16_bf16_fp32(
|
||||
hidden_states, indexer.compressor.fused_wkv_wgate.weight
|
||||
return torch.mm(
|
||||
hidden_states,
|
||||
indexer.compressor.fused_wkv_wgate.weight.T,
|
||||
out_dtype=torch.float32,
|
||||
)
|
||||
|
||||
aux_fns[1] = indexer_weights_proj
|
||||
|
||||
@@ -105,7 +105,7 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
# Tier 2: cuBLAS bf16→fp32
|
||||
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
|
||||
output = ops.router_gemm_bf16_fp32(x, self.weight)
|
||||
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
|
||||
return output, None
|
||||
|
||||
# Tier 3: F.linear (ReplicatedLinear)
|
||||
|
||||
@@ -304,13 +304,6 @@ def cpu_unquantized_gemm(
|
||||
return layer.cpu_linear(x, weight, bias)
|
||||
|
||||
|
||||
def cublas_gemm_bf16_bf16_fp32(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
):
|
||||
return ops.router_gemm_bf16_fp32(x, weight)
|
||||
|
||||
|
||||
def dispatch_unquantized_gemm() -> Callable[..., torch.Tensor]:
|
||||
if current_platform.is_rocm():
|
||||
return rocm_unquantized_gemm
|
||||
|
||||
Reference in New Issue
Block a user