[XPU] FP8 o_proj with fp8_bmm and load-time scale transpose (#48334)

Signed-off-by: Wu, Xiaochang <xiaochang.wu@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Xiaochang Wu
2026-07-20 16:32:03 +08:00
committed by GitHub
co-authored by Kunshang Ji mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f1f1259692
commit 823eaf667d
5 changed files with 124 additions and 15 deletions
@@ -257,7 +257,6 @@ def _fused_inv_rope_fp8_quant_kernel_impl(
)
grid = (tma_aligned_T, n_groups * heads_per_group)
use_gdc = current_platform.is_arch_support_pdl()
pdl_kwargs = {"launch_pdl": True} if use_gdc else {}
_fused_inv_rope_fp8_quant_per_head[grid](
o,
positions,
@@ -281,8 +280,8 @@ def _fused_inv_rope_fp8_quant_kernel_impl(
HALF_ROPE=half_rope,
TMA_ALIGNED_SCALES=tma_aligned_scales,
USE_GDC=use_gdc,
launch_pdl=use_gdc,
num_stages=1,
**pdl_kwargs,
num_warps=1,
)
return fp8_buf, scale_buf
-2
View File
@@ -8,7 +8,6 @@ import regex as re
import torch
import torch.nn as nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
get_ep_group,
@@ -978,7 +977,6 @@ class DeepseekV4DecoderLayer(nn.Module):
return x, residual, post_mix, res_mix
@support_torch_compile
class DeepseekV4Model(nn.Module, EagleModelMixin):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
+27 -9
View File
@@ -89,19 +89,37 @@ class DeepseekV4XPUAttention(DeepseekV4Attention):
return num_heads
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
# XPU uses BF16 reference wo_a path (same as ROCm).
from vllm.models.deepseek_v4.amd.rocm import rocm_inv_rope_einsum
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
fused_inv_rope_fp8_quant,
)
z = rocm_inv_rope_einsum(
self.rotary_emb,
o_fp8, o_scale = fused_inv_rope_fp8_quant(
o,
positions,
self.rope_head_dim,
self.n_local_groups,
self.o_lora_rank,
self.wo_a,
self.rotary_emb.cos_sin_cache,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
tma_aligned_scales=False,
)
return self.wo_b(z.flatten(1))
# Precomputed contiguous [G, K, N] weight and [G, K/bs, N/bs] scale.
wo_a_weight = self.wo_a.bmm_weight
wo_a_scale = self.wo_a.bmm_scale
# TODO: optimize fused_inv_rope_fp8_quant for xpu bmm to
# eliminate o_scale transpose + contiguous
z = torch.ops.vllm.xpu_fp8_bmm(
o_fp8.transpose(0, 1),
wo_a_weight,
torch.bfloat16,
o_scale.transpose(0, 1).contiguous(),
wo_a_scale,
None,
)
return self.wo_b(z.transpose(0, 1).flatten(1))
def forward_mqa(
self,