[XPU]feat: add DeepSeek-V4 XPU attention decode path (#42953)
Signed-off-by: Ma Jian <jian1.ma@intel.com>
This commit is contained in:
@@ -100,6 +100,21 @@ class XPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
"weight_scale_inv" if hasattr(layer, "weight_scale_inv") else "weight_scale"
|
||||
)
|
||||
scale = getattr(layer, scale_attr)
|
||||
# Models with scale_fmt=ue8m0 (e.g. DeepSeek-V4) store weight scales
|
||||
# as float8_e8m0fnu. The oneDNN fp8_gemm kernel dispatches to its
|
||||
# "block quant" path only when NEITHER scale is e8m0:
|
||||
#
|
||||
# is_block_quant = (m1_sc != e8m0) && (m2_sc != e8m0) && ...
|
||||
#
|
||||
# Since activation scales are always float32 (use_ue8m0=False on XPU,
|
||||
# DeepGEMM requires Hopper/Blackwell), an e8m0 weight scale causes
|
||||
# is_block_quant=false and falls into the wrong per-channel path,
|
||||
# producing NaN. Converting e8m0→float32 here at load time (one-time,
|
||||
# negligible overhead for small scale tensors) ensures the kernel sees
|
||||
# matching dtypes and correctly enters the block-quant path with the
|
||||
# actual group_size derived from scale tensor shapes.
|
||||
if scale.dtype == torch.float8_e8m0fnu:
|
||||
scale = scale.to(torch.float32)
|
||||
replace_parameter(layer, scale_attr, scale.data.t().contiguous())
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
|
||||
@@ -147,6 +147,36 @@ class MHCPreOp(CustomOp):
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return self.forward_native(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
|
||||
# --8<-- [start:mhc_post]
|
||||
@CustomOp.register("mhc_post")
|
||||
@@ -215,6 +245,20 @@ class MHCPostOp(CustomOp):
|
||||
comb_res_mix,
|
||||
)
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.forward_native(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
)
|
||||
|
||||
|
||||
# --8<-- [start:hc_head]
|
||||
@CustomOp.register("hc_head")
|
||||
@@ -300,6 +344,36 @@ class HCHeadOp(CustomOp):
|
||||
def forward_native(self, *args, **kwargs):
|
||||
raise NotImplementedError("Native implementation of hc_head is not available")
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> torch.Tensor:
|
||||
hc_mult, hidden_size = hidden_states.shape[-2:]
|
||||
outer_shape = hidden_states.shape[:-2]
|
||||
hs_flat = hidden_states.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = hs_flat.shape[0]
|
||||
|
||||
out = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=hidden_states.device
|
||||
)
|
||||
torch.ops.vllm.hc_head_triton(
|
||||
hs_flat,
|
||||
hc_fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_norm_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
return out.view(*outer_shape, hidden_size)
|
||||
|
||||
|
||||
# --8<-- [start:mhc_fused_post_pre]
|
||||
@CustomOp.register("mhc_fused_post_pre")
|
||||
|
||||
Reference in New Issue
Block a user