Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
96 lines
3.6 KiB
Python
96 lines
3.6 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""DSV4.1 output projection with a small-batch SM100/SM103 fusion."""
|
|
|
|
import torch
|
|
from torch import nn
|
|
|
|
from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
|
|
FlashInferCutedslMxfp8LinearKernel,
|
|
)
|
|
from vllm.model_executor.layers.fusion.quant_activation import (
|
|
QuantizedActivation,
|
|
get_input_quant_key,
|
|
)
|
|
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
|
|
from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj
|
|
from vllm.platforms import current_platform
|
|
|
|
|
|
def _fused_wo_a_max_tokens(layer: nn.Module) -> int:
|
|
max_tokens = getattr(layer, "_fused_wo_a_token_limit", None)
|
|
if max_tokens is None:
|
|
# Additional token tiles only pay off for smaller grids.
|
|
ctas_per_tile = layer.n_local_heads * (layer.o_lora_rank // 128)
|
|
max_tokens = 96 if ctas_per_tile <= 128 else 32
|
|
layer._fused_wo_a_token_limit = max_tokens
|
|
return max_tokens
|
|
|
|
|
|
def _can_fuse_wo_a(layer: nn.Module) -> bool:
|
|
"""Whether the attention layer matches the fused WO-A kernel's layout."""
|
|
method = getattr(layer.wo_b, "scheme", layer.wo_b.quant_method)
|
|
return (
|
|
current_platform.is_device_capability_family(100)
|
|
# WO-A is FP8 with per-32 scales, and WO-B takes MXFP8 input directly.
|
|
and layer._einsum_recipe == (1, 1, 32)
|
|
and get_input_quant_key(layer.wo_b) == kMxfp8Dynamic
|
|
and isinstance(
|
|
getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel
|
|
)
|
|
# A group's heads form one portable (<= 8 CTA) cluster.
|
|
and layer.n_local_heads // layer.n_local_groups <= 8
|
|
and (layer.nope_head_dim, layer.rope_head_dim) == (448, 64)
|
|
and layer.o_lora_rank % 128 == 0
|
|
)
|
|
|
|
|
|
def register_dsv41_o_proj_warmup(layer: nn.Module) -> None:
|
|
"""Warm the fused WO-A token counts ``dsv41_o_proj`` can dispatch."""
|
|
# Draft models load outside the warmup registry; their layers share the
|
|
# target's attention shapes, so the target's registration covers them too.
|
|
if _can_fuse_wo_a(layer):
|
|
from .fused_wo_a import _FUSED_WO_A_KERNEL
|
|
|
|
_FUSED_WO_A_KERNEL.register_warmup(
|
|
max_tokens=_fused_wo_a_max_tokens(layer),
|
|
n_groups=layer.n_local_groups,
|
|
heads_per_group=layer.n_local_heads // layer.n_local_groups,
|
|
o_lora_rank=layer.o_lora_rank,
|
|
)
|
|
|
|
|
|
def dsv41_o_proj(
|
|
layer: nn.Module, attn_out: torch.Tensor, positions: torch.Tensor
|
|
) -> torch.Tensor:
|
|
"""``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100/SM103 batches."""
|
|
o = attn_out[:, : layer.n_local_heads, :]
|
|
cos_sin_cache = layer.rotary_emb.cos_sin_cache
|
|
if 1 <= o.shape[0] <= _fused_wo_a_max_tokens(layer) and _can_fuse_wo_a(layer):
|
|
from .fused_wo_a import _FUSED_WO_A_KERNEL
|
|
|
|
q, scales = _FUSED_WO_A_KERNEL(
|
|
x=o,
|
|
positions=positions,
|
|
rope=cos_sin_cache,
|
|
weight=layer.wo_a.weight,
|
|
weight_scale=layer.wo_a.weight_scale,
|
|
)
|
|
return layer._wo_b_proj(
|
|
QuantizedActivation(q, scales, o.dtype, q.shape, kMxfp8Dynamic)
|
|
)
|
|
return deep_gemm_fp8_o_proj(
|
|
o,
|
|
positions,
|
|
cos_sin_cache,
|
|
layer.wo_a,
|
|
layer._wo_b_proj,
|
|
n_groups=layer.n_local_groups,
|
|
heads_per_group=layer.n_local_heads // layer.n_local_groups,
|
|
nope_dim=layer.nope_head_dim,
|
|
rope_dim=layer.rope_head_dim,
|
|
o_lora_rank=layer.o_lora_rank,
|
|
einsum_recipe=layer._einsum_recipe,
|
|
tma_aligned_scales=layer._tma_aligned_scales,
|
|
)
|