[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -11,7 +11,6 @@ from vllm.model_executor.layers.attention.cross_attention import CrossAttention
|
||||
from vllm.model_executor.layers.attention.encoder_only_attention import (
|
||||
EncoderOnlyAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.attention.prefill_prefix_lm_attention import (
|
||||
PrefillPrefixLMAttention,
|
||||
@@ -35,7 +34,6 @@ __all__ = [
|
||||
"ChunkedLocalAttention",
|
||||
"CrossAttention",
|
||||
"EncoderOnlyAttention",
|
||||
"MLAAttention",
|
||||
"MMEncoderAttention",
|
||||
"PrefillPrefixLMAttention",
|
||||
"RSWAAttention",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,25 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.hpc.gated_mla import (
|
||||
hpc_gated_mla_gemm,
|
||||
hpc_gated_mla_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.hpc.hpc_ihc import (
|
||||
HpcIHCHead,
|
||||
HpcIHCPost,
|
||||
HpcIHCPre,
|
||||
)
|
||||
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
|
||||
from vllm.model_executor.layers.hpc.rope_norm import HpcRopeNorm, QkNormPolicy
|
||||
|
||||
__all__ = [
|
||||
"HpcIHCHead",
|
||||
"HpcIHCPost",
|
||||
"HpcIHCPre",
|
||||
"HpcModule",
|
||||
"HpcRopeNorm",
|
||||
"QkNormPolicy",
|
||||
"hpc_gated_mla_gemm",
|
||||
"hpc_gated_mla_supported",
|
||||
]
|
||||
@@ -1,114 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Enablement gate for the HPC fused gated-MLA GEMM.
|
||||
|
||||
``hpc.gated_mla_gemm`` fuses the MLA output gating -- the gate projection, the
|
||||
sigmoid and the elementwise product -- into a single kernel launch, replacing
|
||||
the eager ``attn_out * sigmoid(hidden_states @ weight.T)`` sequence.
|
||||
|
||||
Kernel constraints:
|
||||
- Requires VLLM_ENABLE_HPC_OPS=1
|
||||
- Requires the hpc package (.so) built for the current arch
|
||||
- Only sm100 / sm103 (compute capability 100, 103)
|
||||
- All three operands must be bfloat16 and contiguous
|
||||
- The gate width must be a multiple of 256
|
||||
- Only elementwise gating (headwise has no fused form)
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.hpc import has_hpc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# hpc.gated_mla_gemm fuses `attn_out * sigmoid(x @ weight.T)` into one launch.
|
||||
# It is built for sm100/sm103 and needs all three operands bf16 + contiguous,
|
||||
# with the gate width a multiple of 256.
|
||||
_HPC_GATED_GEMM_N_ALIGN = 256
|
||||
_HPC_GATED_GEMM_CAPABILITIES: frozenset[int] = frozenset({100, 103})
|
||||
|
||||
|
||||
def hpc_gated_mla_supported(
|
||||
gating_type: str | None, attn_output_gate: torch.nn.Module | None
|
||||
) -> bool:
|
||||
"""Gate for running the MLA output gating through hpc.gated_mla_gemm."""
|
||||
if not envs.VLLM_ENABLE_HPC_OPS:
|
||||
return False
|
||||
|
||||
if attn_output_gate is None:
|
||||
return False
|
||||
|
||||
if not has_hpc():
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: 'hpc' package is not installed. "
|
||||
"Please install the HPC library (.so) to enable the fused kernel."
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
logger.warning_once("HPC gated MLA disabled: only CUDA is supported.")
|
||||
return False
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.to_int() not in _HPC_GATED_GEMM_CAPABILITIES:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: compute capability %s not in %s.",
|
||||
capability,
|
||||
_HPC_GATED_GEMM_CAPABILITIES,
|
||||
)
|
||||
return False
|
||||
|
||||
# Headwise gating broadcasts one scalar per head over v_head_dim; the
|
||||
# kernel only implements the elementwise product.
|
||||
if gating_type != "elementwise":
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gating_type '%s' has no fused form.",
|
||||
gating_type,
|
||||
)
|
||||
return False
|
||||
|
||||
weight = getattr(attn_output_gate, "weight", None)
|
||||
if weight is None or weight.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gate weight dtype is %s, the kernel needs "
|
||||
"bfloat16 (keep the layer out of quantization).",
|
||||
None if weight is None else weight.dtype,
|
||||
)
|
||||
return False
|
||||
|
||||
if weight.shape[0] % _HPC_GATED_GEMM_N_ALIGN != 0:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gate width %d is not a multiple of %d.",
|
||||
weight.shape[0],
|
||||
_HPC_GATED_GEMM_N_ALIGN,
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info_once("HPC gated MLA enabled by set VLLM_ENABLE_HPC_OPS.")
|
||||
return True
|
||||
|
||||
|
||||
def hpc_gated_mla_gemm(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
atten_output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``atten_output * sigmoid(x @ weight.T)``.
|
||||
|
||||
Args:
|
||||
x: ``[m, k]`` bfloat16, contiguous.
|
||||
weight: ``[n, k]`` bfloat16, contiguous and row-major (the kernel
|
||||
transposes it). ``n`` must be a multiple of 256.
|
||||
atten_output: ``[m, n]`` bfloat16, contiguous.
|
||||
|
||||
Returns:
|
||||
``[m, n]`` bfloat16.
|
||||
|
||||
"""
|
||||
from hpc.gemm import gated_mla_gemm
|
||||
|
||||
return gated_mla_gemm(x, weight, atten_output)
|
||||
@@ -1,237 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HPC fused iHC (independent Hyper-Connections) kernels for HY V4.
|
||||
|
||||
Replaces the eager HYV4HCPreLayer / HYV4HCPostLayer / HYV4HCHeadLayer bodies
|
||||
with single-kernel HPC implementations:
|
||||
|
||||
pre : rms square sum + w projection + sigmoid gates + hc-dim weighted sum
|
||||
post : y[n, i, :] = H_post[n, i] * x[n, :] + residual[n, i, :]
|
||||
head : same as pre but hc_mult projection rows and no H_post output
|
||||
|
||||
The eager path issues 20 / 5 / 15 kernels for pre / post / head; each HPC op is
|
||||
a single kernel.
|
||||
|
||||
Constraints:
|
||||
- Requires VLLM_ENABLE_HPC_OPS=1
|
||||
- Requires the hpc package (.so) built for the current arch
|
||||
- Only sm100 / sm103 (compute capability 100, 103)
|
||||
- Only hc_mult == 4 and hidden_size in {4096, 6144}
|
||||
- x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint
|
||||
keeps them out of fp8 quantization via modules_to_not_convert)
|
||||
|
||||
The fused modules own no parameters: their forwards read the weights of the
|
||||
eager layer they replace directly via the captured owner reference.
|
||||
|
||||
NOTE: The reference implementation also offers a cross-layer post+pre fusion
|
||||
(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre.
|
||||
It requires reworking the decoder-layer forward scheduling and is not ported
|
||||
here. TODO: add it once the decoder dataflow is restructured for it.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
|
||||
from vllm.utils.hpc import has_hpc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Shapes the HPC kernels are instantiated for. Other shapes launch nothing,
|
||||
# so gate on this rather than relying on a silent no-op.
|
||||
_SUPPORTED_HIDDEN_SIZES: frozenset[int] = frozenset({4096, 6144})
|
||||
_SUPPORTED_HC_MULTS: frozenset[int] = frozenset({4})
|
||||
_SUPPORTED_CAPABILITIES: frozenset[int] = frozenset({100, 103})
|
||||
|
||||
|
||||
def _ihc_supported(hc_mult: int, hidden_size: int) -> bool:
|
||||
"""Shared gate for all three iHC ops."""
|
||||
if not envs.VLLM_ENABLE_HPC_OPS:
|
||||
return False
|
||||
|
||||
if not has_hpc():
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: 'hpc' package is not installed. "
|
||||
"Please install the HPC library (.so) to enable fused kernels."
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
logger.warning_once("HPC iHC disabled: only CUDA is supported.")
|
||||
return False
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.to_int() not in _SUPPORTED_CAPABILITIES:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: compute capability %s not in %s.",
|
||||
capability,
|
||||
_SUPPORTED_CAPABILITIES,
|
||||
)
|
||||
return False
|
||||
|
||||
if hc_mult not in _SUPPORTED_HC_MULTS:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: hc_mult=%d not in %s.", hc_mult, _SUPPORTED_HC_MULTS
|
||||
)
|
||||
return False
|
||||
|
||||
if hidden_size not in _SUPPORTED_HIDDEN_SIZES:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: hidden_size=%d not in %s.",
|
||||
hidden_size,
|
||||
_SUPPORTED_HIDDEN_SIZES,
|
||||
)
|
||||
return False
|
||||
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
# Batch-invariant mode relies on torch's deterministic reductions; the
|
||||
# HPC kernel reduces in a different order, so keep the two regimes
|
||||
# apart.
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: not supported under VLLM_BATCH_INVARIANT."
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info_once("HPC iHC enabled by set VLLM_ENABLE_HPC_OPS.")
|
||||
return True
|
||||
|
||||
|
||||
class HpcIHCPre(HpcModule):
|
||||
"""Fused iHC pre block.
|
||||
|
||||
Computes, in one kernel:
|
||||
x_flat = x.flatten(1)
|
||||
r = rsqrt(x_flat.square().mean(-1) + rms_norm_eps)
|
||||
mixes = (x_flat @ w.T) * r
|
||||
H_pre = sigmoid(mixes[:, :hc] * hc_scale[0] + hc_base[:hc]) + hc_eps
|
||||
H_post = magnitude * sigmoid(
|
||||
mixes[:, hc:] * hc_scale[1] + hc_base[hc:]) + hc_eps
|
||||
y = sum_i H_pre[:, i] * x[:, i, :]
|
||||
|
||||
Args:
|
||||
hc_mult: HC expand ratio.
|
||||
hidden_size: Model hidden dimension.
|
||||
magnitude: H_post multiplier (config.hc_magnitude).
|
||||
hc_eps: Epsilon added to both gates (config.hc_eps).
|
||||
norm_eps: Epsilon inside the rsqrt (config.rms_norm_eps).
|
||||
fallback_op: The eager HYV4HCPreLayer to source weights from.
|
||||
norm_owner: Optional RMSNorm that immediately follows this pre block.
|
||||
When given, its weight/eps are folded into the kernel so a single
|
||||
launch covers pre + RMSNorm (the caller then skips the separate
|
||||
layernorm). None keeps the original pre-only behaviour.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
magnitude: float,
|
||||
hc_eps: float,
|
||||
norm_eps: float,
|
||||
fallback_op: torch.nn.Module,
|
||||
norm_owner: torch.nn.Module | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
self.magnitude = magnitude
|
||||
self.hc_eps = hc_eps
|
||||
self.norm_eps = norm_eps
|
||||
# Stash the owner outside nn.Module's attribute machinery: callers pass
|
||||
# their own `self` here, and registering that as a submodule would make
|
||||
# the module tree cyclic (state_dict() then hits RecursionError).
|
||||
object.__setattr__(self, "_fallback_op", fallback_op)
|
||||
# Same anti-cycle idiom for the fused RMSNorm owner (a sibling module).
|
||||
object.__setattr__(self, "_norm_owner", norm_owner)
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import hpc
|
||||
|
||||
owner = self._fallback_op
|
||||
norm_owner = self._norm_owner
|
||||
return hpc.fuse_ihc_pre(
|
||||
x,
|
||||
owner.hc_fn.weight,
|
||||
owner.hc_scale,
|
||||
owner.hc_base,
|
||||
self.norm_eps,
|
||||
self.hc_eps,
|
||||
self.magnitude,
|
||||
norm_owner.weight if norm_owner is not None else None,
|
||||
norm_owner.variance_epsilon if norm_owner is not None else 0.0,
|
||||
)
|
||||
|
||||
|
||||
class HpcIHCPost(HpcModule):
|
||||
"""Fused iHC post block: H_post gating plus multi-channel residual add.
|
||||
|
||||
Unlike mHC there is no comb matrix, so each output channel only needs its
|
||||
own residual channel and the whole thing is one fused multiply-add per
|
||||
element.
|
||||
"""
|
||||
|
||||
def __init__(self, hc_mult: int, hidden_size: int) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual: torch.Tensor, H_post: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
import hpc
|
||||
|
||||
return hpc.fuse_ihc_post(x, residual, H_post)
|
||||
|
||||
|
||||
class HpcIHCHead(HpcModule):
|
||||
"""Fused iHC head block: merge the hc channels back into one hidden state.
|
||||
|
||||
Same structure as HpcIHCPre but the projection emits only hc_mult gate
|
||||
logits and there is no H_post output. Called once per forward, after the
|
||||
last decoder layer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
hc_eps: float,
|
||||
norm_eps: float,
|
||||
fallback_op: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
self.hc_eps = hc_eps
|
||||
self.norm_eps = norm_eps
|
||||
# See HpcIHCPre: keep the owner out of the module tree.
|
||||
object.__setattr__(self, "_fallback_op", fallback_op)
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
import hpc
|
||||
|
||||
owner = self._fallback_op
|
||||
return hpc.fuse_ihc_head(
|
||||
x,
|
||||
owner.hc_head_fn.weight,
|
||||
owner.hc_head_scale,
|
||||
owner.hc_head_base,
|
||||
self.norm_eps,
|
||||
self.hc_eps,
|
||||
)
|
||||
@@ -1,15 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
class HpcModule(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
@classmethod
|
||||
def support(cls, *args, **kwargs):
|
||||
return True
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
pass
|
||||
@@ -1,575 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant).
|
||||
|
||||
Decoupled from HpcAttentionImpl; extra params are passed via layer attrs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from enum import IntEnum
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.hpc_attn import HpcAttnMetadata
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_hpc_rope_norm_instances: dict[str, HpcRopeNorm] = {}
|
||||
|
||||
|
||||
class QkNormPolicy(IntEnum):
|
||||
"""Order of QK-RMSNorm relative to RoPE in the fused HPC rope_norm kernel.
|
||||
|
||||
The values are part of the HPC kernel ABI (passed through as ints), so they
|
||||
must stay in sync with the kernel's expectations.
|
||||
"""
|
||||
|
||||
# No QK-Norm: apply RoPE only.
|
||||
NONE = 0
|
||||
# Apply RoPE first, then QK-RMSNorm.
|
||||
ROPE_THEN_NORM = 1
|
||||
# Apply QK-RMSNorm first, then RoPE (e.g. HunYuan V3).
|
||||
NORM_THEN_ROPE = 2
|
||||
|
||||
|
||||
def hpc_rope_norm_forward(
|
||||
qkv: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Top-level custom op: RoPE + QK-Norm + KV-Cache-Write + FP8 Q quant.
|
||||
|
||||
Fully opaque to torch.compile (dynamo).
|
||||
"""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata: Any = forward_context.attn_metadata
|
||||
if isinstance(attn_metadata, dict):
|
||||
attn_metadata = attn_metadata[layer_name]
|
||||
|
||||
if attn_metadata is None:
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
attn_layer = forward_context.no_compile_layers[layer_name]
|
||||
# bind_kv_cache stores the per-layer KV cache as a single tensor
|
||||
# (num_blocks, num_kv_heads, block_size, 2 * head_size), so use it directly.
|
||||
kv_cache = attn_layer.kv_cache
|
||||
|
||||
if kv_cache.numel() == 0:
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
assert kv_cache.dim() == 4, (
|
||||
f"Expected kv_cache to have 4 dims, got {tuple(kv_cache.shape)}"
|
||||
)
|
||||
|
||||
rope_norm = _hpc_rope_norm_instances[layer_name]
|
||||
rope_norm._forward_impl(qkv, kv_cache, attn_metadata, attn_layer, output)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="hpc_rope_norm_forward",
|
||||
op_func=hpc_rope_norm_forward,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
|
||||
|
||||
@CustomOp.register("hpc_rope_norm")
|
||||
class HpcRopeNorm(CustomOp, HpcModule):
|
||||
"""HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant).
|
||||
|
||||
Registered as a sub-module in model layers (e.g. HunYuanAttention).
|
||||
The QK-Norm weights are read directly from the fallback norm modules
|
||||
(built in float32 when HPC is active), eliminating the derived copy.
|
||||
|
||||
forward() is dispatched by CustomOp framework:
|
||||
- In compiled mode: forward_cuda() calls torch.ops.vllm.hpc_rope_norm_forward
|
||||
as a splitting point — internal Python control flow is opaque
|
||||
to torch.compile (its kernel launches can still be CUDA-graph captured).
|
||||
- In eager/native mode: forward_native() falls back to forward_cuda().
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
use_qk_norm: bool,
|
||||
fallback_qnorm: torch.nn.Module | None,
|
||||
fallback_knorm: torch.nn.Module | None,
|
||||
kv_cache_dtype: str,
|
||||
layer_name: str,
|
||||
qk_norm_policy: QkNormPolicy = QkNormPolicy.ROPE_THEN_NORM,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if importlib.util.find_spec("hpc") is None:
|
||||
raise ImportError(
|
||||
"HPCRopeNorm requires the hpc module to be installed. "
|
||||
"Please install it from https://github.com/Tencent/hpc-ops"
|
||||
)
|
||||
|
||||
self.num_heads = num_heads
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.head_dim = head_dim
|
||||
|
||||
self.q_size = num_heads * head_dim
|
||||
self.kv_size = num_kv_heads * head_dim
|
||||
|
||||
# Register as a non-persistent buffer so it participates in sleep
|
||||
# level-2 save/restore (CuMemAllocator) but is excluded from the
|
||||
# checkpoint state_dict.
|
||||
self.register_buffer("cos_sin_cache", cos_sin_cache.float(), persistent=False)
|
||||
|
||||
self.fallback_qnorm = fallback_qnorm
|
||||
self.fallback_knorm = fallback_knorm
|
||||
|
||||
self.head_per_group = num_heads // num_kv_heads
|
||||
|
||||
self.use_fp8 = "fp8" in kv_cache_dtype
|
||||
# The RMSNorm/RoPE ordering is model dependent (e.g. HunYuan V3 applies
|
||||
# QK-Norm before RoPE -> NORM_THEN_ROPE), so it is supplied by the
|
||||
# caller. When QK-Norm is disabled the policy is forced to NONE.
|
||||
self.qk_norm_policy = qk_norm_policy if use_qk_norm else QkNormPolicy.NONE
|
||||
if self.qk_norm_policy != QkNormPolicy.NONE and (
|
||||
fallback_qnorm is None or fallback_knorm is None
|
||||
):
|
||||
raise ValueError(
|
||||
"QK-Norm is enabled but fallback_qnorm/fallback_knorm were not "
|
||||
"supplied; the fused kernel reads their weights directly."
|
||||
)
|
||||
|
||||
# Register layer_name + add self to the global instance registry so the
|
||||
# module-level custom op (hpc_rope_norm_forward) can route back here.
|
||||
self.layer_name: str | None = None
|
||||
self.register_layer_name(layer_name)
|
||||
|
||||
import hpc
|
||||
|
||||
if self.use_fp8:
|
||||
self._quant_type = (
|
||||
hpc.QuantType.QPERTOKEN_PERHEAD_KPERTENSOR_VPERTENSOR.value
|
||||
)
|
||||
else:
|
||||
self._quant_type = None
|
||||
|
||||
@classmethod
|
||||
def support(
|
||||
cls,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
kv_cache_dtype: str,
|
||||
) -> bool:
|
||||
"""Check whether HpcRopeNorm is supported for the given config."""
|
||||
# HpcRopeNorm is only enabled together with the HPC attention backend.
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if (
|
||||
vllm_config is None
|
||||
or vllm_config.attention_config.backend != AttentionBackendEnum.HPC_ATTN
|
||||
):
|
||||
return False
|
||||
|
||||
if kv_cache_dtype not in ("fp8_e4m3", "auto", "bfloat16"):
|
||||
logger.warning_once(
|
||||
f"hpc rope_norm not support kv_cache_dtype:{kv_cache_dtype}, "
|
||||
"only support fp8_e4m3, auto, bfloat16"
|
||||
)
|
||||
return False
|
||||
|
||||
if head_dim not in (128,):
|
||||
logger.warning_once("hpc rope_norm only support head_dim == 128.")
|
||||
return False
|
||||
|
||||
head_per_group = num_heads // num_kv_heads
|
||||
if head_per_group not in (4, 8):
|
||||
logger.warning_once("hpc rope_norm only support head_per_group in [4, 8].")
|
||||
return False
|
||||
|
||||
logger.info_once("enable hpc rope_norm")
|
||||
return True
|
||||
|
||||
def register_layer_name(self, layer_name: str) -> None:
|
||||
"""Register layer_name and add self to the global registry.
|
||||
|
||||
The global registry is needed because the bottom-level torch op
|
||||
(hpc_rope_norm_forward) is a module-level function and needs to
|
||||
route back to the correct instance via layer_name.
|
||||
"""
|
||||
self.layer_name = layer_name
|
||||
_hpc_rope_norm_instances[layer_name] = self
|
||||
logger.debug(
|
||||
"[rope_norm] registered HpcRopeNorm for layer: %s",
|
||||
layer_name,
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> torch.Tensor:
|
||||
"""Native fallback path: delegates to forward_cuda().
|
||||
|
||||
For now, the default native path will use CUDA backend path.
|
||||
Other platforms may override via OOT registration.
|
||||
"""
|
||||
return self.forward_cuda(qkv, layer_name)
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> torch.Tensor:
|
||||
"""CUDA path: invoke the torch custom op as a compile splitting point."""
|
||||
num_tokens = qkv.shape[0]
|
||||
output = torch.empty(
|
||||
(num_tokens, self.num_heads, self.head_dim),
|
||||
dtype=torch.float8_e4m3fn if self.use_fp8 else qkv.dtype,
|
||||
device=qkv.device,
|
||||
)
|
||||
|
||||
torch.ops.vllm.hpc_rope_norm_forward(qkv, output, layer_name)
|
||||
return output
|
||||
|
||||
def _kv_write_scratch(self, cache: torch.Tensor) -> torch.Tensor:
|
||||
"""Throwaway dense K/V pages for the stride-aware fallback path.
|
||||
|
||||
``hpc.rope_norm_store_kv[_fp8]`` always writes the paged cache itself
|
||||
(rotated K/V *and* zero padding for the tail of every request's last
|
||||
block), addressing it densely. When the real pages are not dense those
|
||||
writes would land on unrelated blocks, so they are redirected here by
|
||||
pairing this scratch cache with an all-zero block table; the real data
|
||||
is taken out through ``out_k``/``out_v`` instead.
|
||||
|
||||
Returns a (2, 1, block_size, num_kv_heads, head_dim) tensor: index 0 is
|
||||
the key scratch page, index 1 the value scratch page.
|
||||
"""
|
||||
block_size = cache.shape[1]
|
||||
key = (cache.dtype, block_size, cache.device)
|
||||
if getattr(self, "_kv_scratch_key", None) != key:
|
||||
self._kv_scratch = torch.zeros(
|
||||
2,
|
||||
1,
|
||||
block_size,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
dtype=cache.dtype,
|
||||
device=cache.device,
|
||||
)
|
||||
self._kv_scratch_key = key
|
||||
return self._kv_scratch
|
||||
|
||||
@staticmethod
|
||||
def _zero_pad_last_blocks(
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
) -> None:
|
||||
"""Zero the unused tail slots of each request's last block.
|
||||
|
||||
The HPC attention kernels read whole blocks, so the slots past
|
||||
``seq_len`` must be zero. The fused op normally does this itself; on the
|
||||
fallback path its (densely addressed) padding writes are discarded, so
|
||||
the padding is reapplied here through the real, strided cache views.
|
||||
|
||||
Shapes are static and no value is inspected on the host, so this stays
|
||||
CUDA-graph capturable. It is also self-contained: each request's last
|
||||
block is read, masked and written back, so slots that must be preserved
|
||||
(and requests padded in for graph capture, whose ``seq_len`` is 0) keep
|
||||
their previous contents instead of being redirected somewhere.
|
||||
"""
|
||||
block_size = key_cache.shape[1]
|
||||
seq_lens = seq_lens.long()
|
||||
# seq_len == 0 only happens for graph-capture padding; clamping keeps
|
||||
# the gather in bounds and the all-False mask below makes the write a
|
||||
# no-op for those rows.
|
||||
last_block_idx = (seq_lens - 1).clamp_min(0) // block_size
|
||||
last_block_idx = last_block_idx.clamp_max(block_table.shape[1] - 1)
|
||||
last_block = block_table.gather(1, last_block_idx.unsqueeze(1)).squeeze(1)
|
||||
last_block = last_block.long()
|
||||
|
||||
remainder = seq_lens % block_size
|
||||
# A block that ends exactly on the boundary has no tail to clear.
|
||||
remainder = torch.where(remainder == 0, block_size, remainder)
|
||||
|
||||
offsets = torch.arange(block_size, device=key_cache.device)
|
||||
is_tail = (offsets.unsqueeze(0) >= remainder.unsqueeze(1)) & (
|
||||
seq_lens > 0
|
||||
).unsqueeze(1)
|
||||
is_tail = is_tail[..., None, None]
|
||||
|
||||
for cache in (key_cache, value_cache):
|
||||
view = cache
|
||||
if view.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
|
||||
view = view.view(torch.uint8)
|
||||
blocks = view.index_select(0, last_block)
|
||||
blocks = torch.where(is_tail, blocks.new_zeros(()), blocks)
|
||||
view.index_copy_(0, last_block, blocks)
|
||||
|
||||
@staticmethod
|
||||
def _scatter_kv_cache(
|
||||
cache: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
"""Write dense per-token K (or V) into a paged cache view.
|
||||
|
||||
Used on the stride-aware fallback path (see ``_forward_impl``).
|
||||
``cache`` is a (num_blocks, block_size, num_kv_heads, head_dim) view
|
||||
with arbitrary block/token/head strides; ``src`` is the dense
|
||||
(num_tokens, num_kv_heads, head_dim) buffer produced by the fused HPC
|
||||
kernel. FP8 payloads are moved as raw bytes because ``index_copy_``
|
||||
has no FP8 kernel.
|
||||
|
||||
Padded tokens carry ``PAD_SLOT_ID`` (-1); clamping sends them to the
|
||||
reserved null block instead of tripping ``index_copy_``'s bounds check.
|
||||
This keeps the op CUDA-graph safe (static shape, no host sync), unlike
|
||||
masking the padded rows out.
|
||||
"""
|
||||
# (num_blocks, block_size, ...) -> (num_slots, num_kv_heads, head_dim).
|
||||
# Valid as a view for any layout whose block stride is a whole number
|
||||
# of token strides, which holds for every layout the backend accepts.
|
||||
flat = cache.flatten(0, 1)
|
||||
if flat.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
|
||||
flat = flat.view(torch.uint8)
|
||||
src = src.view(torch.uint8)
|
||||
flat.index_copy_(0, slot_mapping.clamp_min(NULL_BLOCK_ID), src)
|
||||
|
||||
def _forward_impl(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
attn_metadata: HpcAttnMetadata,
|
||||
attn_layer: torch.nn.Module,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
"""Actual forward logic called by the custom op.
|
||||
|
||||
Writes processed q into *output* and attaches extra params
|
||||
(e.g. FP8 scales) to *attn_layer* as attributes.
|
||||
"""
|
||||
import hpc
|
||||
|
||||
# KV cache for the FP8 path is stored as uint8; view it as fp8 so the
|
||||
# rope_norm_store_kv_fp8 kernel can write quantized K/V in-place.
|
||||
# Must happen *before* deriving the K/V views, so that they (and any
|
||||
# buffer whose dtype is taken from them) carry the FP8 dtype.
|
||||
if self.use_fp8:
|
||||
kv_cache = kv_cache.view(torch.float8_e4m3fn)
|
||||
|
||||
# (B, H, N, 2*hs) -> two (B, N, H, hs) K/V views, as in HpcAttentionImpl.
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_dim, dim=-1)
|
||||
|
||||
# hpc.rope_norm_store_kv[_fp8] addresses its K/V caches as *dense*
|
||||
# NHD tensors and ignores their strides, so it can only write the cache
|
||||
# in place when both views are contiguous. Packed layouts (the backend's
|
||||
# LBNHC pages interleave K and V inside the state-content dim, so each
|
||||
# split view has a token stride of 2 * head_dim) must instead take the
|
||||
# rotated K/V out through out_k/out_v and be scattered separately.
|
||||
fused_kv_write = key_cache.is_contiguous() and value_cache.is_contiguous()
|
||||
if not fused_kv_write:
|
||||
logger.warning_once(
|
||||
"HpcRopeNorm: KV cache pages interleave K and V, which "
|
||||
"hpc.rope_norm_store_kv[_fp8] cannot address, so K/V are "
|
||||
"written by a separate scatter. Update hpc-ops to a build with "
|
||||
"strided KV cache support to re-enable the fully fused write."
|
||||
)
|
||||
slot_mapping = attn_metadata.slot_mapping
|
||||
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
num_prefill_reqs = attn_metadata.num_prefills
|
||||
num_decode_reqs = attn_metadata.num_decodes
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
|
||||
qkv = qkv[:num_actual_tokens]
|
||||
|
||||
num_prefill_tokens = num_actual_tokens - num_decode_tokens
|
||||
|
||||
# Per-tensor K/V scales (shape [1]) used by the FP8 kernel.
|
||||
k_scale = attn_layer._k_scale.reshape(1)
|
||||
v_scale = attn_layer._v_scale.reshape(1)
|
||||
|
||||
if self.qk_norm_policy != QkNormPolicy.NONE:
|
||||
assert self.fallback_qnorm is not None
|
||||
assert self.fallback_knorm is not None
|
||||
q_norm_weight = self.fallback_qnorm.weight
|
||||
k_norm_weight = self.fallback_knorm.weight
|
||||
else:
|
||||
q_norm_weight = None
|
||||
k_norm_weight = None
|
||||
|
||||
# --- Prefill ---
|
||||
if num_prefill_reqs > 0:
|
||||
seq_lens_prefill = attn_metadata.seq_lens[num_decode_reqs:]
|
||||
cu_seqlens_prefill = attn_metadata.qo_indptr
|
||||
max_seqlens = attn_metadata.max_query_len
|
||||
block_table_prefill = attn_metadata.block_table_tensor[num_decode_reqs:]
|
||||
qkv_prefill = qkv[num_decode_tokens:]
|
||||
out_q_prefill = output[
|
||||
num_decode_tokens : num_decode_tokens + num_prefill_tokens
|
||||
]
|
||||
|
||||
kv_dtype = key_cache.dtype
|
||||
if fused_kv_write:
|
||||
out_k = out_v = None
|
||||
kcache_arg, vcache_arg = key_cache, value_cache
|
||||
kv_indices_arg = block_table_prefill
|
||||
else:
|
||||
out_k = torch.empty(
|
||||
num_prefill_tokens,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
dtype=kv_dtype,
|
||||
device=qkv.device,
|
||||
)
|
||||
out_v = torch.empty_like(out_k)
|
||||
# Send the kernel's own cache writes to scratch (all block ids
|
||||
# remapped to the single scratch page).
|
||||
scratch = self._kv_write_scratch(key_cache)
|
||||
kcache_arg, vcache_arg = scratch[0], scratch[1]
|
||||
kv_indices_arg = torch.zeros_like(block_table_prefill)
|
||||
|
||||
if self.use_fp8:
|
||||
_, q_scale, split_k_flag = hpc.rope_norm_store_kv_fp8(
|
||||
key_cache=kcache_arg,
|
||||
value_cache=vcache_arg,
|
||||
qkv=qkv_prefill,
|
||||
cos_sin=self.cos_sin_cache,
|
||||
num_seqlen_per_req=seq_lens_prefill,
|
||||
q_index=cu_seqlens_prefill,
|
||||
kvcache_indices=kv_indices_arg,
|
||||
is_prefill=True,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
quant_policy=self._quant_type,
|
||||
max_seqlens=max_seqlens,
|
||||
q_norm_weight=q_norm_weight,
|
||||
k_norm_weight=k_norm_weight,
|
||||
qk_norm_policy=self.qk_norm_policy,
|
||||
out_q=out_q_prefill,
|
||||
out_k=out_k,
|
||||
out_v=out_v,
|
||||
)
|
||||
attn_metadata.hpc_prefill_q_scale = q_scale
|
||||
else:
|
||||
hpc.rope_norm_store_kv(
|
||||
kcache_arg,
|
||||
vcache_arg,
|
||||
qkv_prefill,
|
||||
self.cos_sin_cache,
|
||||
seq_lens_prefill,
|
||||
cu_seqlens_prefill,
|
||||
kv_indices_arg,
|
||||
True, # is_prefill
|
||||
q_norm_weight=q_norm_weight,
|
||||
k_norm_weight=k_norm_weight,
|
||||
out_q=out_q_prefill,
|
||||
out_k=out_k,
|
||||
out_v=out_v,
|
||||
qk_norm_policy=self.qk_norm_policy,
|
||||
)
|
||||
|
||||
if out_k is not None:
|
||||
self._zero_pad_last_blocks(
|
||||
key_cache, value_cache, seq_lens_prefill, block_table_prefill
|
||||
)
|
||||
prefill_slots = slot_mapping[num_decode_tokens:num_actual_tokens]
|
||||
self._scatter_kv_cache(key_cache, out_k, prefill_slots)
|
||||
self._scatter_kv_cache(value_cache, out_v, prefill_slots)
|
||||
|
||||
# --- Decode ---
|
||||
if num_decode_reqs > 0:
|
||||
num_seq_kvcache = attn_metadata.seq_lens[:num_decode_reqs]
|
||||
block_table_decode = attn_metadata.block_table_tensor[:num_decode_reqs]
|
||||
qkv_decode = qkv[:num_decode_tokens]
|
||||
# Single-token decode: q_index is the per-request prefix sum
|
||||
# [0, 1, ..., num_decode_reqs].
|
||||
decode_query_len = attn_metadata.decode_query_len
|
||||
out_q_decode = output[:num_decode_tokens]
|
||||
|
||||
if fused_kv_write:
|
||||
out_k = out_v = None
|
||||
kcache_arg, vcache_arg = key_cache, value_cache
|
||||
kv_indices_arg = block_table_decode
|
||||
else:
|
||||
out_k = torch.empty(
|
||||
num_decode_tokens,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
dtype=key_cache.dtype,
|
||||
device=qkv.device,
|
||||
)
|
||||
out_v = torch.empty_like(out_k)
|
||||
scratch = self._kv_write_scratch(key_cache)
|
||||
kcache_arg, vcache_arg = scratch[0], scratch[1]
|
||||
kv_indices_arg = torch.zeros_like(block_table_decode)
|
||||
|
||||
if self.use_fp8:
|
||||
_, q_scale, split_k_flag = hpc.rope_norm_store_kv_fp8(
|
||||
key_cache=kcache_arg,
|
||||
value_cache=vcache_arg,
|
||||
qkv=qkv_decode,
|
||||
cos_sin=self.cos_sin_cache,
|
||||
num_seqlen_per_req=num_seq_kvcache,
|
||||
q_index=attn_metadata.qo_indptr_decode,
|
||||
kvcache_indices=kv_indices_arg,
|
||||
is_prefill=False,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
quant_policy=self._quant_type,
|
||||
max_seqlens=decode_query_len,
|
||||
q_norm_weight=q_norm_weight,
|
||||
k_norm_weight=k_norm_weight,
|
||||
qk_norm_policy=self.qk_norm_policy,
|
||||
out_q=out_q_decode,
|
||||
out_k=out_k,
|
||||
out_v=out_v,
|
||||
)
|
||||
attn_metadata.hpc_decode_q_scale = q_scale
|
||||
if split_k_flag is not None:
|
||||
attn_metadata.hpc_split_k_flag = split_k_flag
|
||||
else:
|
||||
hpc.rope_norm_store_kv(
|
||||
kcache_arg,
|
||||
vcache_arg,
|
||||
qkv_decode,
|
||||
self.cos_sin_cache,
|
||||
num_seq_kvcache,
|
||||
attn_metadata.qo_indptr_decode,
|
||||
kv_indices_arg,
|
||||
False, # is_prefill
|
||||
q_norm_weight=q_norm_weight,
|
||||
k_norm_weight=k_norm_weight,
|
||||
out_q=out_q_decode,
|
||||
out_k=out_k,
|
||||
out_v=out_v,
|
||||
qk_norm_policy=self.qk_norm_policy,
|
||||
)
|
||||
|
||||
if out_k is not None:
|
||||
self._zero_pad_last_blocks(
|
||||
key_cache, value_cache, num_seq_kvcache, block_table_decode
|
||||
)
|
||||
decode_slots = slot_mapping[:num_decode_tokens]
|
||||
self._scatter_kv_cache(key_cache, out_k, decode_slots)
|
||||
self._scatter_kv_cache(value_cache, out_v, decode_slots)
|
||||
|
||||
# Signal HpcAttentionImpl that KV cache has already been written by
|
||||
# rope_norm_store_kv[_fp8] above, so it should skip its own
|
||||
# reshape_and_cache_flash. Set after the fused kernels ran (either
|
||||
# prefill or decode branch); otherwise the standard KV-write path in
|
||||
# the attention impl would kick in (which is what non-HpcRopeNorm
|
||||
# models rely on).
|
||||
attn_metadata.hpc_kv_written = True
|
||||
@@ -1,646 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_rank
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
sharded_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
from ...linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from ..mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from ..ops.causal_conv1d import causal_conv1d_fn, causal_conv1d_update
|
||||
from ..ops.gather_initial_states import gather_initial_states
|
||||
|
||||
# Empirical lower bound for the KDA gate to avoid numerical underflow.
|
||||
_KDA_GATE_LOGBOUND_MIN = -5.0
|
||||
|
||||
|
||||
def a_log_weight_loader(
|
||||
shard_axis: int,
|
||||
) -> Callable[[torch.Tensor, torch.Tensor], None]:
|
||||
"""Load KDA A_log stored as either old 4D or current 1D weights."""
|
||||
|
||||
def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
shard_size = param.data.shape[shard_axis]
|
||||
start_idx = tp_rank * shard_size
|
||||
|
||||
if loaded_weight.dim() == 4:
|
||||
assert loaded_weight.shape[:2] == (1, 1), (
|
||||
f"Expected old A_log shape (1, 1, H, 1), got {loaded_weight.shape}"
|
||||
)
|
||||
assert loaded_weight.shape[-1] == 1, (
|
||||
f"Expected old A_log last dim to be 1, got {loaded_weight.shape}"
|
||||
)
|
||||
loaded_weight = loaded_weight.view(loaded_weight.shape[2])
|
||||
|
||||
loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size)
|
||||
return default_weight_loader(param, loaded_weight)
|
||||
|
||||
return loader
|
||||
|
||||
|
||||
def _make_fused_conv1d_weight_loader(
|
||||
dims: list[int],
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
) -> Callable[..., None]:
|
||||
sharded_dims = [dim // tp_size for dim in dims]
|
||||
|
||||
def weight_loader(
|
||||
param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: int,
|
||||
) -> None:
|
||||
if loaded_weight.dim() == 2:
|
||||
loaded_weight = loaded_weight.unsqueeze(1)
|
||||
shard_size = sharded_dims[loaded_shard_id]
|
||||
source_start = tp_rank * shard_size
|
||||
target_start = sum(sharded_dims[:loaded_shard_id])
|
||||
loaded_shard = loaded_weight[source_start : source_start + shard_size]
|
||||
param.data[target_start : target_start + shard_size].copy_(loaded_shard)
|
||||
|
||||
return weight_loader
|
||||
|
||||
|
||||
class _KimiGDNMergedColumnParallelLinear(MergedColumnParallelLinear):
|
||||
"""Merged projection with one output replicated across TP ranks.
|
||||
|
||||
The replicated shard is represented as ``size * tp_size`` so the merged
|
||||
parameter reserves ``size`` local rows on every rank. Loading that shard
|
||||
from rank zero then gives every rank the complete checkpoint weight.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_size: int,
|
||||
output_sizes: list[int],
|
||||
replicated_shard_id: int,
|
||||
tp_size: int,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.replicated_shard_id = replicated_shard_id
|
||||
output_sizes = output_sizes.copy()
|
||||
output_sizes[replicated_shard_id] *= tp_size
|
||||
super().__init__(input_size, output_sizes, **kwargs)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
def weight_loader_v2(
|
||||
self,
|
||||
param: BasevLLMParameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
try:
|
||||
super().weight_loader_v2(param, loaded_weight, loaded_shard_id)
|
||||
finally:
|
||||
self.tp_rank = tp_rank
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = param_tp_rank
|
||||
|
||||
|
||||
@PluggableLayer.register("kimi_gated_delta_net_attention")
|
||||
class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
def get_state_dtype(
|
||||
self,
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
if self.model_config is None or self.cache_config is None:
|
||||
raise ValueError("model_config and cache_config must be set")
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
self.model_config.dtype, self.cache_config.mamba_cache_dtype
|
||||
)
|
||||
|
||||
def get_state_shape(
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
self.tp_size,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
conv_kernel_size=self.conv_size,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
kda_config = config.linear_attn_config # type: ignore[attr-defined]
|
||||
assert kda_config is not None, "linear_attn_config must be set"
|
||||
self.head_dim = kda_config["head_dim"]
|
||||
self.num_heads = kda_config["num_heads"]
|
||||
assert self.num_heads % self.tp_size == 0
|
||||
self.local_num_heads = divide(self.num_heads, self.tp_size)
|
||||
|
||||
self.projection_size = self.head_dim * self.num_heads
|
||||
self.local_projection_size = divide(self.projection_size, self.tp_size)
|
||||
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||
self.use_full_rank_gate = kda_config.get("use_full_rank_gate", False)
|
||||
|
||||
if self.use_full_rank_gate:
|
||||
# Keep f_a before the narrow beta shard, then pad each TP-local row
|
||||
# to select the aligned BF16 GEMM path. The padding also avoids an
|
||||
# Inductor correctness issue seen with the row-strided G view.
|
||||
qkvg_output_sizes = [self.projection_size] * 4
|
||||
in_proj_output_sizes = qkvg_output_sizes + [
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
]
|
||||
local_output_size = (
|
||||
4 * self.local_projection_size + self.head_dim + self.local_num_heads
|
||||
)
|
||||
self.in_proj_padding = -local_output_size % 16
|
||||
if self.in_proj_padding:
|
||||
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
|
||||
else:
|
||||
in_proj_output_sizes = [self.projection_size] * 3 + [
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
]
|
||||
self.in_proj_padding = 0
|
||||
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
in_proj_output_sizes,
|
||||
replicated_shard_id=4,
|
||||
tp_size=self.tp_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.in_proj_qkvgfab",
|
||||
)
|
||||
if self.in_proj_padding:
|
||||
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
|
||||
|
||||
self.f_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
self.projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
)
|
||||
self.dt_bias = nn.Parameter(
|
||||
torch.empty(self.local_projection_size, dtype=torch.float32)
|
||||
)
|
||||
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
# One packed parameter and cache let decode run a single conv update.
|
||||
# Prefill slices them back into Q/K/V to obtain dense outputs cheaply.
|
||||
self.conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_size,
|
||||
output_size=3 * self.projection_size,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
delattr(self.conv1d.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.conv1d.weight,
|
||||
{
|
||||
"weight_loader": _make_fused_conv1d_weight_loader(
|
||||
[self.projection_size] * 3,
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
self.A_log = nn.Parameter(
|
||||
torch.empty(self.local_num_heads, dtype=torch.float32)
|
||||
)
|
||||
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
|
||||
|
||||
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
|
||||
if self.gate_lower_bound is not None:
|
||||
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
|
||||
"KDA gate lower bound must be in "
|
||||
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
|
||||
f"Got {self.gate_lower_bound}."
|
||||
)
|
||||
self.use_safe_gate = self.gate_lower_bound is not None
|
||||
additional_config = vllm_config.additional_config
|
||||
backend = (
|
||||
additional_config.get("kda_prefill_backend", "auto")
|
||||
if isinstance(additional_config, dict)
|
||||
else "auto"
|
||||
)
|
||||
backend = "triton" if backend == "auto" else backend
|
||||
assert backend == "triton", (
|
||||
"The shared Kimi GDN layer only supports the Triton KDA "
|
||||
f"prefill backend, got {backend!r}."
|
||||
)
|
||||
if not self.use_full_rank_gate:
|
||||
self.g_a_proj = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_a_proj",
|
||||
)
|
||||
self.g_b_proj = ColumnParallelLinear(
|
||||
self.head_dim,
|
||||
self.projection_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
)
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.projection_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def rearrange_mixed_qkv(
|
||||
self, mixed_qkv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
seq_len = mixed_qkv.shape[0]
|
||||
qkv = mixed_qkv.view(seq_len, 3, self.local_num_heads, self.head_dim)
|
||||
# Materialize all three row-strided inputs with one token-major to
|
||||
# QKV-major permutation. Each unbound tensor is then contiguous.
|
||||
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
|
||||
return qkv.unbind(0)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = hidden_states.size(0)
|
||||
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
|
||||
if self.use_full_rank_gate:
|
||||
split_sizes = [
|
||||
3 * self.local_projection_size,
|
||||
self.local_projection_size,
|
||||
self.head_dim,
|
||||
self.local_num_heads,
|
||||
]
|
||||
if self.in_proj_padding:
|
||||
split_sizes.append(self.in_proj_padding)
|
||||
projected = projected_qkvgfab.split(split_sizes, dim=-1)
|
||||
mixed_qkv, g_proj_states, f_a, beta = projected[:4]
|
||||
else:
|
||||
mixed_qkv, beta, f_a = projected_qkvgfab.split(
|
||||
[
|
||||
3 * self.local_projection_size,
|
||||
self.local_num_heads,
|
||||
self.head_dim,
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
g_proj_states = self.g_b_proj(self.g_a_proj(hidden_states)[0])[0]
|
||||
|
||||
g1 = self.f_b_proj(f_a)[0]
|
||||
beta = beta.unsqueeze(0)
|
||||
g1 = rearrange(g1, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
|
||||
g2 = rearrange(g_proj_states, "... (h d) -> ... h d", d=self.head_dim)
|
||||
|
||||
core_attn_out = torch.empty(
|
||||
(1, num_tokens, self.local_num_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
self._forward(
|
||||
mixed_qkv=mixed_qkv,
|
||||
g1=g1,
|
||||
g2=g2,
|
||||
beta=beta,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
core_attn_out = rearrange(core_attn_out, "1 n h d -> n (h d)")
|
||||
return self.o_proj(core_attn_out)[0]
|
||||
|
||||
@eager_break_during_capture
|
||||
def _forward(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
g1: torch.Tensor,
|
||||
g2: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata_raw is None:
|
||||
return
|
||||
|
||||
# Vendor-specific KDA kernels: AMD/ROCm and NVIDIA keep their own copies
|
||||
# under kimi_k3/{amd,nvidia}/ops so each can diverge independently.
|
||||
# These copies may have different signatures for Kimi-K3, but they agree
|
||||
# on the arguments used here.
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
elif current_platform.is_rocm():
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import ( # type: ignore[assignment]
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
else:
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata_narrowed is None:
|
||||
# Profile/warmup dummy runs skip mamba-family metadata.
|
||||
return
|
||||
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
||||
m = attn_metadata_narrowed
|
||||
has_initial_state = m.has_initial_state
|
||||
non_spec_query_start_loc = m.non_spec_query_start_loc
|
||||
non_spec_state_indices_tensor = m.non_spec_state_indices_tensor
|
||||
spec_sequence_masks = m.spec_sequence_masks
|
||||
spec_token_indx = m.spec_token_indx
|
||||
non_spec_token_indx = m.non_spec_token_indx
|
||||
spec_state_indices_tensor = m.spec_state_indices_tensor
|
||||
spec_query_start_loc = m.spec_query_start_loc
|
||||
num_accepted_tokens = m.num_accepted_tokens
|
||||
num_actual_tokens = m.num_actual_tokens
|
||||
mixed_qkv = mixed_qkv[:num_actual_tokens]
|
||||
g1 = g1[:, :num_actual_tokens]
|
||||
beta = beta[:, :num_actual_tokens]
|
||||
|
||||
constant_caches = self.kv_cache
|
||||
|
||||
conv_state, recurrent_state = constant_caches
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
q_conv_weight, k_conv_weight, v_conv_weight = conv_weights.split(
|
||||
self.local_projection_size, dim=0
|
||||
)
|
||||
q_conv_state, k_conv_state, v_conv_state = conv_state.split(
|
||||
self.local_projection_size, dim=-2
|
||||
)
|
||||
|
||||
# Split tokens into the multi-query spec-decode part and the remaining
|
||||
# (prefill / plain decode) part.
|
||||
if spec_sequence_masks is not None:
|
||||
if m.num_prefills == 0 and m.num_decodes == 0:
|
||||
mixed_qkv_spec = mixed_qkv
|
||||
g1_spec, beta_spec = g1, beta
|
||||
mixed_qkv_ns = g1_ns = beta_ns = None
|
||||
else:
|
||||
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
|
||||
g1_spec = g1.index_select(1, spec_token_indx)
|
||||
beta_spec = beta.index_select(1, spec_token_indx)
|
||||
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
|
||||
g1_ns = g1.index_select(1, non_spec_token_indx)
|
||||
beta_ns = beta.index_select(1, non_spec_token_indx)
|
||||
else:
|
||||
mixed_qkv_spec = g1_spec = beta_spec = None
|
||||
mixed_qkv_ns, g1_ns, beta_ns = mixed_qkv, g1, beta
|
||||
|
||||
# ---------- spec-decode multi-query path ----------
|
||||
core_attn_out_spec = None
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert spec_query_start_loc is not None
|
||||
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
|
||||
spec_max_query_len = spec_state_indices_tensor.size(-1)
|
||||
|
||||
# Sibling beta and, for full-rank gates, output-gate views remain
|
||||
# live, so write the convolution output separately.
|
||||
spec_conv_out = torch.empty(
|
||||
mixed_qkv_spec.shape,
|
||||
dtype=mixed_qkv_spec.dtype,
|
||||
device=mixed_qkv_spec.device,
|
||||
)
|
||||
mixed_qkv_spec = causal_conv1d_update(
|
||||
mixed_qkv_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=spec_conv_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=spec_max_query_len,
|
||||
validate_data=False,
|
||||
out=spec_conv_out,
|
||||
)
|
||||
q_spec, k_spec, v_spec = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in mixed_qkv_spec.split(self.local_projection_size, dim=-1)
|
||||
)
|
||||
spec_cu_seqlens = spec_query_start_loc[: m.num_spec_decodes + 1]
|
||||
# Spec-only batches write directly into core_attn_out.
|
||||
spec_out = (
|
||||
core_attn_out[:, : q_spec.shape[1]]
|
||||
if m.num_prefills == 0 and m.num_decodes == 0
|
||||
else None
|
||||
)
|
||||
core_attn_out_spec, _ = fused_recurrent_kda(
|
||||
q=q_spec,
|
||||
k=k_spec,
|
||||
v=v_spec,
|
||||
raw_g=g1_spec,
|
||||
raw_beta=beta_spec,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=spec_cu_seqlens,
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=spec_out,
|
||||
)
|
||||
|
||||
# ---------- non-spec path (prefill or plain decode) ----------
|
||||
core_attn_out_non_spec = None
|
||||
if mixed_qkv_ns is not None:
|
||||
assert g1_ns is not None and beta_ns is not None
|
||||
if m.num_prefills > 0:
|
||||
q_ns, k_ns, v_ns = mixed_qkv_ns.split(
|
||||
self.local_projection_size, dim=-1
|
||||
)
|
||||
|
||||
# Packed prefill conv would require copying V solely to make
|
||||
# it dense for KDA. Separate calls accept the strided inputs
|
||||
# and produce dense Q/K/V without that extra traffic.
|
||||
# TODO: Use packed conv once every KDA prefill backend accepts
|
||||
# row-strided Q/K/V directly.
|
||||
def _prefill_conv(
|
||||
x: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=m,
|
||||
).transpose(0, 1)
|
||||
|
||||
q_ns = _prefill_conv(q_ns, q_conv_state, q_conv_weight)
|
||||
k_ns = _prefill_conv(k_ns, k_conv_state, k_conv_weight)
|
||||
v_ns = _prefill_conv(v_ns, v_conv_state, v_conv_weight)
|
||||
q_ns, k_ns, v_ns = (
|
||||
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
|
||||
for x in (q_ns, k_ns, v_ns)
|
||||
)
|
||||
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
initial_state = gather_initial_states(
|
||||
recurrent_state,
|
||||
non_spec_state_indices_tensor,
|
||||
has_initial_state,
|
||||
)
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_with_fused_gate(
|
||||
q=q_ns,
|
||||
k=k_ns,
|
||||
v=v_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
)
|
||||
# Init cache
|
||||
recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state
|
||||
|
||||
else:
|
||||
# pure-decode non-spec batch
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
decode_conv_indices = non_spec_state_indices_tensor[
|
||||
: mixed_qkv_ns.size(0)
|
||||
]
|
||||
# Sibling beta and, for full-rank gates, output-gate views
|
||||
# remain live, so write the conv output separately.
|
||||
packed_conv_out = torch.empty(
|
||||
mixed_qkv_ns.shape,
|
||||
dtype=mixed_qkv_ns.dtype,
|
||||
device=mixed_qkv_ns.device,
|
||||
)
|
||||
mixed_qkv_ns = causal_conv1d_update(
|
||||
mixed_qkv_ns,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation="silu",
|
||||
conv_state_indices=decode_conv_indices,
|
||||
validate_data=True,
|
||||
out=packed_conv_out,
|
||||
)
|
||||
core_attn_out_non_spec, _ = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv_ns,
|
||||
raw_g=g1_ns,
|
||||
raw_beta=beta_ns,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=recurrent_state,
|
||||
state_indices=decode_conv_indices,
|
||||
)
|
||||
|
||||
# ---------- merge spec and non-spec outputs ----------
|
||||
if core_attn_out_spec is not None and core_attn_out_non_spec is not None:
|
||||
# Mixed batches require indexed placement in the original order.
|
||||
merged = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_spec.dtype,
|
||||
device=core_attn_out_spec.device,
|
||||
)
|
||||
merged.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||
elif core_attn_out_non_spec is not None:
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
else:
|
||||
assert core_attn_out_spec is not None
|
||||
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
|
||||
@@ -1,626 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.layernorm import RMSNormGated
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
sharded_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops import (
|
||||
chunk_gated_delta_rule,
|
||||
fused_recurrent_gated_delta_rule,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.triton_utils.allocation import set_triton_allocator
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
|
||||
@PluggableLayer.register("olmo_hybrid_gated_delta_net_attention")
|
||||
class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
"""Gated DeltaNet linear attention layer for OLMo Hybrid.
|
||||
|
||||
This implements the linear attention mechanism that replaces sliding window
|
||||
attention in the hybrid architecture.
|
||||
"""
|
||||
|
||||
def get_state_shape(
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||
self.tp_size,
|
||||
self.num_k_heads,
|
||||
self.num_v_heads,
|
||||
self.head_k_dim,
|
||||
self.head_v_dim,
|
||||
self.conv_kernel_size,
|
||||
self.num_spec,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix=prefix)
|
||||
|
||||
assert getattr(config, "linear_use_gate", True), (
|
||||
"OlmoHybridGatedDeltaNet requires linear_use_gate=True"
|
||||
)
|
||||
self.num_k_heads = config.linear_num_key_heads
|
||||
self.num_v_heads = config.linear_num_value_heads
|
||||
self.head_k_dim = config.linear_key_head_dim
|
||||
self.head_v_dim = config.linear_value_head_dim
|
||||
self.conv_kernel_size = config.linear_conv_kernel_dim
|
||||
self.key_dim = self.head_k_dim * self.num_k_heads
|
||||
self.value_dim = self.head_v_dim * self.num_v_heads
|
||||
self.allow_neg_eigval = getattr(config, "linear_allow_neg_eigval", False)
|
||||
|
||||
# Fused QKVG projection: 1 matmul instead of 4
|
||||
self.in_proj_qkvg = MergedColumnParallelLinear(
|
||||
input_size=self.hidden_size,
|
||||
output_sizes=[self.key_dim, self.key_dim, self.value_dim, self.value_dim],
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.in_proj_qkvg",
|
||||
)
|
||||
|
||||
# Separate B and A projections to preserve numerical precision.
|
||||
# Fusing these into one matmul changes FP accumulation order for the
|
||||
# gating scalars, which compounds through the GDN recurrent state.
|
||||
self.b_proj = ColumnParallelLinear(
|
||||
input_size=self.hidden_size,
|
||||
output_size=self.num_v_heads,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.b_proj",
|
||||
)
|
||||
self.a_proj = ColumnParallelLinear(
|
||||
input_size=self.hidden_size,
|
||||
output_size=self.num_v_heads,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.a_proj",
|
||||
)
|
||||
|
||||
# Fused conv1d: single parameter instead of 3
|
||||
self.conv_dim = self.key_dim * 2 + self.value_dim
|
||||
self.conv1d = ColumnParallelLinear(
|
||||
input_size=self.conv_kernel_size,
|
||||
output_size=self.conv_dim,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
delattr(self.conv1d.weight, "weight_loader")
|
||||
set_weight_attrs(
|
||||
self.conv1d.weight,
|
||||
{
|
||||
"weight_loader": _make_fused_conv1d_weight_loader(
|
||||
[self.key_dim, self.key_dim, self.value_dim],
|
||||
self.tp_size,
|
||||
self.tp_rank,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
self.dt_bias = nn.Parameter(
|
||||
torch.ones(self.num_v_heads // self.tp_size),
|
||||
)
|
||||
self.A_log = nn.Parameter(
|
||||
torch.empty(
|
||||
divide(self.num_v_heads, self.tp_size),
|
||||
)
|
||||
)
|
||||
|
||||
set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(0)})
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
# use eps=1e-5 to match FLA's FusedRMSNormGated
|
||||
self.o_norm = RMSNormGated(
|
||||
self.head_v_dim,
|
||||
eps=1e-5,
|
||||
group_size=None,
|
||||
norm_before_gate=True,
|
||||
device=current_platform.current_device(),
|
||||
dtype=config.torch_dtype if hasattr(config, "torch_dtype") else None,
|
||||
)
|
||||
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.value_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
# FLA triton kernels need a PyTorch-backed allocator for scratch
|
||||
# memory (required by triton >= 3.x autotuner). Set once at init.
|
||||
set_triton_allocator(current_platform.current_device())
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def rearrange_mixed_qkv(self, mixed_qkv):
|
||||
if mixed_qkv is None:
|
||||
return None, None, None
|
||||
query, key, value = torch.split(
|
||||
mixed_qkv,
|
||||
[
|
||||
self.key_dim // self.tp_size,
|
||||
self.key_dim // self.tp_size,
|
||||
self.value_dim // self.tp_size,
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
num_k_heads = self.num_k_heads // self.tp_size
|
||||
num_v_heads = self.num_v_heads // self.tp_size
|
||||
|
||||
query = rearrange(query, "l (h d) -> 1 l h d", h=num_k_heads, d=self.head_k_dim)
|
||||
key = rearrange(key, "l (h d) -> 1 l h d", h=num_k_heads, d=self.head_k_dim)
|
||||
value = rearrange(value, "l (h d) -> 1 l h d", h=num_v_heads, d=self.head_v_dim)
|
||||
|
||||
# GQA expansion if needed
|
||||
if num_v_heads > num_k_heads:
|
||||
expand_ratio = num_v_heads // num_k_heads
|
||||
query = query.unsqueeze(3).expand(-1, -1, -1, expand_ratio, -1)
|
||||
query = query.reshape(1, query.shape[1], num_v_heads, self.head_k_dim)
|
||||
key = key.unsqueeze(3).expand(-1, -1, -1, expand_ratio, -1)
|
||||
key = key.reshape(1, key.shape[1], num_v_heads, self.head_k_dim)
|
||||
|
||||
return query.contiguous(), key.contiguous(), value.contiguous()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
# NOTE: We wrap the ENTIRE linear attention forward (projections +
|
||||
# core recurrence + output norm + output projection) in a single
|
||||
# custom op, rather than just wrapping the recurrent core like
|
||||
# other GDN models (e.g. Qwen3Next) do.
|
||||
#
|
||||
# Why: torch.compile with inductor generates fused kernels for
|
||||
# matmuls and pointwise ops. These fused kernels can differ in
|
||||
# floating-point accumulation order from eager-mode cuBLAS,
|
||||
# introducing small numerical differences (~1e-7 per op). For
|
||||
# standard transformer attention this is harmless because each
|
||||
# position is computed independently. But for the GDN recurrent
|
||||
# state, these tiny input differences compound at every timestep
|
||||
# across the full sequence length, causing severe logprob
|
||||
# divergence (e.g. ~15% top-1 agreement with eager baseline).
|
||||
#
|
||||
# By making the full forward opaque to inductor, the projections
|
||||
# and output norm run with eager-mode kernels (cuBLAS, triton),
|
||||
# preserving numerical consistency. The tradeoff is reduced
|
||||
# compilation speedup (~1.5x vs ~3x), but logprob agreement
|
||||
# improves from ~15% to ~83% top-1 vs eager.
|
||||
#
|
||||
# The remaining ~17% divergence comes from inductor compiling
|
||||
# the MLP and transformer attention layers that are NOT wrapped
|
||||
# in custom ops -- their small precision differences propagate
|
||||
# as inputs to the GDN layers from outside.
|
||||
torch.ops.vllm.olmo_hybrid_gdn_full_forward(
|
||||
hidden_states,
|
||||
output,
|
||||
self.prefix,
|
||||
)
|
||||
|
||||
def _full_forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
num_tokens = hidden_states.size(0)
|
||||
|
||||
# ============================================================
|
||||
# Part 1: Input Projection (2 fused matmuls instead of 6)
|
||||
# ============================================================
|
||||
projected_qkvg, _ = self.in_proj_qkvg(hidden_states)
|
||||
conv_dim_sharded = (self.key_dim * 2 + self.value_dim) // self.tp_size
|
||||
mixed_qkv = projected_qkvg[..., :conv_dim_sharded]
|
||||
gate = projected_qkvg[..., conv_dim_sharded:]
|
||||
|
||||
b, _ = self.b_proj(hidden_states)
|
||||
a, _ = self.a_proj(hidden_states)
|
||||
|
||||
# ============================================================
|
||||
# Part 2: Core Attention
|
||||
# ============================================================
|
||||
core_attn_out = torch.zeros(
|
||||
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
self._forward_core(
|
||||
mixed_qkv=mixed_qkv,
|
||||
b=b,
|
||||
a=a,
|
||||
core_attn_out=core_attn_out,
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# Part 3: Output Projection
|
||||
# ============================================================
|
||||
gate = gate.view(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim)
|
||||
core_attn_out_flat = core_attn_out.reshape(-1, core_attn_out.shape[-1])
|
||||
gate_flat = gate.reshape(-1, gate.shape[-1])
|
||||
core_attn_out_normed = self.o_norm(core_attn_out_flat, gate_flat)
|
||||
core_attn_out = core_attn_out_normed.view(
|
||||
num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim
|
||||
)
|
||||
|
||||
core_attn_out = rearrange(core_attn_out, "l h d -> l (h d)")
|
||||
output[:num_tokens], _ = self.o_proj(core_attn_out)
|
||||
|
||||
def _forward_core(
|
||||
self,
|
||||
mixed_qkv: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
core_attn_out: torch.Tensor,
|
||||
):
|
||||
"""Core attention computation (called by custom op)."""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
|
||||
if attn_metadata is None:
|
||||
# V1 profile run
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
attn_metadata = attn_metadata.get(self.prefix) # type: ignore[assignment]
|
||||
if attn_metadata is None:
|
||||
# Profile/warmup dummy runs skip mamba-family metadata.
|
||||
return
|
||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||
has_initial_state = attn_metadata.has_initial_state
|
||||
spec_query_start_loc = attn_metadata.spec_query_start_loc
|
||||
non_spec_query_start_loc = attn_metadata.non_spec_query_start_loc
|
||||
spec_sequence_masks = attn_metadata.spec_sequence_masks
|
||||
spec_token_indx = attn_metadata.spec_token_indx
|
||||
non_spec_token_indx = attn_metadata.non_spec_token_indx
|
||||
spec_state_indices_tensor = attn_metadata.spec_state_indices_tensor
|
||||
non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor
|
||||
self_kv_cache = self.kv_cache
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||
conv_state = (
|
||||
self_kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self_kv_cache[0].transpose(-1, -2)
|
||||
)
|
||||
ssm_state = self_kv_cache[1]
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
num_accepted_tokens = attn_metadata.num_accepted_tokens
|
||||
|
||||
mixed_qkv = mixed_qkv[:num_actual_tokens]
|
||||
b = b[:num_actual_tokens]
|
||||
a = a[:num_actual_tokens]
|
||||
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
|
||||
if spec_sequence_masks is not None:
|
||||
if attn_metadata.num_prefills == 0 and attn_metadata.num_decodes == 0:
|
||||
mixed_qkv_spec = mixed_qkv
|
||||
mixed_qkv_non_spec = None
|
||||
else:
|
||||
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
|
||||
mixed_qkv_non_spec = mixed_qkv.index_select(0, non_spec_token_indx)
|
||||
else:
|
||||
mixed_qkv_spec = None
|
||||
mixed_qkv_non_spec = mixed_qkv
|
||||
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_query_start_loc is not None
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert num_accepted_tokens is not None
|
||||
mixed_qkv_spec = causal_conv1d_update(
|
||||
mixed_qkv_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
None, # no bias
|
||||
self.activation,
|
||||
conv_state_indices=spec_state_indices_tensor[:, 0][
|
||||
: attn_metadata.num_spec_decodes
|
||||
],
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=spec_state_indices_tensor.size(-1),
|
||||
validate_data=False,
|
||||
)
|
||||
|
||||
if attn_metadata.num_prefills > 0:
|
||||
assert mixed_qkv_non_spec is not None
|
||||
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
|
||||
mixed_qkv_non_spec = causal_conv1d_fn(
|
||||
mixed_qkv_non_spec_T,
|
||||
conv_weights,
|
||||
None,
|
||||
activation=self.activation,
|
||||
conv_states=conv_state,
|
||||
has_initial_state=has_initial_state,
|
||||
cache_indices=non_spec_state_indices_tensor,
|
||||
query_start_loc=non_spec_query_start_loc,
|
||||
metadata=attn_metadata,
|
||||
).transpose(0, 1)
|
||||
elif attn_metadata.num_decodes > 0:
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
mixed_qkv_non_spec = causal_conv1d_update(
|
||||
mixed_qkv_non_spec,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
None,
|
||||
self.activation,
|
||||
conv_state_indices=non_spec_state_indices_tensor[
|
||||
: attn_metadata.num_decodes
|
||||
],
|
||||
validate_data=True,
|
||||
)
|
||||
else:
|
||||
mixed_qkv_non_spec = None
|
||||
|
||||
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
|
||||
query_non_spec, key_non_spec, value_non_spec = self.rearrange_mixed_qkv(
|
||||
mixed_qkv_non_spec
|
||||
)
|
||||
|
||||
g, beta = fused_olmo_hybrid_gdn_gating(
|
||||
self.A_log, a, b, self.dt_bias, self.allow_neg_eigval
|
||||
)
|
||||
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_token_indx is not None
|
||||
assert non_spec_token_indx is not None
|
||||
if attn_metadata.num_prefills == 0 and attn_metadata.num_decodes == 0:
|
||||
g_spec = g
|
||||
beta_spec = beta
|
||||
g_non_spec = None
|
||||
beta_non_spec = None
|
||||
else:
|
||||
g_spec = g.index_select(1, spec_token_indx)
|
||||
beta_spec = beta.index_select(1, spec_token_indx)
|
||||
g_non_spec = g.index_select(1, non_spec_token_indx)
|
||||
beta_non_spec = beta.index_select(1, non_spec_token_indx)
|
||||
else:
|
||||
g_spec = None
|
||||
beta_spec = None
|
||||
g_non_spec = g
|
||||
beta_non_spec = beta
|
||||
|
||||
if spec_sequence_masks is not None:
|
||||
assert spec_query_start_loc is not None
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert num_accepted_tokens is not None
|
||||
core_attn_out_spec, last_recurrent_state = fused_recurrent_gated_delta_rule(
|
||||
q=query_spec,
|
||||
k=key_spec,
|
||||
v=value_spec,
|
||||
g=g_spec,
|
||||
beta=beta_spec,
|
||||
initial_state=ssm_state,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=spec_query_start_loc[: attn_metadata.num_spec_decodes + 1],
|
||||
ssm_state_indices=spec_state_indices_tensor,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
else:
|
||||
core_attn_out_spec, last_recurrent_state = None, None
|
||||
|
||||
if attn_metadata.num_prefills > 0:
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
assert non_spec_query_start_loc is not None
|
||||
initial_state = ssm_state[non_spec_state_indices_tensor].contiguous()
|
||||
initial_state[~has_initial_state, ...] = 0
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
ssm_state[non_spec_state_indices_tensor] = last_recurrent_state.to(
|
||||
ssm_state.dtype
|
||||
)
|
||||
elif attn_metadata.num_decodes > 0:
|
||||
assert non_spec_query_start_loc is not None
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
core_attn_out_non_spec, last_recurrent_state = (
|
||||
fused_recurrent_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=ssm_state,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=non_spec_query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
],
|
||||
ssm_state_indices=non_spec_state_indices_tensor,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
)
|
||||
else:
|
||||
core_attn_out_non_spec, last_recurrent_state = None, None
|
||||
|
||||
if spec_sequence_masks is not None and core_attn_out_non_spec is not None:
|
||||
merged_out = torch.empty(
|
||||
(1, num_actual_tokens, *core_attn_out_spec.shape[2:]),
|
||||
dtype=core_attn_out_non_spec.dtype,
|
||||
device=core_attn_out_non_spec.device,
|
||||
)
|
||||
merged_out.index_copy_(1, spec_token_indx, core_attn_out_spec)
|
||||
merged_out.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[:num_actual_tokens] = merged_out.squeeze(0)
|
||||
elif spec_sequence_masks is not None:
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0)
|
||||
else:
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0)
|
||||
|
||||
|
||||
def _make_fused_conv1d_weight_loader(dims, tp_size, tp_rank):
|
||||
"""Weight loader for loading separate HF conv weights into a fused conv1d.
|
||||
|
||||
dims: list of original (un-sharded) dims per section,
|
||||
e.g. [key_dim, key_dim, value_dim]
|
||||
"""
|
||||
sharded_dims = [d // tp_size for d in dims]
|
||||
|
||||
def weight_loader(param, loaded_weight, loaded_shard_id=None):
|
||||
if loaded_weight.dim() == 2:
|
||||
loaded_weight = loaded_weight.unsqueeze(1)
|
||||
dim = dims[loaded_shard_id]
|
||||
shard_size = dim // tp_size
|
||||
tp_start = tp_rank * shard_size
|
||||
sharded_weight = loaded_weight[tp_start : tp_start + shard_size]
|
||||
offset = sum(sharded_dims[:loaded_shard_id])
|
||||
param.data[offset : offset + shard_size].copy_(sharded_weight)
|
||||
|
||||
return weight_loader
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def olmo_hybrid_gdn_full_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Full linear attention forward wrapped as a custom op.
|
||||
|
||||
Prevents inductor from compiling the projections around the GDN core,
|
||||
which would introduce numerical divergence that compounds through
|
||||
the recurrent state.
|
||||
"""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._full_forward(
|
||||
hidden_states=hidden_states,
|
||||
output=output,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="olmo_hybrid_gdn_full_forward",
|
||||
op_func=olmo_hybrid_gdn_full_forward,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fused_olmo_hybrid_gdn_gating_kernel(
|
||||
g,
|
||||
beta_output,
|
||||
A_log,
|
||||
a,
|
||||
b,
|
||||
dt_bias,
|
||||
seq_len,
|
||||
allow_neg_eigval: tl.constexpr,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
beta: tl.constexpr,
|
||||
threshold: tl.constexpr,
|
||||
BLK_HEADS: tl.constexpr,
|
||||
):
|
||||
i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS)
|
||||
off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off
|
||||
mask = head_off < NUM_HEADS
|
||||
blk_A_log = tl.load(A_log + head_off, mask=mask)
|
||||
blk_a = tl.load(a + off, mask=mask)
|
||||
blk_b = tl.load(b + off, mask=mask)
|
||||
blk_bias = tl.load(dt_bias + head_off, mask=mask)
|
||||
|
||||
# g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
|
||||
x = blk_a.to(tl.float32) + blk_bias.to(tl.float32)
|
||||
softplus_x = tl.where(
|
||||
beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x
|
||||
)
|
||||
blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x
|
||||
tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask)
|
||||
|
||||
# beta = self.b_proj(hidden_states).sigmoid()
|
||||
# if self.allow_neg_eigval: beta = beta * 2.0
|
||||
blk_beta_output = tl.sigmoid(blk_b.to(tl.float32))
|
||||
if allow_neg_eigval:
|
||||
blk_beta_output = blk_beta_output * 2.0
|
||||
tl.store(
|
||||
beta_output + off, blk_beta_output.to(beta_output.dtype.element_ty), mask=mask
|
||||
)
|
||||
|
||||
|
||||
def fused_olmo_hybrid_gdn_gating(
|
||||
A_log: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
allow_neg_eigval: bool = False,
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch, num_heads = a.shape
|
||||
seq_len = 1
|
||||
grid = (batch, seq_len, triton.cdiv(num_heads, 8))
|
||||
g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device)
|
||||
beta_output = torch.empty(1, batch, num_heads, dtype=torch.float32, device=b.device)
|
||||
fused_olmo_hybrid_gdn_gating_kernel[grid](
|
||||
g,
|
||||
beta_output,
|
||||
A_log,
|
||||
a,
|
||||
b,
|
||||
dt_bias,
|
||||
seq_len,
|
||||
allow_neg_eigval,
|
||||
num_heads,
|
||||
beta,
|
||||
threshold,
|
||||
8,
|
||||
num_warps=1,
|
||||
)
|
||||
return g, beta_output
|
||||
@@ -54,7 +54,12 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
|
||||
try:
|
||||
# inc/ loader was removed in the SM86 mini-vLLM cut.
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
except ImportError:
|
||||
INCConfig = type("AbsentINCConfig", (), {})
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
sharded_weight_loader,
|
||||
)
|
||||
|
||||
@@ -1,247 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import Int32, cute
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
from .kernel_h import h_cutedsl
|
||||
from .kernel_kkt_inv_uw import kkt_inv_uw_cutedsl
|
||||
from .kernel_o import o_cutedsl
|
||||
|
||||
|
||||
class PrepMetaKernel:
|
||||
def __init__(self, BT: int) -> None:
|
||||
self.BT = BT
|
||||
self.num_warps = 8
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
chunk_offsets: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
block = (self.num_warps * 32, 1, 1)
|
||||
self.kernel(
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
chunk_offsets,
|
||||
).launch(grid=(1, 1, 1), block=block, stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
chunk_offsets: cute.Tensor,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
|
||||
num_seqs = cu_seqlens.shape[0] - 1
|
||||
num_warps = self.num_warps
|
||||
tb_size = num_warps * 32
|
||||
|
||||
if tid == 0:
|
||||
chunk_offsets[0] = 0
|
||||
|
||||
coarsen = cute.ceil_div(num_seqs, tb_size)
|
||||
seq_start = tid * coarsen
|
||||
num_iters = cutlass.min(seq_start + coarsen, num_seqs) - seq_start
|
||||
|
||||
# First pass: compute this thread's total chunk count.
|
||||
thread_sum = Int32(0)
|
||||
for i in range(num_iters):
|
||||
seq_id = seq_start + i
|
||||
seqlen = cu_seqlens[seq_id + 1] - cu_seqlens[seq_id]
|
||||
thread_sum += cute.ceil_div(seqlen, self.BT)
|
||||
|
||||
# warp parallel scan
|
||||
cu_num_chunks = thread_sum
|
||||
for i in cutlass.range_constexpr(5):
|
||||
offset = cutlass.const_expr(1 << i)
|
||||
lower = cute.arch.shuffle_sync_up(
|
||||
cu_num_chunks, offset=offset, mask_and_clamp=0
|
||||
)
|
||||
if lane_id >= offset:
|
||||
cu_num_chunks += lower
|
||||
|
||||
# cross-warp cumsum (CTA-wide)
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
warp_num_chunks = smem.allocate_array(Int32, num_warps)
|
||||
if lane_id == 31:
|
||||
warp_num_chunks[warp_id] = cu_num_chunks
|
||||
cute.arch.sync_threads()
|
||||
|
||||
for i in cutlass.range_constexpr(1, num_warps):
|
||||
if warp_id >= i:
|
||||
cu_num_chunks += warp_num_chunks[i - 1]
|
||||
|
||||
chunk_start = cu_num_chunks - thread_sum
|
||||
|
||||
# Second pass: recompute per-sequence chunk counts and write results.
|
||||
for i in range(num_iters):
|
||||
seq_id = seq_start + i
|
||||
seqlen = cu_seqlens[seq_id + 1] - cu_seqlens[seq_id]
|
||||
num_chunks = cute.ceil_div(seqlen, self.BT)
|
||||
chunk_end = chunk_start + num_chunks
|
||||
chunk_offsets[seq_id + 1] = chunk_end
|
||||
|
||||
for chunk_id in range(num_chunks):
|
||||
chunk_indices[chunk_start + chunk_id, 0] = seq_id
|
||||
chunk_indices[chunk_start + chunk_id, 1] = chunk_id
|
||||
|
||||
chunk_start = chunk_end
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(BT: int):
|
||||
cu_entries = cute.sym_int()
|
||||
upper_bound_chunks = cute.sym_int()
|
||||
|
||||
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
|
||||
chunk_indices = make_fake_tensor(Int32, (upper_bound_chunks, 2), divisibility=2)
|
||||
chunk_offsets = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
|
||||
|
||||
kernel = PrepMetaKernel(BT)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
chunk_offsets,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def _upper_bound_chunks(num_seqs: int, total_tokens: int, chunk_size: int) -> int:
|
||||
return (num_seqs - 1) + triton.cdiv(total_tokens - (num_seqs - 1), chunk_size)
|
||||
|
||||
|
||||
def prepare_metadata_cutedsl(
|
||||
cu_seqlens: torch.Tensor,
|
||||
total_tokens: int,
|
||||
chunk_size: int = 64,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_seqs = cu_seqlens.numel() - 1
|
||||
upper_bound_chunks = _upper_bound_chunks(num_seqs, total_tokens, chunk_size)
|
||||
chunk_offsets = cu_seqlens.new_empty(num_seqs + 1, dtype=torch.int32)
|
||||
chunk_indices = cu_seqlens.new_empty((upper_bound_chunks, 2), dtype=torch.int32)
|
||||
|
||||
PrepMetaKernel.compile(chunk_size)(cu_seqlens, chunk_indices, chunk_offsets)
|
||||
return chunk_indices, chunk_offsets
|
||||
|
||||
|
||||
def chunk_gated_delta_rule_cutedsl(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
initial_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
chunk_indices: torch.Tensor,
|
||||
chunk_offsets: torch.Tensor,
|
||||
core_attn_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the GDN chunk CuteDSL prefill kernels.
|
||||
|
||||
Args:
|
||||
q: Query tensor with shape ``[1, T, H, K]``.
|
||||
k: Key tensor with shape ``[1, T, H, K]``.
|
||||
v: Value tensor with shape ``[1, T, Hv, V]``.
|
||||
g: Log-space decay tensor with shape ``[1, T, Hv]``.
|
||||
beta: Delta-rule beta tensor with shape ``[1, T, Hv]``.
|
||||
initial_state: Recurrent state with shape ``[N, Hv, V, K]``.
|
||||
cu_seqlens: Cumulative sequence lengths with shape ``[N + 1]``.
|
||||
chunk_indices: Chunk index metadata with shape ``[NT, 2]``.
|
||||
chunk_offsets: Cumulative chunk offsets with shape ``[N + 1]``.
|
||||
core_attn_out: Optional output buffer with shape ``[T, Hv, V]``.
|
||||
|
||||
Returns:
|
||||
A tuple ``(output, final_state)`` where ``output`` has shape
|
||||
``[1, T, Hv, V]`` and ``final_state`` has shape ``[N, Hv, V, K]``.
|
||||
When ``core_attn_out`` is provided, ``output`` is an unsqueezed view of
|
||||
that buffer.
|
||||
|
||||
"""
|
||||
q = q.squeeze(0)
|
||||
k = k.squeeze(0)
|
||||
v = v.squeeze(0)
|
||||
g = g.squeeze(0)
|
||||
beta = beta.squeeze(0)
|
||||
|
||||
_, _, K_dim = k.shape
|
||||
_, num_v_heads, V_dim = v.shape
|
||||
chunk_size = 64
|
||||
upper_bound_chunks = chunk_indices.shape[0]
|
||||
pad_t = upper_bound_chunks * chunk_size
|
||||
total_chunks_ptr = chunk_offsets[-1:]
|
||||
|
||||
g_cu = torch.empty_like(g, dtype=torch.float32)
|
||||
u = q.new_empty(pad_t, num_v_heads, V_dim)
|
||||
w = q.new_empty(pad_t, num_v_heads, K_dim)
|
||||
|
||||
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
|
||||
kkt_inv_uw_cutedsl(
|
||||
k,
|
||||
v,
|
||||
u,
|
||||
w,
|
||||
g,
|
||||
beta,
|
||||
g_cu,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks_ptr,
|
||||
num_sms=num_sms,
|
||||
)
|
||||
|
||||
h = k.new_empty(upper_bound_chunks, num_v_heads, V_dim, K_dim)
|
||||
v_new = q.new_empty(pad_t, num_v_heads, V_dim)
|
||||
final_state = torch.empty_like(initial_state)
|
||||
h_cutedsl(
|
||||
k,
|
||||
u,
|
||||
w,
|
||||
v_new,
|
||||
g_cu,
|
||||
h,
|
||||
initial_state,
|
||||
final_state,
|
||||
cu_seqlens,
|
||||
chunk_offsets,
|
||||
)
|
||||
|
||||
output = core_attn_out if core_attn_out is not None else torch.empty_like(v)
|
||||
scale = K_dim**-0.5
|
||||
o_cutedsl(
|
||||
q,
|
||||
k,
|
||||
v_new,
|
||||
h,
|
||||
g_cu,
|
||||
output,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks_ptr,
|
||||
scale,
|
||||
num_sms=num_sms,
|
||||
)
|
||||
return output.unsqueeze(0), final_state
|
||||
|
||||
|
||||
__all__ = [
|
||||
"chunk_gated_delta_rule_cutedsl",
|
||||
"prepare_metadata_cutedsl",
|
||||
]
|
||||
@@ -1,751 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
|
||||
from cutlass.cute.nvgpu import cpasync, warp
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import (
|
||||
EVICT_FIRST,
|
||||
_tcgen05,
|
||||
cvt,
|
||||
fence_before_tma_store,
|
||||
simple_tma_copy,
|
||||
)
|
||||
|
||||
|
||||
class Sm100ChunkHKernel:
|
||||
"""For each sequence, compute the chunk recurrent update.
|
||||
|
||||
The input V tile is the U output from the KKT/UW kernel. For each chunk:
|
||||
V_new = U - W @ H.T
|
||||
(we actually do V_new.T = U.T - H @ W.T instead)
|
||||
|
||||
H_scaled = H * exp(g_last)
|
||||
V_scaled = V_new * exp(g_last - g)
|
||||
H_new = H_scaled + V_scaled.T @ K
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
H: int,
|
||||
Hv: int,
|
||||
K_dim: int,
|
||||
V_dim: int,
|
||||
h_dtype: cutlass.Numeric = Float32,
|
||||
BT: int = 64,
|
||||
num_stages: int = 2,
|
||||
) -> None:
|
||||
assert Hv % H == 0
|
||||
assert K_dim == V_dim == 128
|
||||
assert BT == 64
|
||||
self.H = H
|
||||
self.Hv = Hv
|
||||
self.K_dim = K_dim
|
||||
self.V_dim = V_dim
|
||||
self.h_dtype = h_dtype
|
||||
self.BT = BT
|
||||
self.num_stages = num_stages
|
||||
self.num_warps = 10
|
||||
|
||||
@cute.jit
|
||||
def _make_bf16_tma_args(
|
||||
self,
|
||||
tensor: cute.Tensor,
|
||||
dim: cutlass.Constexpr[int],
|
||||
op: cpasync.TmaCopyOp,
|
||||
stages: cutlass.Constexpr[int],
|
||||
):
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
slayout = cute.make_layout(
|
||||
(self.BT, 1, (64, dim // 64), stages),
|
||||
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
return cpasync.make_tiled_tma_atom(
|
||||
op,
|
||||
cute.logical_divide(tensor, (None, None, 64)),
|
||||
slayout,
|
||||
cta_tiler=(self.BT, 1, dim),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _make_h_tma_args(self, tensor: cute.Tensor, op: cpasync.TmaCopyOp):
|
||||
# number of elements to fill 128B
|
||||
num_elems = 128 // (tensor.element_type.width // 8)
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
slayout = cute.make_layout(
|
||||
(1, 1, self.V_dim, (num_elems, self.K_dim // num_elems)),
|
||||
stride=(0, 0, num_elems, (1, self.V_dim * num_elems)),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
return cpasync.make_tiled_tma_atom(
|
||||
op,
|
||||
cute.logical_divide(tensor, (None, None, None, num_elems)),
|
||||
slayout,
|
||||
cta_tiler=(1, 1, self.V_dim, self.K_dim),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
K: cute.Tensor,
|
||||
V: cute.Tensor,
|
||||
W: cute.Tensor,
|
||||
V_new: cute.Tensor,
|
||||
g_cu: cute.Tensor,
|
||||
h: cute.Tensor,
|
||||
h0: cute.Tensor,
|
||||
ht: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_offsets: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
|
||||
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
|
||||
|
||||
K_tma = self._make_bf16_tma_args(K, self.K_dim, tma_g2s, self.num_stages)
|
||||
V_tma = self._make_bf16_tma_args(V, self.V_dim, tma_g2s, self.num_stages)
|
||||
W_tma = self._make_bf16_tma_args(W, self.K_dim, tma_g2s, self.num_stages)
|
||||
V_new_tma = self._make_bf16_tma_args(V_new, self.V_dim, tma_s2g, 1)
|
||||
H0_tma = self._make_h_tma_args(h0, tma_g2s)
|
||||
HT_tma = self._make_h_tma_args(ht, tma_s2g)
|
||||
H_tma = self._make_h_tma_args(h, tma_s2g)
|
||||
|
||||
grid = (self.Hv, h0.shape[0], 1)
|
||||
block = (self.num_warps * 32, 1, 1)
|
||||
self.kernel(
|
||||
K_tma,
|
||||
V_tma,
|
||||
W_tma,
|
||||
V_new_tma,
|
||||
H0_tma,
|
||||
HT_tma,
|
||||
H_tma,
|
||||
g_cu,
|
||||
cu_seqlens,
|
||||
chunk_offsets,
|
||||
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
K_tma: cpasync.TmaInfo,
|
||||
V_tma: cpasync.TmaInfo,
|
||||
W_tma: cpasync.TmaInfo,
|
||||
V_new_tma: cpasync.TmaInfo,
|
||||
H0_tma: cpasync.TmaInfo,
|
||||
HT_tma: cpasync.TmaInfo,
|
||||
H_tma: cpasync.TmaInfo,
|
||||
g_cu: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_offsets: cute.Tensor,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
head_id, seq_id, _ = cute.arch.block_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
|
||||
BT = self.BT
|
||||
V_dim = self.V_dim
|
||||
K_dim = self.K_dim
|
||||
num_stages = self.num_stages
|
||||
is_f32 = self.h_dtype == Float32
|
||||
|
||||
def allocate_tensor(smem, dtype, layout):
|
||||
return smem.allocate_tensor(
|
||||
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
|
||||
)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
|
||||
# remove size=1 modes
|
||||
sW = allocate_tensor(smem, BFloat16, W_tma.smem_layout)[None, 0, None, None]
|
||||
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
|
||||
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
|
||||
sH0 = allocate_tensor(smem, self.h_dtype, H0_tma.smem_layout)[0, 0, None, None]
|
||||
sH = allocate_tensor(smem, BFloat16, H_tma.smem_layout)[0, 0, None, None]
|
||||
sV_new = allocate_tensor(smem, BFloat16, V_new_tma.smem_layout)[
|
||||
None, 0, None, 0
|
||||
]
|
||||
|
||||
s_v_scale = smem.allocate_array(Float32, BT)
|
||||
tma_mbar = smem.allocate_array(Int64, num_stages)
|
||||
wh_in_mbar = smem.allocate_array(Int64, num_stages)
|
||||
wh_done_mbar = smem.allocate_array(Int64, num_stages)
|
||||
vk_in_mbar = smem.allocate_array(Int64, num_stages)
|
||||
vk_done_mbar = smem.allocate_array(Int64, num_stages)
|
||||
h0_mbar = smem.allocate_array(Int64, 1)
|
||||
taddr = smem.allocate(Int32, 4)
|
||||
|
||||
wh_tmem = 0
|
||||
vk_tmem = wh_tmem + BT
|
||||
h_tmem_base = vk_tmem + K_dim
|
||||
v_tmem_base = h_tmem_base + K_dim // 2
|
||||
|
||||
if warp_id == 0:
|
||||
with cute.arch.elect_one():
|
||||
for i in cutlass.range_constexpr(num_stages):
|
||||
cute.arch.mbarrier_init(tma_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(wh_in_mbar + i, 256)
|
||||
cute.arch.mbarrier_init(wh_done_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(vk_in_mbar + i, 256)
|
||||
cute.arch.mbarrier_init(vk_done_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(h0_mbar, 1)
|
||||
cute.arch.mbarrier_init_fence()
|
||||
elif warp_id == 1:
|
||||
cpasync.prefetch_descriptor(H0_tma.atom)
|
||||
cpasync.prefetch_descriptor(W_tma.atom)
|
||||
cpasync.prefetch_descriptor(V_tma.atom)
|
||||
cpasync.prefetch_descriptor(K_tma.atom)
|
||||
cpasync.prefetch_descriptor(HT_tma.atom)
|
||||
cpasync.prefetch_descriptor(H_tma.atom)
|
||||
cpasync.prefetch_descriptor(V_new_tma.atom)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
bos = cu_seqlens[seq_id]
|
||||
eos = cu_seqlens[seq_id + 1]
|
||||
seqlen = eos - bos
|
||||
num_chunks = cute.ceil_div(seqlen, BT)
|
||||
|
||||
if warp_id == 9:
|
||||
# TMA warp
|
||||
stage_id = 0
|
||||
parity = 1
|
||||
|
||||
k_head_id = head_id // (self.Hv // self.H)
|
||||
chunk_offset = chunk_offsets[seq_id]
|
||||
|
||||
# load H0
|
||||
with cute.arch.elect_one():
|
||||
H0_size = V_dim * K_dim * self.h_dtype.width // 8
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(h0_mbar, H0_size)
|
||||
simple_tma_copy(
|
||||
H0_tma.atom,
|
||||
H0_tma.tma_tensor[seq_id, head_id, None, None],
|
||||
sH0,
|
||||
h0_mbar,
|
||||
)
|
||||
|
||||
# shape: ((BT, num_BT_tiles), (64, 2))
|
||||
gW_tiles = cute.logical_divide(
|
||||
W_tma.tma_tensor[None, head_id, None], (BT, None)
|
||||
)
|
||||
gV_tiles = cute.logical_divide(
|
||||
V_tma.tma_tensor[None, head_id, None], (BT, None)
|
||||
)
|
||||
gK_tiles = cute.logical_divide(
|
||||
cute.domain_offset((bos, 0), K_tma.tma_tensor[None, k_head_id, None]),
|
||||
(BT, None),
|
||||
)
|
||||
|
||||
for chunk_id in range(num_chunks):
|
||||
mbar = tma_mbar + stage_id
|
||||
gW = gW_tiles[(None, chunk_offset + chunk_id), None]
|
||||
gV = gV_tiles[(None, chunk_offset + chunk_id), None]
|
||||
gK = gK_tiles[(None, chunk_id), None]
|
||||
|
||||
# wait for MMA to release the buffer
|
||||
cute.arch.mbarrier_wait(vk_done_mbar + stage_id, parity)
|
||||
|
||||
# load W, V (i.e. U), and K
|
||||
with cute.arch.elect_one():
|
||||
STAGE_SIZE = BT * (K_dim + V_dim + K_dim) * 2
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
|
||||
simple_tma_copy(
|
||||
W_tma.atom, gW, sW[None, None, stage_id], mbar, EVICT_FIRST
|
||||
)
|
||||
simple_tma_copy(
|
||||
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
|
||||
)
|
||||
simple_tma_copy(K_tma.atom, gK, sK[None, None, stage_id], mbar)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id == 8:
|
||||
# MMA warp
|
||||
_tcgen05.alloc(taddr)
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
wh_idesc = _tcgen05.make_bf16_idesc(V_dim, BT, negate_A=True)
|
||||
vk_idesc = _tcgen05.make_bf16_idesc(V_dim, K_dim, transpose_B=True)
|
||||
|
||||
# LBO=BT*128 is ignored for K-major
|
||||
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
|
||||
|
||||
# when using BF16 state, H is read from smem for the 1st iteration
|
||||
# variable names in this conditional branch can't be the same as those
|
||||
# in the mainloop below due to CuteDSL restrictions.
|
||||
if cutlass.const_expr(not is_f32):
|
||||
##### 1st MMA: V_new.T = V.T - H @ W.T #####
|
||||
Haddr0 = sH0[None, None].iterator.toint()
|
||||
Waddr0 = sW[None, None, stage_id].iterator.toint()
|
||||
hdesc0_base = sdesc_template | (Haddr0 >> 4)
|
||||
wdesc0_base = sdesc_template | (Waddr0 >> 4)
|
||||
|
||||
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
|
||||
cute.arch.mbarrier_wait(wh_in_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(K_dim // 64):
|
||||
for j in cutlass.range_constexpr(64 // 16):
|
||||
hdesc0 = hdesc0_base | ((i * V_dim * 128 + j * 32) >> 4)
|
||||
wdesc0 = wdesc0_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
_tcgen05.mma_f16(wh_tmem, hdesc0, wdesc0, wh_idesc, True)
|
||||
_tcgen05.commit(wh_done_mbar + stage_id)
|
||||
|
||||
##### 2nd MMA: H_new = H + V_new.T @ K #####
|
||||
Kaddr0 = sK[None, None, stage_id].iterator.toint()
|
||||
kdesc0_base = sdesc_template | (Kaddr0 >> 4)
|
||||
|
||||
cute.arch.mbarrier_wait(vk_in_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for k in cutlass.range_constexpr(BT // 16):
|
||||
vtmem0 = v_tmem_base + k * 8
|
||||
kdesc0 = kdesc0_base | ((k * 16 * 128) >> 4)
|
||||
_tcgen05.mma_ts_f16(vk_tmem, vtmem0, kdesc0, vk_idesc, True)
|
||||
_tcgen05.commit(vk_done_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
num_iters = num_chunks - int(not is_f32)
|
||||
for _ in range(num_iters):
|
||||
##### 1st MMA: V_new.T = V.T - H @ W.T #####
|
||||
Waddr = sW[None, None, stage_id].iterator.toint()
|
||||
wdesc_base = sdesc_template | (Waddr >> 4)
|
||||
|
||||
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
|
||||
cute.arch.mbarrier_wait(wh_in_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(K_dim // 64):
|
||||
for j in cutlass.range_constexpr(64 // 16):
|
||||
htmem = h_tmem_base + i * 32 + j * 8
|
||||
wdesc = wdesc_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
_tcgen05.mma_ts_f16(wh_tmem, htmem, wdesc, wh_idesc, True)
|
||||
_tcgen05.commit(wh_done_mbar + stage_id)
|
||||
|
||||
##### 2nd MMA: H_new = H + V_new.T @ K #####
|
||||
Kaddr = sK[None, None, stage_id].iterator.toint()
|
||||
kdesc_base = sdesc_template | (Kaddr >> 4)
|
||||
|
||||
cute.arch.mbarrier_wait(vk_in_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for k in cutlass.range_constexpr(BT // 16):
|
||||
vtmem = v_tmem_base + k * 8
|
||||
kdesc = kdesc_base | ((k * 16 * 128) >> 4)
|
||||
_tcgen05.mma_ts_f16(vk_tmem, vtmem, kdesc, vk_idesc, True)
|
||||
_tcgen05.commit(vk_done_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id >= 4:
|
||||
# H warps
|
||||
tid_ = tid % 128
|
||||
warp_id_ = warp_id % 4
|
||||
chunk_offset = chunk_offsets[seq_id]
|
||||
|
||||
stage_id = 0
|
||||
vk_stage_id = 0
|
||||
vk_parity = 0
|
||||
|
||||
op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_16B = cute.make_copy_atom(op, Float32, num_bits_per_copy=128)
|
||||
|
||||
##### chunk_id = 0 #####
|
||||
if True:
|
||||
chunk_id = 0
|
||||
end_t = min(bos + (chunk_id + 1) * BT, eos)
|
||||
last_idx = end_t - 1
|
||||
h_scale = cute.math.exp(g_cu[last_idx, head_id], fastmath=True)
|
||||
|
||||
# for 1st chunk, wait for H0 transfer from gmem
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(h0_mbar, 0)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
|
||||
# when H0 is FP32, we need to pack it to BF16
|
||||
# also store to smem for TMA store later.
|
||||
if cutlass.const_expr(is_f32):
|
||||
for i in cutlass.range_constexpr(K_dim // 32):
|
||||
# H0 smem layout: (V_dim, (32, K_dim/32))
|
||||
h_f32 = cute.make_rmem_tensor(32, Float32)
|
||||
cute.copy(cp_16B, sH0[tid_, (None, i)], h_f32)
|
||||
|
||||
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
|
||||
h_bf16.store(h_f32.load().to(BFloat16))
|
||||
_tcgen05.st(
|
||||
warp_id_ * 32, h_tmem_base + i * 16, "32x32b", 16, h_bf16
|
||||
)
|
||||
|
||||
# H smem layout: (V_dim, (64, K_dim/64))
|
||||
dst = cute.local_tile(sH[tid_, None], (32,), (i,))
|
||||
cute.copy(cp_16B, h_bf16, dst)
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
|
||||
|
||||
# scale H for 2nd MMA
|
||||
for i in cutlass.range_constexpr(K_dim // 32):
|
||||
h_f32 = cute.make_rmem_tensor(32, Float32)
|
||||
|
||||
if cutlass.const_expr(is_f32):
|
||||
cute.copy(cp_16B, sH0[tid_, (None, i)], h_f32)
|
||||
|
||||
else:
|
||||
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
|
||||
sH_src = cute.local_tile(sH0[tid_, None], (32,), (i,))
|
||||
cute.copy(cp_16B, sH_src, h_bf16)
|
||||
h_f32.store(
|
||||
cvt.bf16x2_to_fp32x2(
|
||||
cute.recast_tensor(h_bf16, Uint32)
|
||||
).load()
|
||||
)
|
||||
|
||||
for j in cutlass.range(32, vectorize=True):
|
||||
h_f32[j] *= h_scale
|
||||
_tcgen05.st(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32, h_f32)
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
|
||||
|
||||
# for BF16 H0, we issue TMA store from H0 smem
|
||||
# for FP32 H0, we issue TMA store from H smem (after packing)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id_ == 3:
|
||||
h_src = sH if cutlass.const_expr(is_f32) else sH0
|
||||
h_dst = H_tma.tma_tensor[
|
||||
chunk_offset + chunk_id, head_id, None, None
|
||||
]
|
||||
simple_tma_copy(H_tma.atom, h_src, h_dst)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
|
||||
# When H0 is BF16, and there is only 1 chunk, storing
|
||||
# the final state to sH0 can race before this store
|
||||
# has finished. hence, we need to wait here.
|
||||
if cutlass.const_expr(not is_f32):
|
||||
cute.arch.cp_async_bulk_wait_group(0, read=True)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
|
||||
##### subsequent chunks #####
|
||||
for chunk_id in range(1, num_chunks):
|
||||
end_t = min(bos + (chunk_id + 1) * BT, eos)
|
||||
last_idx = end_t - 1
|
||||
h_scale = cute.math.exp(g_cu[last_idx, head_id], fastmath=True)
|
||||
|
||||
# wait for H from previous vk MMA
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(vk_done_mbar + vk_stage_id, vk_parity)
|
||||
vk_stage_id = (vk_stage_id + 1) % num_stages
|
||||
if vk_stage_id == 0:
|
||||
vk_parity ^= 1
|
||||
elif warp_id_ == 3:
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_wait_group(0, read=True)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
# load FP32 H from tmem, convert to BF16, store to tmem for 1st MMA,
|
||||
# store to smem for TMA store later.
|
||||
for i in cutlass.range_constexpr(K_dim // 32):
|
||||
h_f32 = _tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32)
|
||||
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
|
||||
h_bf16.store(h_f32.to(BFloat16))
|
||||
_tcgen05.st(
|
||||
warp_id_ * 32, h_tmem_base + i * 16, "32x32b", 16, h_bf16
|
||||
)
|
||||
|
||||
# H smem layout: (V_dim, (64, K_dim/64))
|
||||
dst = cute.local_tile(sH[tid_, None], (32,), (i,))
|
||||
cute.copy(cp_16B, h_bf16, dst)
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
|
||||
|
||||
# scale H for 2nd MMA
|
||||
for i in cutlass.range_constexpr(K_dim // 32):
|
||||
h_f32 = cute.make_rmem_tensor(32, Float32)
|
||||
h_f32.store(
|
||||
_tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32)
|
||||
)
|
||||
for j in cutlass.range(32, vectorize=True):
|
||||
h_f32[j] *= h_scale
|
||||
_tcgen05.st(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32, h_f32)
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
|
||||
|
||||
# issue TMA store for O kernel
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id_ == 3:
|
||||
h_dst = H_tma.tma_tensor[
|
||||
chunk_offset + chunk_id, head_id, None, None
|
||||
]
|
||||
simple_tma_copy(H_tma.atom, sH, h_dst)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
|
||||
# handle final state. reuse H0 smem.
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(vk_done_mbar + vk_stage_id, vk_parity)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(K_dim // 32):
|
||||
h_f32 = cute.make_rmem_tensor(32, Float32)
|
||||
h_f32.store(_tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32))
|
||||
|
||||
if cutlass.const_expr(is_f32):
|
||||
cute.copy(cp_16B, h_f32, sH0[tid_, (None, i)])
|
||||
|
||||
else:
|
||||
h_bf16 = cute.make_rmem_tensor(32, BFloat16)
|
||||
h_bf16.store(h_f32.load().to(BFloat16))
|
||||
sH0_dst = cute.local_tile(sH0[tid_, None], (32,), (i,))
|
||||
cute.copy(cp_16B, h_bf16, sH0_dst)
|
||||
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
|
||||
if warp_id_ == 0:
|
||||
ht_dst = HT_tma.tma_tensor[seq_id, head_id, None, None]
|
||||
simple_tma_copy(HT_tma.atom, sH0, ht_dst)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
if warp_id_ == 1:
|
||||
_tcgen05.dealloc()
|
||||
|
||||
else:
|
||||
# V warps
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
chunk_offset = chunk_offsets[seq_id]
|
||||
|
||||
ldsm_trans_op = warp.LdMatrix8x8x16bOp(num_matrices=4, transpose=True)
|
||||
stsm_trans_op = warp.StMatrix8x8x16bOp(num_matrices=4, transpose=True)
|
||||
ldsm_trans_atom = cute.make_copy_atom(ldsm_trans_op, BFloat16)
|
||||
stsm_trans_atom = cute.make_copy_atom(stsm_trans_op, BFloat16)
|
||||
|
||||
# ((BT, num_BT_tiles), V_dim)
|
||||
gV_new_tiles = cute.logical_divide(
|
||||
V_new_tma.tma_tensor[None, head_id, None], (BT, None)
|
||||
)
|
||||
|
||||
# sV shape: [BT, (64, V_dim/64), num_stages]
|
||||
# sV_view shape: [BT, (8, (8,2)), num_stages]
|
||||
sV_view = cute.logical_divide(sV, (None, 8, None))
|
||||
sV_new_view = cute.logical_divide(sV_new, (None, 8))
|
||||
|
||||
# [BT, 8, num_stages]
|
||||
s_col = warp_id * 4 + (lane_id // 8)
|
||||
sV_view = sV_view[None, (None, s_col), None]
|
||||
sV_new_view = sV_new_view[None, (None, s_col)]
|
||||
|
||||
for chunk_id in range(num_chunks):
|
||||
# wait for V to arrive
|
||||
if warp_id == 0:
|
||||
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=128)
|
||||
|
||||
# unpack V BF16->FP32, then store to tmem for 1st MMA
|
||||
# V smem layout: [BT, (64, V_dim/64)] / [BT, V_dim]
|
||||
# each iteration, CTA loads [8, V_dim] tile
|
||||
# (warp loads [8, 32] tile)
|
||||
for i in cutlass.range_constexpr(BT // 8):
|
||||
s_row = i * 8 + (lane_id % 8)
|
||||
v_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
cute.copy(ldsm_trans_atom, sV_view[s_row, None, stage_id], v_bf16)
|
||||
v_fp32 = cvt.bf16x2_to_fp32x2(cute.recast_tensor(v_bf16, Uint32))
|
||||
v_fp32 = cute.logical_divide(v_fp32, 4) # (4, 2)
|
||||
|
||||
tcol = wh_tmem + i * 8
|
||||
_tcgen05.st(warp_id * 32 + 0, tcol, "16x256b", 1, v_fp32[None, 0])
|
||||
_tcgen05.st(warp_id * 32 + 16, tcol, "16x256b", 1, v_fp32[None, 1])
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(wh_in_mbar + stage_id)
|
||||
|
||||
# load g_cu for scaling
|
||||
if tid < BT:
|
||||
end_t = min(bos + (chunk_id + 1) * BT, eos)
|
||||
last_idx = end_t - 1
|
||||
t = bos + chunk_id * BT + tid
|
||||
val = Float32(0.0)
|
||||
if t < eos:
|
||||
val = cute.math.exp(
|
||||
g_cu[last_idx, head_id] - g_cu[t, head_id],
|
||||
fastmath=True,
|
||||
)
|
||||
s_v_scale[tid] = val
|
||||
|
||||
# wait for 1st MMA to finish
|
||||
if warp_id == 2:
|
||||
cute.arch.mbarrier_wait(wh_done_mbar + stage_id, parity)
|
||||
elif warp_id == 3:
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_wait_group(0, read=True)
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(BT // 8):
|
||||
v_new = cute.make_rmem_tensor((4, 2), Float32)
|
||||
tcol = wh_tmem + i * 8
|
||||
v_new[None, 0].store(
|
||||
_tcgen05.ld(warp_id * 32 + 0, tcol, "16x256b", 1)
|
||||
)
|
||||
v_new[None, 1].store(
|
||||
_tcgen05.ld(warp_id * 32 + 16, tcol, "16x256b", 1)
|
||||
)
|
||||
v_new_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
v_new_bf16.store(v_new.load().to(BFloat16))
|
||||
|
||||
# scale V_new for 2nd MMA
|
||||
scale0 = s_v_scale[i * 8 + (lane_id % 4) * 2 + 0]
|
||||
scale1 = s_v_scale[i * 8 + (lane_id % 4) * 2 + 1]
|
||||
v_scaled = cute.make_rmem_tensor(8, Float32)
|
||||
for k in cutlass.range_constexpr(4):
|
||||
v_scaled[k * 2] = v_new[k * 2] * scale0
|
||||
v_scaled[k * 2 + 1] = v_new[k * 2 + 1] * scale1
|
||||
v_scaled_bf16 = v_scaled.load().to(BFloat16).reshape((4, 2))
|
||||
|
||||
# store V_new BF16 for O kernel
|
||||
s_row = i * 8 + (lane_id % 8)
|
||||
cute.copy(stsm_trans_atom, v_new_bf16, sV_new_view[s_row, None])
|
||||
|
||||
# store to tmem
|
||||
tcol = v_tmem_base + i * 4
|
||||
_tcgen05.st(
|
||||
warp_id * 32 + 0, tcol, "16x128b", 1, v_scaled_bf16[None, 0]
|
||||
)
|
||||
_tcgen05.st(
|
||||
warp_id * 32 + 16, tcol, "16x128b", 1, v_scaled_bf16[None, 1]
|
||||
)
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(vk_in_mbar + stage_id)
|
||||
|
||||
# issue TMA store for V_new
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id == 3:
|
||||
gV = gV_new_tiles[(None, chunk_offset + chunk_id), None]
|
||||
simple_tma_copy(V_new_tma.atom, sV_new, gV)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(
|
||||
H: int,
|
||||
Hv: int,
|
||||
K_dim: int,
|
||||
V_dim: int,
|
||||
h_dtype: cutlass.Numeric = Float32,
|
||||
BT: int = 64,
|
||||
num_stages: int = 2,
|
||||
):
|
||||
total_t = cute.sym_int()
|
||||
pad_t = cute.sym_int()
|
||||
total_chunks_n = cute.sym_int()
|
||||
num_sequences = cute.sym_int()
|
||||
cu_entries = cute.sym_int()
|
||||
|
||||
K = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
|
||||
V = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
|
||||
W = make_fake_tensor(BFloat16, (pad_t, Hv, K_dim), divisibility=16)
|
||||
V_new = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
|
||||
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
|
||||
h = make_fake_tensor(
|
||||
BFloat16, (total_chunks_n, Hv, V_dim, K_dim), divisibility=16
|
||||
)
|
||||
h0 = make_fake_tensor(
|
||||
h_dtype, (num_sequences, Hv, V_dim, K_dim), divisibility=16
|
||||
)
|
||||
ht = make_fake_tensor(
|
||||
h_dtype, (num_sequences, Hv, V_dim, K_dim), divisibility=16
|
||||
)
|
||||
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
|
||||
chunk_offsets = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
|
||||
|
||||
kernel = Sm100ChunkHKernel(H, Hv, K_dim, V_dim, h_dtype, BT, num_stages)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
K,
|
||||
V,
|
||||
W,
|
||||
V_new,
|
||||
g_cu,
|
||||
h,
|
||||
h0,
|
||||
ht,
|
||||
cu_seqlens,
|
||||
chunk_offsets,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def h_cutedsl(
|
||||
K: torch.Tensor,
|
||||
V: torch.Tensor,
|
||||
W: torch.Tensor,
|
||||
V_new: torch.Tensor,
|
||||
g_cu: torch.Tensor,
|
||||
h: torch.Tensor,
|
||||
h0: torch.Tensor,
|
||||
ht: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
chunk_offsets: torch.Tensor,
|
||||
BT: int = 64,
|
||||
num_stages: int = 2,
|
||||
) -> None:
|
||||
"""Compute H/V_new with the same argument order as the CUDA wrapper."""
|
||||
_, H, K_dim = K.shape
|
||||
_, Hv, V_dim = V.shape
|
||||
h_dtype = {
|
||||
torch.bfloat16: BFloat16,
|
||||
torch.float32: Float32,
|
||||
}[h0.dtype]
|
||||
Sm100ChunkHKernel.compile(H, Hv, K_dim, V_dim, h_dtype, BT, num_stages)(
|
||||
K,
|
||||
V,
|
||||
W,
|
||||
V_new,
|
||||
g_cu,
|
||||
h,
|
||||
h0,
|
||||
ht,
|
||||
cu_seqlens,
|
||||
chunk_offsets,
|
||||
)
|
||||
|
||||
|
||||
h_v2b_cutedsl = h_cutedsl
|
||||
@@ -1,925 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
|
||||
from cutlass.cute.nvgpu import cpasync, warp
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import (
|
||||
EVICT_FIRST,
|
||||
_bf16x2_neg,
|
||||
_bf16x2_sub,
|
||||
_tcgen05,
|
||||
cvt,
|
||||
fence_before_tma_store,
|
||||
mma_sync,
|
||||
simple_tma_copy,
|
||||
)
|
||||
|
||||
|
||||
class Sm100ChunkUWKernel:
|
||||
"""Compute per-chunk KKT inverse preprocessing and U/W tiles.
|
||||
|
||||
Gamma[i,j] = exp(g_cu[i] - g_cu[j])
|
||||
A = strictLower(beta * (K @ K.T) * Gamma)
|
||||
Ai = inverse(I + A)
|
||||
U = (Ai * beta) @ V
|
||||
W = (Ai * beta * exp(g_cu)) @ K
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
H: int,
|
||||
Hv: int,
|
||||
K_dim: int,
|
||||
V_dim: int,
|
||||
num_stages: int = 2,
|
||||
) -> None:
|
||||
assert Hv % H == 0
|
||||
assert K_dim == V_dim == 128
|
||||
self.H = H
|
||||
self.Hv = Hv
|
||||
self.K_dim = K_dim
|
||||
self.V_dim = V_dim
|
||||
self.num_stages = num_stages
|
||||
|
||||
# hard-code
|
||||
self.BT = 64
|
||||
self.num_warps = 4 + 4 + 4
|
||||
|
||||
@cute.jit
|
||||
def _make_tma_args(
|
||||
self,
|
||||
tensor: cute.Tensor,
|
||||
dim: cutlass.Constexpr[int],
|
||||
num_stages: int,
|
||||
op: cpasync.TmaCopyOp,
|
||||
):
|
||||
# logical layout: [BT, dim]
|
||||
# permute for TMA: [dim/64, BT, 64] with swizzling
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
slayout = cute.make_layout(
|
||||
(self.BT, 1, (64, dim // 64), num_stages),
|
||||
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
|
||||
# we need to convert gmem layout to (T, H, (64, D/64)) for make_tiled_tma_atom()
|
||||
# to emit a single 4D TMA. otherwise, it will emit (D/64)x 3D TMA.
|
||||
return cpasync.make_tiled_tma_atom(
|
||||
op,
|
||||
cute.logical_divide(tensor, (None, None, 64)),
|
||||
slayout,
|
||||
cta_tiler=(self.BT, 1, dim),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
K: cute.Tensor,
|
||||
V: cute.Tensor,
|
||||
U: cute.Tensor,
|
||||
W: cute.Tensor,
|
||||
g: cute.Tensor,
|
||||
beta: cute.Tensor,
|
||||
g_cu: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
total_chunks: cute.Tensor,
|
||||
num_sms: Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
|
||||
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
|
||||
|
||||
K_tma = self._make_tma_args(K, self.K_dim, self.num_stages, tma_g2s)
|
||||
V_tma = self._make_tma_args(V, self.V_dim, self.num_stages, tma_g2s)
|
||||
U_tma = self._make_tma_args(U, self.V_dim, 1, tma_s2g)
|
||||
W_tma = self._make_tma_args(W, self.K_dim, 1, tma_s2g)
|
||||
|
||||
grid = (num_sms // self.Hv, self.Hv, 1)
|
||||
block = (self.num_warps * 32, 1, 1)
|
||||
self.kernel(
|
||||
K_tma,
|
||||
V_tma,
|
||||
U_tma,
|
||||
W_tma,
|
||||
g,
|
||||
beta,
|
||||
g_cu,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
K_tma: cpasync.TmaInfo,
|
||||
V_tma: cpasync.TmaInfo,
|
||||
U_tma: cpasync.TmaInfo,
|
||||
W_tma: cpasync.TmaInfo,
|
||||
g: cute.Tensor,
|
||||
beta: cute.Tensor,
|
||||
g_cu: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
total_chunks: cute.Tensor,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
bid, head_id, _ = cute.arch.block_idx()
|
||||
grid_x, _, _ = cute.arch.grid_dim()
|
||||
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
k_head_id = head_id // (self.Hv // self.H)
|
||||
|
||||
BT = self.BT
|
||||
K_dim = self.K_dim
|
||||
V_dim = self.V_dim
|
||||
num_stages = self.num_stages
|
||||
|
||||
INV_BAR = 1
|
||||
EPI_BAR = 2
|
||||
SCAN_BAR = 3
|
||||
TMEM_ALLOC_BAR = 4
|
||||
|
||||
def allocate_tensor(smem, dtype, layout):
|
||||
return smem.allocate_tensor(
|
||||
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
|
||||
)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
|
||||
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
|
||||
sU = allocate_tensor(smem, BFloat16, U_tma.smem_layout)[None, 0, None, 0]
|
||||
sW = allocate_tensor(smem, BFloat16, W_tma.smem_layout)[None, 0, None, 0]
|
||||
|
||||
sA_ptr = smem.allocate_array(BFloat16, BT * BT, byte_alignment=16)
|
||||
sAi_ptr = smem.allocate_array(BFloat16, BT * BT, byte_alignment=16)
|
||||
|
||||
s_beta = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
|
||||
s_g_cu = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
|
||||
s_beta_g = smem.allocate_tensor(Float32, cute.make_layout((BT, num_stages)))
|
||||
|
||||
tma_mbar = smem.allocate_array(Int64, num_stages)
|
||||
prep_mbar = smem.allocate_array(Int64, num_stages)
|
||||
mma_kkt_mbar = smem.allocate_array(Int64, num_stages)
|
||||
inv_mbar = smem.allocate_array(Int64, num_stages)
|
||||
mma_u_mbar = smem.allocate_array(Int64, num_stages)
|
||||
mma_w_mbar = smem.allocate_array(Int64, num_stages)
|
||||
epi_mbar = smem.allocate_array(Int64, num_stages)
|
||||
taddr = smem.allocate(Int32, 4)
|
||||
|
||||
kkt_tmem = 0
|
||||
U_tmem_base = kkt_tmem + BT
|
||||
Ab_tmem_base = U_tmem_base + V_dim * num_stages
|
||||
assert Ab_tmem_base + (BT // 2) <= 512
|
||||
|
||||
# prepare ldmatrix/stmatrix ops
|
||||
ldsm_op = warp.LdMatrix8x8x16bOp(num_matrices=4)
|
||||
stsm_op = warp.StMatrix8x8x16bOp(num_matrices=4)
|
||||
ldsm_trans_op = warp.LdMatrix8x8x16bOp(num_matrices=4, transpose=True)
|
||||
ldsm_atom = cute.make_copy_atom(ldsm_op, BFloat16)
|
||||
stsm_atom = cute.make_copy_atom(stsm_op, BFloat16)
|
||||
ldsm_trans_atom = cute.make_copy_atom(ldsm_trans_op, BFloat16)
|
||||
|
||||
if warp_id == 0:
|
||||
with cute.arch.elect_one():
|
||||
for i in cutlass.range_constexpr(num_stages):
|
||||
cute.arch.mbarrier_init(tma_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(prep_mbar + i, 64)
|
||||
cute.arch.mbarrier_init(mma_kkt_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(inv_mbar + i, 128)
|
||||
cute.arch.mbarrier_init(mma_u_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(mma_w_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(epi_mbar + i, 128)
|
||||
cute.arch.mbarrier_init_fence()
|
||||
elif warp_id == 1:
|
||||
cpasync.prefetch_descriptor(K_tma.atom)
|
||||
cpasync.prefetch_descriptor(V_tma.atom)
|
||||
cpasync.prefetch_descriptor(U_tma.atom)
|
||||
cpasync.prefetch_descriptor(W_tma.atom)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
num_global_chunks = total_chunks[0]
|
||||
if warp_id == 11:
|
||||
# TMA warp
|
||||
stage_id = 0
|
||||
parity = 1
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
|
||||
# since off_t is not a multiple of BT, we need to use
|
||||
# domain_offset() to shift the pointer first.
|
||||
mbar = tma_mbar + stage_id
|
||||
gK = cute.local_tile(
|
||||
cute.domain_offset(
|
||||
(bos, 0), K_tma.tma_tensor[None, k_head_id, None]
|
||||
),
|
||||
tiler=(BT, K_dim),
|
||||
coord=(chunk_id, 0),
|
||||
)
|
||||
gV = cute.local_tile(
|
||||
cute.domain_offset((bos, 0), V_tma.tma_tensor[None, head_id, None]),
|
||||
tiler=(BT, V_dim),
|
||||
coord=(chunk_id, 0),
|
||||
)
|
||||
|
||||
# when UW MMA is done, K and V TMA buffers are released
|
||||
cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity)
|
||||
|
||||
with cute.arch.elect_one():
|
||||
STAGE_SIZE = BT * (K_dim + V_dim) * 2
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
|
||||
simple_tma_copy(K_tma.atom, gK, sK[None, None, stage_id], mbar)
|
||||
simple_tma_copy(
|
||||
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
|
||||
)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id == 10:
|
||||
# MMA warp
|
||||
_tcgen05.alloc(taddr)
|
||||
cute.arch.barrier(barrier_id=TMEM_ALLOC_BAR, number_of_threads=160)
|
||||
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
kkt_idesc = _tcgen05.make_bf16_idesc(BT, BT)
|
||||
u_idesc = _tcgen05.make_bf16_idesc(BT, V_dim, transpose_B=True)
|
||||
w_idesc = _tcgen05.make_bf16_idesc(BT, K_dim, transpose_B=True)
|
||||
|
||||
# LBO=BT*128 is ignored for K-major
|
||||
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
U_tmem = U_tmem_base + V_dim * stage_id
|
||||
W_tmem = U_tmem | (16 << 16)
|
||||
Ab_tmem = Ab_tmem_base
|
||||
Abg_tmem = Ab_tmem_base | (16 << 16)
|
||||
|
||||
##### KKT MMA: KKT = K @ K.T #####
|
||||
kaddr = sK[None, None, stage_id].iterator.toint()
|
||||
kdesc_base = sdesc_template | (kaddr >> 4)
|
||||
|
||||
# wait for TMA data to arrive
|
||||
# kkt tmem is guaranteed to be free as this is issued
|
||||
# after the previous kkt's consumer (inv warps)
|
||||
cute.arch.mbarrier_wait(tma_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(K_dim // 64):
|
||||
for j in cutlass.range_constexpr(64 // 16):
|
||||
kdesc = kdesc_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
enable_d = (i > 0) or (j > 0)
|
||||
_tcgen05.mma_f16(kkt_tmem, kdesc, kdesc, kkt_idesc, enable_d)
|
||||
_tcgen05.commit(mma_kkt_mbar + stage_id)
|
||||
|
||||
##### U/W MMA: U = Ab @ V, W = Abg @ K #####
|
||||
vaddr = sV[None, None, stage_id].iterator.toint()
|
||||
vdesc = sdesc_template | (vaddr >> 4)
|
||||
kdesc = sdesc_template | (kaddr >> 4)
|
||||
|
||||
# wait for epilogue to release tmem buffer
|
||||
cute.arch.mbarrier_wait(epi_mbar + stage_id, parity ^ 1)
|
||||
cute.arch.mbarrier_wait(inv_mbar + stage_id, parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(BT // 16):
|
||||
_tcgen05.mma_ts_f16(W_tmem, Abg_tmem + i * 8, kdesc, w_idesc, i > 0)
|
||||
kdesc += (16 * 128) >> 4
|
||||
_tcgen05.commit(mma_w_mbar + stage_id)
|
||||
|
||||
for i in cutlass.range_constexpr(BT // 16):
|
||||
_tcgen05.mma_ts_f16(U_tmem, Ab_tmem + i * 8, vdesc, u_idesc, i > 0)
|
||||
vdesc += (16 * 128) >> 4
|
||||
_tcgen05.commit(mma_u_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
cute.arch.mbarrier_wait(epi_mbar + stage_id, parity ^ 1)
|
||||
_tcgen05.dealloc()
|
||||
|
||||
elif warp_id >= 8:
|
||||
# dedicated prep warps for beta and gate, consumed by INV warps
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
tid_ = tid % 128
|
||||
warp_id_ = warp_id % 4
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
eos = cu_seqlens[seq_id + 1]
|
||||
off_t = bos + chunk_id * BT
|
||||
t = off_t + tid_
|
||||
|
||||
in_bounds = t < eos
|
||||
beta_val = beta[t, head_id] if in_bounds else Float32(0.0)
|
||||
g_val = g[t, head_id] if in_bounds else Float32(0.0)
|
||||
|
||||
# warp-local prefix scan
|
||||
for i in cutlass.range_constexpr(5):
|
||||
offset = cutlass.const_expr(1 << i)
|
||||
lower = cute.arch.shuffle_sync_up(g_val, offset, mask_and_clamp=0)
|
||||
if lane_id >= offset:
|
||||
g_val += lower
|
||||
|
||||
# Delay the stage-reuse wait until just before touching smem:
|
||||
# global loads and the warp-local scan do not use staged buffers.
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(inv_mbar + stage_id, parity ^ 1)
|
||||
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
|
||||
|
||||
# Store beta and the per-warp scan totals for the cross-warp fixup.
|
||||
s_beta[tid_, stage_id] = beta_val
|
||||
if lane_id == 31:
|
||||
s_g_cu[warp_id_, stage_id] = g_val
|
||||
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
|
||||
|
||||
# Add the sum from the lower prep warp.
|
||||
if warp_id_ == 1:
|
||||
g_val += s_g_cu[0, stage_id]
|
||||
cute.arch.barrier(barrier_id=SCAN_BAR, number_of_threads=BT)
|
||||
|
||||
if in_bounds:
|
||||
g_cu[t, head_id] = g_val
|
||||
|
||||
s_g_cu[tid_, stage_id] = g_val
|
||||
s_beta_g[tid_, stage_id] = beta_val * cute.math.exp(g_val)
|
||||
cute.arch.mbarrier_arrive(prep_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id >= 4:
|
||||
# inv warps
|
||||
tid_ = tid % 128
|
||||
warp_id_ = warp_id % 4
|
||||
|
||||
def store_ab_abg(
|
||||
Ai_f32, s_beta, s_beta_g, warp_id_, lane_id, tile_col, Ab_tmem_base
|
||||
):
|
||||
# compute Ab and Abg from Ai, then store to tmem
|
||||
beta_col = cute.make_rmem_tensor((2, 2), Float32)
|
||||
beta_g_col = cute.make_rmem_tensor((2, 2), Float32)
|
||||
|
||||
for i in cutlass.range_constexpr(2):
|
||||
base = tile_col * 16 + i * 8 + (lane_id % 4) * 2
|
||||
for j in cutlass.range_constexpr(2):
|
||||
beta_col[j, i] = s_beta[base + j]
|
||||
beta_g_col[j, i] = s_beta_g[base + j]
|
||||
|
||||
# without conversion to TensorSSA, cutlass.range(vectorize=True) fails
|
||||
beta_col = beta_col.load()
|
||||
beta_g_col = beta_g_col.load()
|
||||
|
||||
Ab_f32 = cute.make_rmem_tensor(8, Float32)
|
||||
Abg_f32 = cute.make_rmem_tensor(8, Float32)
|
||||
for i in cutlass.range(8, vectorize=True):
|
||||
scale_idx = (i // 4) * 2 + (i % 2)
|
||||
Ab_f32[i] = Ai_f32[i] * beta_col[scale_idx]
|
||||
Abg_f32[i] = Ai_f32[i] * beta_g_col[scale_idx]
|
||||
|
||||
Ab = Ab_f32.load().to(BFloat16)
|
||||
Abg = Abg_f32.load().to(BFloat16)
|
||||
Ab_tmem = Ab_tmem_base + tile_col * 8
|
||||
_tcgen05.st(warp_id_ * 32, Ab_tmem, "16x128b", 2, Ab)
|
||||
_tcgen05.st(warp_id_ * 32 + 16, Ab_tmem, "16x128b", 2, Abg)
|
||||
|
||||
# clear the Ab/Abg tmem buffer once before mainloop
|
||||
# this is to keep the upper triangular tiles zeros
|
||||
cute.arch.barrier(barrier_id=TMEM_ALLOC_BAR, number_of_threads=160)
|
||||
zeros_ = cute.make_rmem_tensor(BT // 2, Float32)
|
||||
zeros_.fill(0.0)
|
||||
_tcgen05.st(warp_id_ * 32, Ab_tmem_base, "32x32b", BT // 2, zeros_)
|
||||
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
# for sA, we can avoid bank conflict without using swizzling because
|
||||
# this is only used as tmp buffer between rmem<->smem, no gmem interactions.
|
||||
# to do so, we can put (8,8) tile contiguous in memory. logically, we are
|
||||
# partitioning (64,64) tile into 4x (16,16) tiles. to make indexing easier
|
||||
# later, we view (16,16) tile as (32,8) tile here.
|
||||
# (4,4) is (row_tile, col_tile).
|
||||
sA_layout = cute.make_layout(((8, 32), (4, 4)))
|
||||
sA = cute.make_tensor(sA_ptr, sA_layout)
|
||||
sAi = cute.make_tensor(sAi_ptr, sA_layout)
|
||||
|
||||
# pre-compute ldmatrix addresses
|
||||
sA_ldsm = sA[(None, lane_id), None]
|
||||
sAi_ldsm = sAi[(None, lane_id), None]
|
||||
|
||||
# init Ai smem buffer with zeros (only the first 48 rows)
|
||||
zeros_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
zeros_bf16.fill(0.0)
|
||||
for i in cutlass.range_constexpr(3):
|
||||
cute.copy(stsm_atom, zeros_bf16, sAi_ldsm[None, (i, warp_id_)])
|
||||
|
||||
# indices for ldmatrix layout later
|
||||
row_indices = cute.make_rmem_tensor((1, 2, 1), Int32)
|
||||
row_indices[0, 0, 0] = warp_id_ * 16 + (lane_id // 4)
|
||||
row_indices[0, 1, 0] = warp_id_ * 16 + (lane_id // 4) + 8
|
||||
row_indices = row_indices.load()
|
||||
|
||||
col_indices = cute.make_rmem_tensor((2, 1, 2), Int32)
|
||||
col_indices[0, 0, 0] = (lane_id % 4) * 2 + 0
|
||||
col_indices[1, 0, 0] = (lane_id % 4) * 2 + 1
|
||||
col_indices[0, 0, 1] = (lane_id % 4) * 2 + 8
|
||||
col_indices[1, 0, 1] = (lane_id % 4) * 2 + 9
|
||||
col_indices = col_indices.load()
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
eos = cu_seqlens[seq_id + 1]
|
||||
off_t = bos + chunk_id * BT
|
||||
|
||||
##### Phase 1: A = strictLower(beta * kkt * Gamma) #####
|
||||
# Ab/Abg share one tmem slot across stages. The MMA warp commits
|
||||
# tcgen05 groups in program order: KKT_i, W_i, U_i, KKT_{i+1}.
|
||||
# Waiting for KKT_i means the previous W/U commits have completed,
|
||||
# so the INV warps can safely overwrite Ab/Abg for this iteration.
|
||||
# Wait for prep warps to publish beta/gate and for KKT MMA.
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(prep_mbar + stage_id, parity)
|
||||
cute.arch.mbarrier_wait(mma_kkt_mbar + stage_id, parity)
|
||||
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
# tmem 16x256b layout / ldmatrix layout
|
||||
beta_row = cute.make_rmem_tensor(2, Float32)
|
||||
g_cu_row = cute.make_rmem_tensor(2, Float32)
|
||||
for i in cutlass.range_constexpr(2):
|
||||
idx = warp_id_ * 16 + i * 8 + (lane_id // 4)
|
||||
beta_row[i] = s_beta[idx, stage_id]
|
||||
g_cu_row[i] = s_g_cu[idx, stage_id]
|
||||
|
||||
# mode0 is 8 rows together
|
||||
# mode1 is top and bottom 8 rows
|
||||
# mode2 is groups of 16 rows
|
||||
beta_row = beta_row.load().reshape((1, 2, 1))
|
||||
g_cu_row = g_cu_row.load().reshape((1, 2, 1))
|
||||
|
||||
# mode0 is 2 consecutive elems
|
||||
# mode1 is top and bottom 8 rows
|
||||
# mode2 is next 8 columns
|
||||
# mode3 is repeating that 16x16 tile pattern
|
||||
kkt = _tcgen05.ld(kkt_tmem, 0, "16x256b", BT // 8)
|
||||
kkt = kkt.reshape((2, 2, 2, BT // 16))
|
||||
|
||||
for i in cutlass.range_constexpr(BT // 16):
|
||||
# mode0 is 2 elems next to each other
|
||||
# mode1 is 4 pairs of elems on 1 row
|
||||
# mode2 is top and bottom 8 rows
|
||||
# mode3 is next 16 columns
|
||||
col_coord = (None, lane_id % 4, None, i)
|
||||
s_g_cu_view = cute.make_tensor(
|
||||
s_g_cu[None, stage_id].iterator, (2, 4, 2, BT // 16)
|
||||
)
|
||||
g_cu_col = s_g_cu_view[col_coord].load().reshape((2, 1, 2))
|
||||
|
||||
Gamma = cute.math.exp(g_cu_row - g_cu_col, fastmath=True)
|
||||
A = kkt[None, None, None, i] * beta_row * Gamma
|
||||
|
||||
# strict lower mask
|
||||
# NOTE: for OOB t position, s_beta is filled with zeros.
|
||||
# hence, we don't need to apply bounds check for columns.
|
||||
A_masked = cute.where(row_indices > col_indices + i * 16, A, 0.0)
|
||||
|
||||
# pack to BF16
|
||||
# CuteDSL doesn't generate cvt.bf16x2.f32 here for some reasons
|
||||
packed = cute.make_rmem_tensor(4, Uint32)
|
||||
for j in cutlass.range_constexpr(4):
|
||||
packed[j] = cvt.fp32x2_to_bf16x2(
|
||||
A_masked[j * 2], A_masked[j * 2 + 1]
|
||||
)
|
||||
|
||||
# store to smem
|
||||
cute.copy(
|
||||
stsm_atom,
|
||||
cute.recast_tensor(packed, BFloat16),
|
||||
sA_ldsm[None, (warp_id_, i)],
|
||||
)
|
||||
|
||||
# use sync warp instead of bar.sync because for block-diagonal inverse,
|
||||
# each warp reads its own private smem memory.
|
||||
cute.arch.sync_warp()
|
||||
|
||||
##### Phase 2: matrix inverse #####
|
||||
# we use Newton-Schulz iterations to compute the inverse
|
||||
# of the four 16x16 diagonal blocks.
|
||||
# Ai_new = 2 Ai - Ai @ M @ Ai
|
||||
# where M = I + A
|
||||
#
|
||||
# we do this with 2 MMAs:
|
||||
# 1. -AiM = Ai @ (-M)
|
||||
# 2. Ai_new = 2 Ai + (-AiM) @ Ai
|
||||
zeros_f32 = cute.make_rmem_tensor(4, Float32)
|
||||
zeros_f32.fill(0.0)
|
||||
|
||||
Ai_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
mma_B_bf16 = cute.make_rmem_tensor((4, 2), BFloat16)
|
||||
M_bf16 = cute.make_rmem_tensor((4, 2), BFloat16)
|
||||
acc = cute.make_rmem_tensor((4, 2), Float32)
|
||||
|
||||
# ldmatrix copies require rank-1 destination views.
|
||||
mma_B_bf16_ldsm = cute.group_modes(mma_B_bf16, 0, 2)
|
||||
|
||||
# Packed aliases for BF16x2 arithmetic.
|
||||
Ai = cute.recast_tensor(Ai_bf16, Uint32)
|
||||
M = cute.recast_tensor(M_bf16, Uint32)
|
||||
|
||||
# construct rmem-backed identity matrix
|
||||
eye = cute.make_rmem_tensor(4, Uint32)
|
||||
eye[0] = Uint32(lane_id % 9 == 0) * Uint32(0x00003F80) + Uint32(
|
||||
lane_id % 9 == 4
|
||||
) * Uint32(0x3F800000)
|
||||
eye[1] = 0
|
||||
eye[2] = 0
|
||||
eye[3] = eye[0]
|
||||
|
||||
# initial guess: Ai = I-A
|
||||
cute.copy(ldsm_atom, sA_ldsm[None, (warp_id_, warp_id_)], Ai_bf16)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
Ai[i] = _bf16x2_sub(eye[i], Ai[i])
|
||||
|
||||
# (4, 2)
|
||||
Ai_f32 = cute.logical_divide(cvt.bf16x2_to_fp32x2(Ai), 4)
|
||||
|
||||
# M is holding -(I+A), stay constant throughout the iterations
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sA_ldsm[None, (warp_id_, warp_id_)],
|
||||
cute.group_modes(M_bf16, 0, 2),
|
||||
)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
M[i] = _bf16x2_sub(_bf16x2_neg(eye[i]), M[i])
|
||||
|
||||
# The fourth round keeps correlated-key KKT blocks stable in BF16.
|
||||
for _ in cutlass.range_constexpr(4):
|
||||
# First MMA: -AiM = Ai @ (-M)
|
||||
cute.copy(stsm_atom, Ai_bf16, sA_ldsm[None, (warp_id_, warp_id_)])
|
||||
cute.arch.sync_warp()
|
||||
acc[None, 0] = mma_sync(Ai_bf16, M_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, M_bf16[None, 1], zeros_f32)
|
||||
Ai_bf16.store(acc.load().to(BFloat16))
|
||||
|
||||
# Second MMA: Ai_new = 2Ai + (-AiM) @ Ai
|
||||
for j in cutlass.range(8, vectorize=True):
|
||||
Ai_f32[j] *= 2.0
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sA_ldsm[None, (warp_id_, warp_id_)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
Ai_f32[None, 0] = mma_sync(
|
||||
Ai_bf16, mma_B_bf16[None, 0], Ai_f32[None, 0]
|
||||
)
|
||||
Ai_f32[None, 1] = mma_sync(
|
||||
Ai_bf16, mma_B_bf16[None, 1], Ai_f32[None, 1]
|
||||
)
|
||||
Ai_bf16.store(Ai_f32.load().to(BFloat16))
|
||||
|
||||
cute.copy(stsm_atom, Ai_bf16, sAi_ldsm[None, (warp_id_, warp_id_)])
|
||||
store_ab_abg(
|
||||
Ai_f32,
|
||||
s_beta[None, stage_id],
|
||||
s_beta_g[None, stage_id],
|
||||
warp_id_,
|
||||
lane_id,
|
||||
warp_id_,
|
||||
Ab_tmem_base,
|
||||
)
|
||||
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
|
||||
|
||||
# off-diagonal by 1
|
||||
# given
|
||||
# [ Ai00 ]
|
||||
# [ A10 Ai11 ]
|
||||
# [ A20 A21 Ai22 ]
|
||||
# [ A30 A31 A32 Ai33]
|
||||
# warp1: Ai10 = -Ai11 @ A10 @ Ai00
|
||||
# warp2: Ai21 = -Ai22 @ A21 @ Ai11
|
||||
# warp3: Ai32 = -Ai33 @ A32 @ Ai22
|
||||
if warp_id_ >= 1:
|
||||
neg_Ai = cute.make_rmem_tensor(4, Uint32)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
neg_Ai[i] = _bf16x2_neg(Ai[i])
|
||||
neg_Ai_bf16 = cute.recast_tensor(neg_Ai, BFloat16)
|
||||
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sA_ldsm[None, (warp_id_, warp_id_ - 1)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(neg_Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(neg_Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
Ai_bf16.store(acc.load().to(BFloat16))
|
||||
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (warp_id_ - 1, warp_id_ - 1)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
Ai_bf16.store(acc.load().to(BFloat16))
|
||||
store_ab_abg(
|
||||
acc,
|
||||
s_beta[None, stage_id],
|
||||
s_beta_g[None, stage_id],
|
||||
warp_id_,
|
||||
lane_id,
|
||||
warp_id_ - 1,
|
||||
Ab_tmem_base,
|
||||
)
|
||||
cute.copy(
|
||||
stsm_atom,
|
||||
Ai_bf16,
|
||||
sAi_ldsm[None, (warp_id_, warp_id_ - 1)],
|
||||
)
|
||||
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
|
||||
|
||||
# off-diagonal by 2
|
||||
# warp2: Ai20 = -Ai22 @ (A20 @ Ai00 + A21 @ Ai10)
|
||||
# warp3: Ai31 = -Ai33 @ (A31 @ Ai11 + A32 @ Ai21)
|
||||
if warp_id_ >= 2:
|
||||
tile_col = warp_id_ - 2
|
||||
cute.copy(
|
||||
ldsm_atom,
|
||||
sA_ldsm[None, (warp_id_, tile_col)],
|
||||
Ai_bf16,
|
||||
)
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (tile_col, tile_col)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
|
||||
cute.copy(
|
||||
ldsm_atom, sA_ldsm[None, (warp_id_, tile_col + 1)], Ai_bf16
|
||||
)
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (tile_col + 1, tile_col)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], acc[None, 0])
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], acc[None, 1])
|
||||
|
||||
tmp = cute.make_rmem_tensor(8, BFloat16)
|
||||
tmp.store(acc.load().to(BFloat16))
|
||||
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (warp_id_, tile_col)])
|
||||
cute.arch.sync_warp()
|
||||
|
||||
cute.copy(ldsm_atom, sAi_ldsm[None, (warp_id_, warp_id_)], Ai_bf16)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
Ai[i] = _bf16x2_neg(Ai[i])
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (warp_id_, tile_col)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
tmp.store(acc.load().to(BFloat16))
|
||||
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (warp_id_, tile_col)])
|
||||
store_ab_abg(
|
||||
acc,
|
||||
s_beta[None, stage_id],
|
||||
s_beta_g[None, stage_id],
|
||||
warp_id_,
|
||||
lane_id,
|
||||
tile_col,
|
||||
Ab_tmem_base,
|
||||
)
|
||||
cute.arch.barrier(barrier_id=INV_BAR, number_of_threads=128)
|
||||
|
||||
# off-diagonal by 3
|
||||
# warp3: Ai30 = -Ai33 @ (A30 @ Ai00 + A31 @ Ai10 + A32 @ Ai20)
|
||||
if warp_id_ == 3:
|
||||
cute.copy(ldsm_atom, sA_ldsm[None, (3, 0)], Ai_bf16)
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (0, 0)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
|
||||
for i in cutlass.range_constexpr(1, 3):
|
||||
cute.copy(ldsm_atom, sA_ldsm[None, (3, i)], Ai_bf16)
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (i, 0)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(
|
||||
Ai_bf16, mma_B_bf16[None, 0], acc[None, 0]
|
||||
)
|
||||
acc[None, 1] = mma_sync(
|
||||
Ai_bf16, mma_B_bf16[None, 1], acc[None, 1]
|
||||
)
|
||||
|
||||
tmp = cute.make_rmem_tensor(8, BFloat16)
|
||||
tmp.store(acc.load().to(BFloat16))
|
||||
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (3, 0)])
|
||||
cute.arch.sync_warp()
|
||||
|
||||
cute.copy(ldsm_atom, sAi_ldsm[None, (3, 3)], Ai_bf16)
|
||||
for i in cutlass.range_constexpr(4):
|
||||
Ai[i] = _bf16x2_neg(Ai[i])
|
||||
cute.copy(
|
||||
ldsm_trans_atom,
|
||||
sAi_ldsm[None, (3, 0)],
|
||||
mma_B_bf16_ldsm,
|
||||
)
|
||||
acc[None, 0] = mma_sync(Ai_bf16, mma_B_bf16[None, 0], zeros_f32)
|
||||
acc[None, 1] = mma_sync(Ai_bf16, mma_B_bf16[None, 1], zeros_f32)
|
||||
tmp.store(acc.load().to(BFloat16))
|
||||
cute.copy(stsm_atom, tmp, sAi_ldsm[None, (3, 0)])
|
||||
store_ab_abg(
|
||||
acc,
|
||||
s_beta[None, stage_id],
|
||||
s_beta_g[None, stage_id],
|
||||
warp_id_,
|
||||
lane_id,
|
||||
0,
|
||||
Ab_tmem_base,
|
||||
)
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(inv_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
else:
|
||||
# epi warps
|
||||
stage_id = 0
|
||||
parity = 0
|
||||
|
||||
# ((BT, num_global_chunks), V_dim)
|
||||
gU_tiles = cute.logical_divide(
|
||||
U_tma.tma_tensor[None, head_id, None], (BT, None)
|
||||
)
|
||||
gW_tiles = cute.logical_divide(
|
||||
W_tma.tma_tensor[None, head_id, None], (BT, None)
|
||||
)
|
||||
|
||||
# sW shape: [BT, (64, K_dim/64)]
|
||||
# sW_view shape: [(8, 2), (4, K_dim/64)]
|
||||
s_row = warp_id * 16 + lane_id % 16 # select the rows of [16,16] tile
|
||||
sW_view = cute.zipped_divide(
|
||||
sW[s_row, None],
|
||||
tiler=cute.make_layout((8, 2)),
|
||||
)
|
||||
sU_view = cute.zipped_divide(
|
||||
sU[s_row, None],
|
||||
tiler=cute.make_layout((8, 2)),
|
||||
)
|
||||
|
||||
# select the 8 columns within [16,16] tile
|
||||
sW_view = sW_view[(None, lane_id // 16), None]
|
||||
sU_view = sU_view[(None, lane_id // 16), None]
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
# wait for W MMA + previous TMA store to finish
|
||||
U_tmem = U_tmem_base + V_dim * stage_id
|
||||
if warp_id == 0:
|
||||
cute.arch.mbarrier_wait(mma_w_mbar + stage_id, parity)
|
||||
elif warp_id == 1:
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_wait_group(0, read=True)
|
||||
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
w_f32 = _tcgen05.ld(warp_id * 32 + 16, U_tmem, "16x256b", K_dim // 8)
|
||||
_tcgen05.wait_ld()
|
||||
w_bf16 = cute.make_rmem_tensor((8, K_dim // 16), BFloat16)
|
||||
w_bf16.store(w_f32.to(BFloat16))
|
||||
cute.copy(stsm_atom, w_bf16, sW_view)
|
||||
|
||||
# wait for U MMA + issue W TMA store
|
||||
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id == 0:
|
||||
cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity)
|
||||
elif warp_id == 1:
|
||||
# don't need to commit
|
||||
simple_tma_copy(
|
||||
W_tma.atom, sW, gW_tiles[(None, global_chunk_id), None]
|
||||
)
|
||||
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
u_f32 = _tcgen05.ld(warp_id * 32, U_tmem, "16x256b", V_dim // 8)
|
||||
_tcgen05.wait_ld()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(epi_mbar + stage_id)
|
||||
u_bf16 = cute.make_rmem_tensor((8, V_dim // 16), BFloat16)
|
||||
u_bf16.store(u_f32.to(BFloat16))
|
||||
cute.copy(stsm_atom, u_bf16, sU_view)
|
||||
|
||||
cute.arch.barrier(barrier_id=EPI_BAR, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id == 1:
|
||||
simple_tma_copy(
|
||||
U_tma.atom, sU, gU_tiles[(None, global_chunk_id), None]
|
||||
)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(H: int, Hv: int, K_dim: int, V_dim: int, num_stages: int = 2):
|
||||
total_t = cute.sym_int()
|
||||
pad_t = cute.sym_int()
|
||||
total_chunks_n = cute.sym_int()
|
||||
num_sequences = cute.sym_int()
|
||||
|
||||
K = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
|
||||
V = make_fake_tensor(BFloat16, (total_t, Hv, V_dim), divisibility=16)
|
||||
U = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
|
||||
W = make_fake_tensor(BFloat16, (pad_t, Hv, K_dim), divisibility=16)
|
||||
g = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
|
||||
beta = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
|
||||
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
|
||||
cu_seqlens = make_fake_tensor(Int32, (num_sequences,), divisibility=1)
|
||||
chunk_indices = make_fake_tensor(Int32, (total_chunks_n, 2), divisibility=2)
|
||||
total_chunks = make_fake_tensor(Int32, (1,), divisibility=1)
|
||||
|
||||
kernel = Sm100ChunkUWKernel(H, Hv, K_dim, V_dim, num_stages)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
K,
|
||||
V,
|
||||
U,
|
||||
W,
|
||||
g,
|
||||
beta,
|
||||
g_cu,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
Int32(148),
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def kkt_inv_uw_cutedsl(
|
||||
K: torch.Tensor,
|
||||
V: torch.Tensor,
|
||||
U: torch.Tensor,
|
||||
W: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
g_cu: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
chunk_indices: torch.Tensor,
|
||||
total_chunks: torch.Tensor,
|
||||
num_sms: int = 148,
|
||||
) -> None:
|
||||
_, Hv, V_dim = V.shape
|
||||
_, H, K_dim = K.shape
|
||||
|
||||
Sm100ChunkUWKernel.compile(H, Hv, K_dim, V_dim)(
|
||||
K,
|
||||
V,
|
||||
U,
|
||||
W,
|
||||
g,
|
||||
beta,
|
||||
g_cu,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
num_sms,
|
||||
)
|
||||
@@ -1,632 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute
|
||||
from cutlass.cute.nvgpu import cpasync, warp
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import (
|
||||
EVICT_FIRST,
|
||||
_tcgen05,
|
||||
cvt,
|
||||
fence_before_tma_store,
|
||||
simple_tma_copy,
|
||||
)
|
||||
|
||||
|
||||
class Sm100ChunkOKernel:
|
||||
"""Compute per-token output from recurrent and intra-chunk terms.
|
||||
|
||||
Gamma[i,j] = exp(g_cu[i] - g_cu[j])
|
||||
P = mask((Q @ K.T) * Gamma)
|
||||
O = scale * (exp(g_cu) * (Q @ H.T) + P @ V)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
H: int,
|
||||
Hv: int,
|
||||
K_dim: int,
|
||||
V_dim: int,
|
||||
BT: int = 64,
|
||||
num_stages: int = 2,
|
||||
) -> None:
|
||||
assert Hv % H == 0
|
||||
assert K_dim == 128
|
||||
assert V_dim == 128
|
||||
assert BT == 64
|
||||
self.H = H
|
||||
self.Hv = Hv
|
||||
self.K_dim = K_dim
|
||||
self.V_dim = V_dim
|
||||
self.BT = BT
|
||||
self.num_stages = num_stages
|
||||
self.num_warps = 10
|
||||
|
||||
@cute.jit
|
||||
def _make_bf16_tma_args(
|
||||
self,
|
||||
tensor: cute.Tensor,
|
||||
dim: cutlass.Constexpr[int],
|
||||
op: cpasync.TmaCopyOp,
|
||||
stages: cutlass.Constexpr[int],
|
||||
):
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
slayout = cute.make_layout(
|
||||
(self.BT, 1, (64, dim // 64), stages),
|
||||
stride=(64, 0, (1, self.BT * 64), self.BT * dim),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
return cpasync.make_tiled_tma_atom(
|
||||
op,
|
||||
cute.logical_divide(tensor, (None, None, 64)),
|
||||
slayout,
|
||||
cta_tiler=(self.BT, 1, dim),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _make_h_tma_args(
|
||||
self,
|
||||
tensor: cute.Tensor,
|
||||
op: cpasync.TmaCopyOp,
|
||||
stages: cutlass.Constexpr[int],
|
||||
):
|
||||
num_elems = 128 // (tensor.element_type.width // 8)
|
||||
swizzle_128B = cute.make_swizzle(3, 4, 3)
|
||||
slayout = cute.make_layout(
|
||||
(1, 1, self.V_dim, (num_elems, self.K_dim // num_elems), stages),
|
||||
stride=(
|
||||
0,
|
||||
0,
|
||||
num_elems,
|
||||
(1, self.V_dim * num_elems),
|
||||
self.V_dim * self.K_dim,
|
||||
),
|
||||
)
|
||||
slayout = cute.make_composed_layout(swizzle_128B, 0, slayout)
|
||||
return cpasync.make_tiled_tma_atom(
|
||||
op,
|
||||
cute.logical_divide(tensor, (None, None, None, num_elems)),
|
||||
slayout,
|
||||
cta_tiler=(1, 1, self.V_dim, self.K_dim),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
q: cute.Tensor,
|
||||
k: cute.Tensor,
|
||||
v_new_chunks: cute.Tensor,
|
||||
h: cute.Tensor,
|
||||
g_cu: cute.Tensor,
|
||||
o: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
total_chunks: cute.Tensor,
|
||||
scale: Float32,
|
||||
num_sms: Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (num_sms // self.Hv, self.Hv, 1)
|
||||
block = (self.num_warps * 32, 1, 1)
|
||||
tma_g2s = cpasync.CopyBulkTensorTileG2SOp()
|
||||
tma_s2g = cpasync.CopyBulkTensorTileS2GOp()
|
||||
Q_tma = self._make_bf16_tma_args(q, self.K_dim, tma_g2s, self.num_stages)
|
||||
K_tma = self._make_bf16_tma_args(k, self.K_dim, tma_g2s, self.num_stages)
|
||||
V_tma = self._make_bf16_tma_args(
|
||||
v_new_chunks, self.V_dim, tma_g2s, self.num_stages
|
||||
)
|
||||
H_tma = self._make_h_tma_args(h, tma_g2s, self.num_stages)
|
||||
O_tma = self._make_bf16_tma_args(o, self.V_dim, tma_s2g, 1)
|
||||
self.kernel(
|
||||
Q_tma,
|
||||
K_tma,
|
||||
V_tma,
|
||||
H_tma,
|
||||
O_tma,
|
||||
g_cu,
|
||||
o,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
scale,
|
||||
).launch(grid=grid, block=block, min_blocks_per_mp=1, stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
Q_tma: cpasync.TmaInfo,
|
||||
K_tma: cpasync.TmaInfo,
|
||||
V_tma: cpasync.TmaInfo,
|
||||
H_tma: cpasync.TmaInfo,
|
||||
O_tma: cpasync.TmaInfo,
|
||||
g_cu: cute.Tensor,
|
||||
o: cute.Tensor,
|
||||
cu_seqlens: cute.Tensor,
|
||||
chunk_indices: cute.Tensor,
|
||||
total_chunks: cute.Tensor,
|
||||
scale: Float32,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
bid, v_head_id, _ = cute.arch.block_idx()
|
||||
grid_x, _, _ = cute.arch.grid_dim()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
|
||||
BT = self.BT
|
||||
K_dim = self.K_dim
|
||||
V_dim = self.V_dim
|
||||
num_stages = self.num_stages
|
||||
|
||||
heads_per_qk = self.Hv // self.H
|
||||
k_head_id = v_head_id // heads_per_qk
|
||||
num_global_chunks = total_chunks[0]
|
||||
|
||||
def allocate_tensor(smem, dtype, layout):
|
||||
return smem.allocate_tensor(
|
||||
dtype, layout.outer, byte_alignment=128, swizzle=layout.inner
|
||||
)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
sQ = allocate_tensor(smem, BFloat16, Q_tma.smem_layout)[None, 0, None, None]
|
||||
sK = allocate_tensor(smem, BFloat16, K_tma.smem_layout)[None, 0, None, None]
|
||||
sV = allocate_tensor(smem, BFloat16, V_tma.smem_layout)[None, 0, None, None]
|
||||
sH = allocate_tensor(smem, BFloat16, H_tma.smem_layout)[0, 0, None, None, None]
|
||||
sO = allocate_tensor(smem, BFloat16, O_tma.smem_layout)[None, 0, None, 0]
|
||||
|
||||
s_g_cu = smem.allocate_array(Float32, BT)
|
||||
qk_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||
hv_full_mbar = smem.allocate_array(Int64, num_stages)
|
||||
qk_empty_mbar = smem.allocate_array(Int64, num_stages)
|
||||
pv_mma_mbar = smem.allocate_array(Int64, num_stages)
|
||||
qk_mbar = smem.allocate_array(Int64, 1)
|
||||
mask_mbar = smem.allocate_array(Int64, 1)
|
||||
epi_mbar = smem.allocate_array(Int64, 1)
|
||||
taddr = smem.allocate(Int32, 4)
|
||||
|
||||
qk_tmem = 0
|
||||
p_tmem = 64
|
||||
out_tmem = 128
|
||||
qh_tmem = 256
|
||||
|
||||
if warp_id == 0:
|
||||
with cute.arch.elect_one():
|
||||
for i in cutlass.range_constexpr(num_stages):
|
||||
cute.arch.mbarrier_init(qk_full_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(qk_empty_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(hv_full_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(pv_mma_mbar + i, 1)
|
||||
cute.arch.mbarrier_init(qk_mbar, 1)
|
||||
cute.arch.mbarrier_init(mask_mbar, 128)
|
||||
cute.arch.mbarrier_init(epi_mbar, 128)
|
||||
cute.arch.mbarrier_init_fence()
|
||||
elif warp_id == 9:
|
||||
cpasync.prefetch_descriptor(Q_tma.atom)
|
||||
cpasync.prefetch_descriptor(K_tma.atom)
|
||||
cpasync.prefetch_descriptor(V_tma.atom)
|
||||
cpasync.prefetch_descriptor(H_tma.atom)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
if warp_id == 9:
|
||||
# TMA warp
|
||||
stage_id = 0
|
||||
parity = 1
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
|
||||
# copy Q and K
|
||||
q_tile = cute.local_tile(
|
||||
cute.domain_offset(
|
||||
(bos, 0), Q_tma.tma_tensor[None, k_head_id, None]
|
||||
),
|
||||
tiler=(BT, K_dim),
|
||||
coord=(chunk_id, 0),
|
||||
)
|
||||
k_tile = cute.local_tile(
|
||||
cute.domain_offset(
|
||||
(bos, 0), K_tma.tma_tensor[None, k_head_id, None]
|
||||
),
|
||||
tiler=(BT, K_dim),
|
||||
coord=(chunk_id, 0),
|
||||
)
|
||||
mbar = qk_full_mbar + stage_id
|
||||
|
||||
cute.arch.mbarrier_wait(qk_empty_mbar + stage_id, parity)
|
||||
|
||||
with cute.arch.elect_one():
|
||||
STAGE_SIZE = BT * (K_dim + K_dim) * 2
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE)
|
||||
simple_tma_copy(Q_tma.atom, q_tile, sQ[None, None, stage_id], mbar)
|
||||
simple_tma_copy(K_tma.atom, k_tile, sK[None, None, stage_id], mbar)
|
||||
|
||||
# copy H and V
|
||||
gH = H_tma.tma_tensor[global_chunk_id, v_head_id, None, None]
|
||||
gV = cute.local_tile(
|
||||
V_tma.tma_tensor[None, v_head_id, None],
|
||||
tiler=(BT, V_dim),
|
||||
coord=(global_chunk_id, 0),
|
||||
)
|
||||
mbar = hv_full_mbar + stage_id
|
||||
|
||||
cute.arch.mbarrier_wait(pv_mma_mbar + stage_id, parity)
|
||||
|
||||
with cute.arch.elect_one():
|
||||
H_STAGE_SIZE = V_dim * K_dim * 2
|
||||
V_STAGE_SIZE = BT * V_dim * 2
|
||||
cute.arch.mbarrier_arrive_and_expect_tx(
|
||||
mbar, H_STAGE_SIZE + V_STAGE_SIZE
|
||||
)
|
||||
simple_tma_copy(
|
||||
H_tma.atom, gH, sH[None, None, stage_id], mbar, EVICT_FIRST
|
||||
)
|
||||
simple_tma_copy(
|
||||
V_tma.atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST
|
||||
)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
parity ^= 1
|
||||
|
||||
elif warp_id == 8:
|
||||
# MMA warp
|
||||
_tcgen05.alloc(taddr)
|
||||
|
||||
# LBO=BT*128 is ignored for K-major
|
||||
sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128)
|
||||
qk_idesc = _tcgen05.make_bf16_idesc(BT, BT)
|
||||
qh_idesc = _tcgen05.make_bf16_idesc(BT, V_dim)
|
||||
pv_idesc = _tcgen05.make_bf16_idesc(BT, V_dim, transpose_B=True)
|
||||
|
||||
stage_id = 0
|
||||
tma_parity = 0
|
||||
mask_parity = 0
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
qaddr = sQ[None, None, stage_id].iterator.toint()
|
||||
kaddr = sK[None, None, stage_id].iterator.toint()
|
||||
haddr = sH[None, None, stage_id].iterator.toint()
|
||||
vaddr = sV[None, None, stage_id].iterator.toint()
|
||||
qdesc_base = sdesc_template | (qaddr >> 4)
|
||||
kdesc_base = sdesc_template | (kaddr >> 4)
|
||||
hdesc_base = sdesc_template | (haddr >> 4)
|
||||
vdesc_base = sdesc_template | (vaddr >> 4)
|
||||
|
||||
##### 1st MMA: Q @ K.T #####
|
||||
# do this first to unblock mask(QK)
|
||||
cute.arch.mbarrier_wait(epi_mbar, mask_parity ^ 1)
|
||||
cute.arch.mbarrier_wait(qk_full_mbar + stage_id, tma_parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
for i in cutlass.range_constexpr(K_dim // 64):
|
||||
for j in cutlass.range_constexpr(64 // 16):
|
||||
qdesc = qdesc_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
kdesc = kdesc_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
_tcgen05.mma_f16(
|
||||
qk_tmem, qdesc, kdesc, qk_idesc, (i > 0) or (j > 0)
|
||||
)
|
||||
_tcgen05.commit(qk_mbar)
|
||||
|
||||
##### 2nd MMA: Q @ H.T #####
|
||||
cute.arch.mbarrier_wait(hv_full_mbar + stage_id, tma_parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
for i in cutlass.range_constexpr(K_dim // 64):
|
||||
for j in cutlass.range_constexpr(64 // 16):
|
||||
qdesc = qdesc_base | ((i * BT * 128 + j * 32) >> 4)
|
||||
hdesc = hdesc_base | ((i * V_dim * 128 + j * 32) >> 4)
|
||||
_tcgen05.mma_f16(
|
||||
qh_tmem, qdesc, hdesc, qh_idesc, (i > 0) or (j > 0)
|
||||
)
|
||||
_tcgen05.commit(qk_empty_mbar + stage_id)
|
||||
|
||||
##### 3rd MMA: P @ V #####
|
||||
# stalled by mask(QK)
|
||||
cute.arch.mbarrier_wait(mask_mbar, mask_parity)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
for i in cutlass.range_constexpr(BT // 16):
|
||||
vdesc = vdesc_base | ((i * 16 * 128) >> 4)
|
||||
_tcgen05.mma_ts_f16(
|
||||
out_tmem, p_tmem + i * 8, vdesc, pv_idesc, i > 0
|
||||
)
|
||||
_tcgen05.commit(pv_mma_mbar + stage_id)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
tma_parity ^= 1
|
||||
mask_parity ^= 1
|
||||
|
||||
# wait for epilogue to finish for deallocation
|
||||
cute.arch.mbarrier_wait(epi_mbar, mask_parity ^ 1)
|
||||
_tcgen05.dealloc()
|
||||
|
||||
elif warp_id >= 4:
|
||||
# masking warps
|
||||
warp_id_ = warp_id % 4
|
||||
tid_ = tid % 128
|
||||
row0 = warp_id_ * 16 + lane_id // 4
|
||||
row1 = row0 + 8
|
||||
|
||||
parity = 0
|
||||
|
||||
# for ldmatrix layout later
|
||||
row_indices = cute.make_rmem_tensor(2, Int32)
|
||||
row_indices[0] = warp_id_ * 16 + lane_id // 4
|
||||
row_indices[1] = warp_id_ * 16 + lane_id // 4 + 8
|
||||
row_indices = row_indices.load().reshape((1, 2))
|
||||
|
||||
col_indices = cute.make_rmem_tensor(2, Int32)
|
||||
col_indices[0] = (lane_id % 4) * 2
|
||||
col_indices[1] = (lane_id % 4) * 2 + 1
|
||||
col_indices = col_indices.load().reshape((2, 1))
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
if tid_ < BT:
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
eos = cu_seqlens[seq_id + 1]
|
||||
|
||||
t_ = bos + chunk_id * BT + tid_
|
||||
s_g_cu[tid_] = g_cu[t_, v_head_id] if t_ < eos else Float32(0.0)
|
||||
|
||||
# wait for QK MMA
|
||||
if warp_id_ == 0:
|
||||
cute.arch.mbarrier_wait(qk_mbar, parity)
|
||||
cute.arch.barrier(barrier_id=1, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
qk = _tcgen05.ld(warp_id_ * 32, qk_tmem, "16x256b", BT // 8)
|
||||
qk = qk.reshape((2, 2, BT // 8))
|
||||
_tcgen05.wait_ld()
|
||||
|
||||
g_cu_rows = cute.make_rmem_tensor(2, Float32)
|
||||
g_cu_rows[0] = s_g_cu[row0]
|
||||
g_cu_rows[1] = s_g_cu[row1]
|
||||
g_cu_rows = g_cu_rows.load().reshape((1, 2))
|
||||
|
||||
for i in cutlass.range_constexpr(BT // 8):
|
||||
col = i * 8 + (lane_id % 4) * 2
|
||||
g_cu_cols = cute.make_rmem_tensor(2, Float32)
|
||||
g_cu_cols[0] = s_g_cu[col]
|
||||
g_cu_cols[1] = s_g_cu[col + 1]
|
||||
g_cu_cols = g_cu_cols.load().reshape((2, 1))
|
||||
|
||||
# apply gamma and causal mask
|
||||
Gamma = cute.math.exp(g_cu_rows - g_cu_cols, fastmath=True)
|
||||
tmp = qk[None, None, i] * Gamma
|
||||
tmp = cute.where(row_indices >= col_indices + i * 8, tmp, 0.0)
|
||||
|
||||
# CuteDSL can't emit cvt.bf16x2.f32 here
|
||||
attn_lo = cute.make_rmem_tensor(2, Uint32)
|
||||
attn_lo[0] = cvt.fp32x2_to_bf16x2(tmp[0, 0], tmp[1, 0])
|
||||
attn_lo[1] = cvt.fp32x2_to_bf16x2(tmp[0, 1], tmp[1, 1])
|
||||
_tcgen05.st(warp_id_ * 32, p_tmem + i * 4, "16x128b", 1, attn_lo)
|
||||
|
||||
_tcgen05.wait_st()
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(mask_mbar)
|
||||
|
||||
parity ^= 1
|
||||
|
||||
else:
|
||||
# epilogue warps
|
||||
# for ldmatrix layout later
|
||||
row0 = warp_id * 16 + lane_id // 4
|
||||
row1 = row0 + 8
|
||||
|
||||
stage_id = 0
|
||||
mma_parity = 0
|
||||
|
||||
op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_4B = cute.make_copy_atom(op, BFloat16, num_bits_per_copy=32)
|
||||
stsm_op = warp.StMatrix8x8x16bOp(num_matrices=4, transpose=False)
|
||||
stsm_atom = cute.make_copy_atom(stsm_op, BFloat16)
|
||||
|
||||
# ldmatrix layout
|
||||
# [total_seq_len, ((2, 4, WIDTH/8), V_DIM/WIDTH)]
|
||||
WIDTH = 64
|
||||
o_view = cute.logical_divide(
|
||||
o[None, v_head_id, None],
|
||||
(None, cute.make_layout((2, 4, WIDTH // 8))),
|
||||
)
|
||||
# select lane: [total_seq_len, 2, WIDTH/8, V_DIM/WIDTH]
|
||||
o_view = o_view[None, ((None, lane_id % 4, None), None)]
|
||||
|
||||
for global_chunk_id in range(bid, num_global_chunks, grid_x):
|
||||
seq_id = chunk_indices[global_chunk_id, 0]
|
||||
chunk_id = chunk_indices[global_chunk_id, 1]
|
||||
bos = cu_seqlens[seq_id]
|
||||
eos = cu_seqlens[seq_id + 1]
|
||||
chunk_start = bos + chunk_id * BT
|
||||
full_chunk = chunk_start + BT <= eos
|
||||
|
||||
g_cu_rows = cute.make_rmem_tensor(2, Float32)
|
||||
g_cu_rows.fill(0.0)
|
||||
|
||||
# load g_cu
|
||||
if chunk_start + row0 < eos:
|
||||
g_cu_rows[0] = cute.math.exp(
|
||||
g_cu[chunk_start + row0, v_head_id], fastmath=True
|
||||
)
|
||||
if chunk_start + row1 < eos:
|
||||
g_cu_rows[1] = cute.math.exp(
|
||||
g_cu[chunk_start + row1, v_head_id], fastmath=True
|
||||
)
|
||||
g_cu_rows = g_cu_rows.load().reshape((1, 2, 1))
|
||||
|
||||
if warp_id == 0:
|
||||
cute.arch.mbarrier_wait(pv_mma_mbar + stage_id, mma_parity)
|
||||
elif warp_id == 3 and full_chunk:
|
||||
cute.arch.cp_async_bulk_wait_group(0, read=True)
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=128)
|
||||
_tcgen05.fence_after_thread_sync()
|
||||
|
||||
if full_chunk:
|
||||
# use TMA store: tmem->rmem->smem->gmem
|
||||
for i in cutlass.range_constexpr(V_dim // WIDTH):
|
||||
qh = _tcgen05.ld(
|
||||
warp_id * 32, qh_tmem + i * WIDTH, "16x256b", WIDTH // 8
|
||||
)
|
||||
pv = _tcgen05.ld(
|
||||
warp_id * 32, out_tmem + i * WIDTH, "16x256b", WIDTH // 8
|
||||
)
|
||||
_tcgen05.wait_ld()
|
||||
if i == V_dim // WIDTH - 1:
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(epi_mbar)
|
||||
|
||||
qh = qh.reshape((2, 2, WIDTH // 8))
|
||||
pv = pv.reshape((2, 2, WIDTH // 8))
|
||||
|
||||
out_f32 = scale * (g_cu_rows * qh + pv)
|
||||
out_bf16 = cute.make_rmem_tensor((8, WIDTH // 16), BFloat16)
|
||||
out_bf16.store(out_f32.to(BFloat16).reshape((8, WIDTH // 16)))
|
||||
|
||||
# TODO: issue single cute.copy()
|
||||
for j in cutlass.range_constexpr(WIDTH // 16):
|
||||
s_row = warp_id * 16 + lane_id % 16
|
||||
s_col = i * (WIDTH // 8) + j * 2 + lane_id // 16
|
||||
sO_tile = cute.local_tile(sO[s_row, None], (8,), (s_col,))
|
||||
cute.copy(stsm_atom, out_bf16[None, j], sO_tile)
|
||||
|
||||
cute.arch.barrier(barrier_id=2, number_of_threads=128)
|
||||
fence_before_tma_store()
|
||||
if warp_id == 3:
|
||||
gO = cute.local_tile(
|
||||
cute.domain_offset(
|
||||
(bos, 0), O_tma.tma_tensor[None, v_head_id, None]
|
||||
),
|
||||
tiler=(BT, V_dim),
|
||||
coord=(chunk_id, 0),
|
||||
)
|
||||
simple_tma_copy(O_tma.atom, sO, gO)
|
||||
with cute.arch.elect_one():
|
||||
cute.arch.cp_async_bulk_commit_group()
|
||||
|
||||
else:
|
||||
# direct gmem store
|
||||
# TODO: explore doing multiple 1D TMAs
|
||||
for i in cutlass.range_constexpr(V_dim // WIDTH):
|
||||
qh = _tcgen05.ld(
|
||||
warp_id * 32, qh_tmem + i * WIDTH, "16x256b", WIDTH // 8
|
||||
)
|
||||
pv = _tcgen05.ld(
|
||||
warp_id * 32, out_tmem + i * WIDTH, "16x256b", WIDTH // 8
|
||||
)
|
||||
_tcgen05.wait_ld()
|
||||
if i == V_dim // WIDTH - 1:
|
||||
_tcgen05.fence_before_thread_sync()
|
||||
cute.arch.mbarrier_arrive(epi_mbar)
|
||||
|
||||
qh = qh.reshape((2, 2, WIDTH // 8))
|
||||
pv = pv.reshape((2, 2, WIDTH // 8))
|
||||
|
||||
out_f32 = scale * (g_cu_rows * qh + pv)
|
||||
out_bf16 = cute.make_rmem_tensor((2, 2, WIDTH // 8), BFloat16)
|
||||
out_bf16.store(out_f32.to(BFloat16))
|
||||
|
||||
if chunk_start + row0 < eos:
|
||||
cute.copy(
|
||||
cp_4B,
|
||||
out_bf16[None, 0, None],
|
||||
o_view[chunk_start + row0, None, None, i],
|
||||
)
|
||||
if chunk_start + row1 < eos:
|
||||
cute.copy(
|
||||
cp_4B,
|
||||
out_bf16[None, 1, None],
|
||||
o_view[chunk_start + row1, None, None, i],
|
||||
)
|
||||
|
||||
stage_id = (stage_id + 1) % num_stages
|
||||
if stage_id == 0:
|
||||
mma_parity ^= 1
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(
|
||||
H: int,
|
||||
Hv: int,
|
||||
K_dim: int,
|
||||
V_dim: int,
|
||||
BT: int = 64,
|
||||
num_stages: int = 2,
|
||||
):
|
||||
total_t = cute.sym_int()
|
||||
pad_t = cute.sym_int()
|
||||
total_chunks_n = cute.sym_int()
|
||||
cu_entries = cute.sym_int()
|
||||
|
||||
q = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
|
||||
k = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16)
|
||||
v_new = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16)
|
||||
h = make_fake_tensor(
|
||||
BFloat16, (total_chunks_n, Hv, V_dim, K_dim), divisibility=16
|
||||
)
|
||||
g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4)
|
||||
o = make_fake_tensor(BFloat16, (total_t, Hv, V_dim), divisibility=16)
|
||||
cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1)
|
||||
chunk_indices = make_fake_tensor(Int32, (total_chunks_n, 2), divisibility=2)
|
||||
total_chunks = make_fake_tensor(Int32, (1,), divisibility=1)
|
||||
|
||||
kernel = Sm100ChunkOKernel(
|
||||
H,
|
||||
Hv,
|
||||
K_dim,
|
||||
V_dim,
|
||||
BT,
|
||||
num_stages,
|
||||
)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
q,
|
||||
k,
|
||||
v_new,
|
||||
h,
|
||||
g_cu,
|
||||
o,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
Float32(1.0),
|
||||
Int32(148),
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def o_cutedsl(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v_new: torch.Tensor,
|
||||
h: torch.Tensor,
|
||||
g_cu: torch.Tensor,
|
||||
o: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
chunk_indices: torch.Tensor,
|
||||
total_chunks: torch.Tensor,
|
||||
scale: float,
|
||||
num_sms: int = 148,
|
||||
) -> None:
|
||||
_, H, K_dim = q.shape
|
||||
_, Hv, V_dim = o.shape
|
||||
|
||||
Sm100ChunkOKernel.compile(H, Hv, K_dim, V_dim)(
|
||||
q,
|
||||
k,
|
||||
v_new,
|
||||
h,
|
||||
g_cu,
|
||||
o,
|
||||
cu_seqlens,
|
||||
chunk_indices,
|
||||
total_chunks,
|
||||
float(scale),
|
||||
num_sms,
|
||||
)
|
||||
@@ -1,255 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MLAModules:
|
||||
"""Modules used in MLA."""
|
||||
|
||||
kv_a_layernorm: torch.nn.Module
|
||||
kv_b_proj: torch.nn.Module
|
||||
rotary_emb: torch.nn.Module | None
|
||||
o_proj: torch.nn.Module
|
||||
fused_qkv_a_proj: torch.nn.Module | None
|
||||
kv_a_proj_with_mqa: torch.nn.Module | None
|
||||
q_a_layernorm: torch.nn.Module | None
|
||||
q_b_proj: torch.nn.Module | None
|
||||
q_proj: torch.nn.Module | None
|
||||
indexer: torch.nn.Module | None
|
||||
is_sparse: bool
|
||||
topk_indices_buffer: torch.Tensor | None
|
||||
indexer_rotary_emb: torch.nn.Module | None = None
|
||||
g_proj: torch.nn.Module | None = None
|
||||
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None
|
||||
|
||||
|
||||
# --8<-- [start:multi_head_latent_attention]
|
||||
@PluggableLayer.register("multi_head_latent_attention")
|
||||
class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
"""Pluggable MLA layer which allows OOT backends to add
|
||||
custom implementations of the outer MLA layer (including rope & o_proj).
|
||||
Note that currently oot platforms can still use CustomOp.register_oot to
|
||||
replace MLA layer entirely, although we use PluggableLayer to register
|
||||
this layer now.
|
||||
|
||||
This class takes positions and hidden_states as input.
|
||||
The input tensors can either contain prefill tokens or decode tokens.
|
||||
The class does the following:
|
||||
|
||||
1. MLA Preprocess.
|
||||
2. Perform multi-head attention to prefill tokens and
|
||||
multi-query attention to decode tokens separately.
|
||||
3. Return the output tensor.
|
||||
"""
|
||||
|
||||
# --8<-- [end:multi_head_latent_attention]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
scale: float,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
mla_modules: MLAModules,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
skip_topk: bool = False,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
allow_short_prefill_indexer_scoring_skip: bool = False,
|
||||
fuse_qkv_rmsnorm: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.num_heads = num_heads
|
||||
self.fused_qkv_a_proj = mla_modules.fused_qkv_a_proj
|
||||
self.kv_a_proj_with_mqa = mla_modules.kv_a_proj_with_mqa
|
||||
self.q_a_layernorm = mla_modules.q_a_layernorm
|
||||
self.q_b_proj = mla_modules.q_b_proj
|
||||
self.q_proj = mla_modules.q_proj
|
||||
self.kv_a_layernorm = mla_modules.kv_a_layernorm
|
||||
self.kv_b_proj = mla_modules.kv_b_proj
|
||||
self.rotary_emb = mla_modules.rotary_emb
|
||||
self.o_proj = mla_modules.o_proj
|
||||
self.indexer = mla_modules.indexer
|
||||
self.indexer_rope_emb = mla_modules.indexer_rotary_emb
|
||||
self.is_sparse = mla_modules.is_sparse
|
||||
self.g_proj = mla_modules.g_proj
|
||||
|
||||
# Whether to skip top-k token selection computation in this layer.
|
||||
# When True, the indexer will not be called, and the layer will reuse
|
||||
# the topk_tokens buffer written by a previous layer in the same pass.
|
||||
# Refer: https://arxiv.org/abs/2603.12201 for more details.
|
||||
self.skip_topk = skip_topk
|
||||
# When True, fuse the q_a and kv_a RMSNorms into a single kernel launch
|
||||
# (MLA layers with q-LoRA). Opt-in; default False preserves other models.
|
||||
self.fuse_qkv_rmsnorm = fuse_qkv_rmsnorm
|
||||
# qrep is active when the query projection is a DCP-group-sharded layer
|
||||
# that materializes the full group head set locally.
|
||||
q_proj_layer = self.q_b_proj if self.q_lora_rank is not None else self.q_proj
|
||||
self.dcp_q_replicate = getattr(q_proj_layer, "qrep_active", False)
|
||||
if self.indexer is not None:
|
||||
assert hasattr(self.indexer, "topk_tokens")
|
||||
self.topk_tokens = self.indexer.topk_tokens
|
||||
self.topk_indices_buffer = mla_modules.topk_indices_buffer
|
||||
|
||||
self.mla_attn = MLAAttention(
|
||||
num_heads=self.num_heads,
|
||||
scale=scale,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
q_lora_rank=self.q_lora_rank,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
dcp_q_replicate=self.dcp_q_replicate,
|
||||
use_sparse=self.is_sparse,
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
index_group_builder=mla_modules.index_group_builder,
|
||||
non_causal_multi_token_decode=non_causal_multi_token_decode,
|
||||
)
|
||||
indexer_op = getattr(self.indexer, "indexer_op", None)
|
||||
if indexer_op is not None and hasattr(
|
||||
indexer_op, "dense_mha_metadata_layer_name"
|
||||
):
|
||||
enable_short_prefill_scoring_skip = (
|
||||
allow_short_prefill_indexer_scoring_skip
|
||||
and not self.skip_topk
|
||||
and not getattr(indexer_op, "use_pcp", False)
|
||||
and current_platform.is_cuda()
|
||||
)
|
||||
# The indexer and main MLA use independent decode thresholds and
|
||||
# may classify the same short extend differently. Bind the main
|
||||
# MLA layer name so the eager indexer op can check whether the
|
||||
# batch's top-k indices will be consumed.
|
||||
# PCP is excluded because indexer cache/scoring ownership differs
|
||||
# across ranks and the no-consumer invariant has not been
|
||||
# established there.
|
||||
indexer_op.dense_mha_metadata_layer_name = (
|
||||
self.mla_attn.layer_name if enable_short_prefill_scoring_skip else ""
|
||||
)
|
||||
self.prefix = prefix
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
llama_4_scaling: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
q_c = None
|
||||
kv_lora = None
|
||||
|
||||
if self.q_lora_rank is not None:
|
||||
assert self.fused_qkv_a_proj is not None, (
|
||||
"fused_qkv_a_proj is required when q_lora_rank is not None"
|
||||
)
|
||||
assert self.q_a_layernorm is not None, (
|
||||
"q_a_layernorm is required when q_lora_rank is not None"
|
||||
)
|
||||
assert self.q_b_proj is not None, (
|
||||
"q_b_proj is required when q_lora_rank is not None"
|
||||
)
|
||||
|
||||
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
|
||||
q_c, kv_lora = qkv_lora.split(
|
||||
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
|
||||
dim=-1,
|
||||
)
|
||||
q_proj_layer = self.q_b_proj
|
||||
if not self.fuse_qkv_rmsnorm:
|
||||
q_c = self.q_a_layernorm(q_c)
|
||||
q_proj_input = q_c
|
||||
else:
|
||||
assert self.kv_a_proj_with_mqa is not None, (
|
||||
"kv_a_proj_with_mqa is required when q_lora_rank is None"
|
||||
)
|
||||
assert self.q_proj is not None, (
|
||||
"q_proj is required when q_lora_rank is None"
|
||||
)
|
||||
kv_lora = self.kv_a_proj_with_mqa(hidden_states)[0]
|
||||
q_proj_layer = self.q_proj
|
||||
q_proj_input = hidden_states
|
||||
|
||||
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
|
||||
if self.fuse_qkv_rmsnorm and q_c is not None:
|
||||
# Fuse q_a + kv_a RMSNorm into one kernel launch (q_c is still the
|
||||
# pre-norm projection output here).
|
||||
assert self.q_a_layernorm is not None
|
||||
q_proj_input, kv_c_normed = fused_q_kv_rmsnorm(
|
||||
q_c,
|
||||
kv_c,
|
||||
self.q_a_layernorm.weight.data,
|
||||
self.kv_a_layernorm.weight.data,
|
||||
self.q_a_layernorm.variance_epsilon,
|
||||
)
|
||||
q_c = q_proj_input
|
||||
else:
|
||||
kv_c_normed = self.kv_a_layernorm(kv_c)
|
||||
# Add head dim of 1 to k_pe
|
||||
k_pe = k_pe.unsqueeze(1)
|
||||
|
||||
q = q_proj_layer(q_proj_input)[0]
|
||||
heads = self.num_heads
|
||||
if self.dcp_q_replicate:
|
||||
heads *= q_proj_layer.group_size
|
||||
q = q.view(-1, heads, self.qk_head_dim)
|
||||
|
||||
if self.rotary_emb is not None:
|
||||
q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb(
|
||||
positions, q[..., self.qk_nope_head_dim :], k_pe
|
||||
)
|
||||
|
||||
if self.indexer and self.is_sparse and not self.skip_topk:
|
||||
self.indexer(hidden_states, q_c, positions, self.indexer_rope_emb)
|
||||
if self.is_sparse:
|
||||
self.mla_attn.impl.record_logical_topk_ready() # type: ignore[attr-defined]
|
||||
|
||||
if llama_4_scaling is not None:
|
||||
q *= llama_4_scaling
|
||||
|
||||
q_dcp_replicated = None
|
||||
if self.dcp_q_replicate:
|
||||
q_dcp_replicated, q = q, q_proj_layer._local_view(q)
|
||||
|
||||
attn_out = self.mla_attn(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_shape=(hidden_states.shape[0], self.num_heads * self.v_head_dim),
|
||||
q_dcp_replicated=q_dcp_replicated,
|
||||
)
|
||||
|
||||
if self.g_proj is not None:
|
||||
attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid()
|
||||
|
||||
return self.o_proj(attn_out)[0]
|
||||
@@ -29,13 +29,10 @@ QuantizationMethods = Literal[
|
||||
"humming",
|
||||
"compressed-tensors",
|
||||
"experts_int8",
|
||||
"quark",
|
||||
"moe_wna16",
|
||||
"torchao",
|
||||
"inc",
|
||||
"mxfp4",
|
||||
"gpt_oss_mxfp4",
|
||||
"deepseek_v4_fp8",
|
||||
"online",
|
||||
# Below are online quant shorthand names (see vllm.config.quantization).
|
||||
# Listed here as strings to avoid a circular import; kept in sync with
|
||||
@@ -115,21 +112,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
|
||||
# lazy import to avoid triggering `torch.compile` too early
|
||||
from vllm.config.quantization import _ONLINE_SHORTHANDS
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
|
||||
if current_platform.is_cuda() or current_platform.is_rocm():
|
||||
# The v4.1 class is the v4 one extended to accept model_type
|
||||
# "deepseek_v41" and its 32x32 MXFP8 linear layout. V4.1 has
|
||||
# platform-specific implementations for both CUDA and ROCm.
|
||||
from vllm.models.deepseek_v41 import (
|
||||
DeepseekV4FP8Config as DeepseekV41FP8Config,
|
||||
)
|
||||
|
||||
deepseek_config: type[QuantizationConfig] = DeepseekV41FP8Config
|
||||
else:
|
||||
from vllm.models.deepseek_v4 import DeepseekV4FP8Config
|
||||
|
||||
deepseek_config = DeepseekV4FP8Config
|
||||
|
||||
from .auto_awq import AutoAWQConfig
|
||||
from .auto_gptq import AutoGPTQConfig
|
||||
@@ -141,7 +123,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
from .fp8 import Fp8Config
|
||||
from .fp_quant import FPQuantConfig
|
||||
from .humming import HummingConfig
|
||||
from .inc import INCConfig
|
||||
from .modelopt import (
|
||||
ModelOptFp8Config,
|
||||
ModelOptMixedPrecisionConfig,
|
||||
@@ -169,13 +150,10 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"gptq_marlin": AutoGPTQConfig,
|
||||
"compressed-tensors": CompressedTensorsConfig,
|
||||
"experts_int8": ExpertsInt8Config,
|
||||
"quark": QuarkConfig,
|
||||
"moe_wna16": MoeWNA16Config,
|
||||
"torchao": TorchAOConfig,
|
||||
"inc": INCConfig,
|
||||
"mxfp4": Mxfp4Config,
|
||||
"gpt_oss_mxfp4": GptOssMxfp4Config,
|
||||
"deepseek_v4_fp8": deepseek_config,
|
||||
"humming": HummingConfig,
|
||||
"online": OnlineQuantizationConfig,
|
||||
# MiniMax-style checkpoints tag `quant_method: "mxfp8"`; load with the
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .inc import INCConfig
|
||||
|
||||
__all__ = ["INCConfig"]
|
||||
@@ -1,237 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from .inc import INCConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class INCLayerConfig:
|
||||
bits: int
|
||||
group_size: int | tuple[int, int]
|
||||
sym: bool
|
||||
packing_format: str
|
||||
backend: str
|
||||
data_type: str
|
||||
quantized: bool
|
||||
|
||||
@property
|
||||
def is_gptq(self) -> bool:
|
||||
return "gptq" in self.packing_format or "gptq" in self.backend
|
||||
|
||||
@property
|
||||
def is_awq(self) -> bool:
|
||||
return "awq" in self.packing_format or "awq" in self.backend
|
||||
|
||||
@property
|
||||
def is_wna16_int(self) -> bool:
|
||||
return self.data_type == "int" and self.quantized
|
||||
|
||||
@property
|
||||
def is_mxfp4(self) -> bool:
|
||||
return "mx_fp" in self.data_type and self.bits == 4
|
||||
|
||||
@property
|
||||
def is_mxfp8(self) -> bool:
|
||||
return "mx_fp" in self.data_type and self.bits == 8
|
||||
|
||||
@property
|
||||
def is_fp8_block(self) -> bool:
|
||||
return (
|
||||
self.data_type == "fp"
|
||||
and self.bits == 8
|
||||
and self.sym
|
||||
and self.quantized
|
||||
and self.packing_format == "auto_round:fp8"
|
||||
and isinstance(self.group_size, tuple)
|
||||
and len(self.group_size) == 2
|
||||
)
|
||||
|
||||
|
||||
class INCConfigParser:
|
||||
def __init__(self, config: "INCConfig") -> None:
|
||||
self._config = config
|
||||
|
||||
def resolve(self, layer: "torch.nn.Module", layer_name: str) -> INCLayerConfig:
|
||||
bits, group_size, sym = self._resolve_raw(layer, layer_name)
|
||||
return INCLayerConfig(
|
||||
bits=bits,
|
||||
group_size=group_size,
|
||||
sym=sym,
|
||||
packing_format=self._config.packing_format,
|
||||
backend=self._config.backend,
|
||||
data_type=self._config.data_type,
|
||||
quantized=bits < 16,
|
||||
)
|
||||
|
||||
def get_layer_config(
|
||||
self, layer: "torch.nn.Module", layer_name: str
|
||||
) -> tuple[int, int | tuple[int, int], bool]:
|
||||
layer_config = self.resolve(layer, layer_name)
|
||||
return layer_config.bits, layer_config.group_size, layer_config.sym
|
||||
|
||||
def _resolve_raw(
|
||||
self, layer: "torch.nn.Module", layer_name: str
|
||||
) -> tuple[int, int | tuple[int, int], bool]:
|
||||
REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
|
||||
|
||||
def is_explicitly_configured(name: str) -> bool:
|
||||
"""Return True if *name* has an explicit entry in extra_config,
|
||||
either via exact key match or via a regex pattern key."""
|
||||
if not self._config.extra_config:
|
||||
return False
|
||||
if name in self._config.extra_config:
|
||||
return True
|
||||
for pattern in self._config.extra_config:
|
||||
if not isinstance(pattern, str) or not any(
|
||||
c in REGEX_SPECIAL_CHARS for c in pattern
|
||||
):
|
||||
continue
|
||||
try:
|
||||
if re.search(re.compile(pattern), name) is not None:
|
||||
return True
|
||||
except re.error:
|
||||
continue
|
||||
return False
|
||||
|
||||
def get_config(
|
||||
name: str, quantized: bool = True
|
||||
) -> tuple[int, int | tuple[int, int], bool]:
|
||||
if not self._config.extra_config:
|
||||
return (
|
||||
self._config.weight_bits if quantized else 16,
|
||||
self._config.group_size if quantized else -1,
|
||||
self._config.sym if quantized else True,
|
||||
)
|
||||
|
||||
if name in self._config.extra_config:
|
||||
cfg = self._config.extra_config[name]
|
||||
return (
|
||||
cfg.get("bits", self._config.weight_bits if quantized else 16),
|
||||
self._normalize_group_size(
|
||||
cfg.get(
|
||||
"group_size",
|
||||
self._config.group_size if quantized else -1,
|
||||
)
|
||||
),
|
||||
cfg.get("sym", self._config.sym if quantized else True),
|
||||
)
|
||||
|
||||
regex_special_chars = set(r"*+?^$()[]{}|\\")
|
||||
for pattern, cfg in self._config.extra_config.items():
|
||||
if not isinstance(pattern, str) or not any(
|
||||
c in regex_special_chars for c in pattern
|
||||
):
|
||||
continue
|
||||
|
||||
try:
|
||||
if re.search(re.compile(pattern), name) is not None:
|
||||
return (
|
||||
cfg.get(
|
||||
"bits",
|
||||
self._config.weight_bits if quantized else 16,
|
||||
),
|
||||
self._normalize_group_size(
|
||||
cfg.get(
|
||||
"group_size",
|
||||
self._config.group_size if quantized else -1,
|
||||
)
|
||||
),
|
||||
cfg.get("sym", self._config.sym if quantized else True),
|
||||
)
|
||||
except re.error:
|
||||
continue
|
||||
|
||||
return (
|
||||
self._config.weight_bits if quantized else 16,
|
||||
self._config.group_size if quantized else -1,
|
||||
self._config.sym if quantized else True,
|
||||
)
|
||||
|
||||
if self._config.extra_config and layer_name in self._config.extra_config:
|
||||
return get_config(layer_name)
|
||||
|
||||
# Suffix match: handle cases where extra_config keys use short names
|
||||
# (e.g. "lm_head") but the layer_name is fully qualified
|
||||
# (e.g. "model.language_model.lm_head") due to model nesting.
|
||||
if self._config.extra_config:
|
||||
for cfg_key in self._config.extra_config:
|
||||
if layer_name.endswith(f".{cfg_key}"):
|
||||
return get_config(cfg_key)
|
||||
|
||||
quantized = not isinstance(layer, ParallelLMHead)
|
||||
if self._config.block_name_to_quantize:
|
||||
quantized = any(
|
||||
layer_name.startswith(name)
|
||||
for name in self._config.block_name_to_quantize
|
||||
)
|
||||
|
||||
if self._config.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
|
||||
moe_configs = [
|
||||
get_config(name, quantized)
|
||||
for name in self._config.extra_config
|
||||
if name.startswith(layer_name)
|
||||
]
|
||||
if moe_configs:
|
||||
if len(set(moe_configs)) == 1:
|
||||
return moe_configs[0]
|
||||
raise ValueError(
|
||||
f"Fused MoE layer '{layer_name}' requires "
|
||||
f"consistent quant config for all sub-layers"
|
||||
)
|
||||
|
||||
if self._config.extra_config:
|
||||
for fusion_key, sub_keys in self._config.packed_modules_mapping.items():
|
||||
if fusion_key in layer_name and layer_name.count(fusion_key) == 1:
|
||||
sub_names = [
|
||||
layer_name.replace(fusion_key, sub_key) for sub_key in sub_keys
|
||||
]
|
||||
# Only trigger if at least one sub_name is explicitly
|
||||
# configured in extra_config (via exact match or regex).
|
||||
# This prevents false matches when a short fusion_key
|
||||
# (e.g. "qkv") is merely a substring of a longer layer
|
||||
# name (e.g. "in_proj_qkvz") and none of the generated
|
||||
# sub_names are actually configured.
|
||||
if not any(is_explicitly_configured(n) for n in sub_names):
|
||||
continue
|
||||
sub_configs = [get_config(name, quantized) for name in sub_names]
|
||||
if len(set(sub_configs)) == 1:
|
||||
return sub_configs[0]
|
||||
raise ValueError(
|
||||
f"Fused module '{layer_name}' requires "
|
||||
f"consistent quant config for {sub_names}"
|
||||
)
|
||||
|
||||
return get_config(layer_name, quantized)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_group_size(
|
||||
group_size: int | list[int] | tuple[int, int],
|
||||
) -> int | tuple[int, int]:
|
||||
"""Normalize INC group_size into either an int or a 2-D int tuple."""
|
||||
if isinstance(group_size, (list, tuple)):
|
||||
if len(group_size) != 2 or not all(
|
||||
isinstance(value, int) for value in group_size
|
||||
):
|
||||
raise ValueError(
|
||||
"INC block-wise FP8 requires group_size to be a 2-D "
|
||||
f"integer sequence, but found {group_size!r}."
|
||||
)
|
||||
return (group_size[0], group_size[1])
|
||||
|
||||
if not isinstance(group_size, int):
|
||||
raise ValueError(
|
||||
"INC group_size must be an int or a 2-D integer sequence, "
|
||||
f"but found {type(group_size).__name__}."
|
||||
)
|
||||
|
||||
return group_size
|
||||
@@ -1,368 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fractions import Fraction
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
QuantizationMethods,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
from .config_parser import INCConfigParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class INCConfig(QuantizationConfig):
|
||||
"""Config class for Intel Neural Compressor (INC).
|
||||
Repo: https://github.com/intel/neural-compressor
|
||||
"""
|
||||
|
||||
SUPPORTED_BITS = {2, 3, 4, 5, 6, 7, 8}
|
||||
DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
|
||||
SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
|
||||
SUPPORTED_FORMATS = {
|
||||
"auto_round:auto_gptq",
|
||||
"auto_round:auto_awq",
|
||||
"auto_round:fp8",
|
||||
"auto_round:llm_compressor",
|
||||
}
|
||||
SUPPORTED_BACKENDS = {
|
||||
"auto",
|
||||
"gptq",
|
||||
"gptq:marlin",
|
||||
"awq",
|
||||
"awq:marlin",
|
||||
"marlin",
|
||||
}
|
||||
|
||||
FP8_BLOCK_BITS = 8
|
||||
FP8_BLOCK_DATA_TYPE = "fp"
|
||||
FP8_BLOCK_PACKING_FORMAT = "auto_round:fp8"
|
||||
FP8_BLOCK_FMT = "e4m3"
|
||||
FP8_BLOCK_ACTIVATION_SCHEME = "dynamic"
|
||||
|
||||
MXFP8_BITS = 8
|
||||
MXFP8_GROUP_SIZE = 32
|
||||
MXFP8_DATA_TYPE = "mx_fp"
|
||||
MXFP8_PACKING_FORMAT = "auto_round:llm_compressor"
|
||||
|
||||
@staticmethod
|
||||
def _normalize_data_type(data_type: str) -> str:
|
||||
# Keep a small alias for branches that already experimented with "fp8".
|
||||
return "fp" if data_type == "fp8" else data_type
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_bits: int,
|
||||
group_size: int | tuple[int, int],
|
||||
sym: bool = True,
|
||||
packing_format: str = DEFAULT_INT_PACKING_FORMAT,
|
||||
block_name_to_quantize: str | list[str] | None = None,
|
||||
extra_config: dict[str, Any] | None = None,
|
||||
data_type: str = "int",
|
||||
backend: str = "auto",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
data_type = self._normalize_data_type(data_type)
|
||||
group_size = INCConfigParser._normalize_group_size(group_size)
|
||||
|
||||
if weight_bits not in self.SUPPORTED_BITS:
|
||||
raise ValueError(
|
||||
f"Unsupported weight_bits: {weight_bits}, "
|
||||
f"currently only support {self.SUPPORTED_BITS}."
|
||||
)
|
||||
# auto-round mxfp data_type is e.g. "mx_fp4" / "mx_fp4e2m1"; match the
|
||||
# "mx_fp" family by substring like auto_round.compressors.is_mx_fp.
|
||||
is_mxfp = "mx_fp" in data_type
|
||||
if data_type not in self.SUPPORTED_DTYPES and not is_mxfp:
|
||||
raise ValueError(
|
||||
f"Unsupported data_type: {data_type}, "
|
||||
f"currently only support {self.SUPPORTED_DTYPES}."
|
||||
)
|
||||
if packing_format not in self.SUPPORTED_FORMATS:
|
||||
raise ValueError(
|
||||
f"Unsupported packing_format: {packing_format}, "
|
||||
f"currently only support {self.SUPPORTED_FORMATS}."
|
||||
)
|
||||
if backend not in self.SUPPORTED_BACKENDS:
|
||||
raise ValueError(
|
||||
f"Unsupported backend: {backend}, "
|
||||
f"currently only support {self.SUPPORTED_BACKENDS}."
|
||||
)
|
||||
|
||||
self.weight_bits = weight_bits
|
||||
self.group_size = group_size
|
||||
self.sym = sym
|
||||
self.packing_format = packing_format
|
||||
self.block_name_to_quantize = (
|
||||
block_name_to_quantize.split(",")
|
||||
if isinstance(block_name_to_quantize, str)
|
||||
else block_name_to_quantize
|
||||
)
|
||||
self.extra_config = extra_config
|
||||
self.data_type = data_type
|
||||
self.backend = backend
|
||||
self.pack_factor = Fraction(32, weight_bits)
|
||||
self.config_parser = INCConfigParser(self)
|
||||
|
||||
self._validate_supported_quantization()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"INCConfig(weight_bits={self.weight_bits}, "
|
||||
f"group_size={self.group_size}, sym={self.sym})"
|
||||
)
|
||||
|
||||
@property
|
||||
def is_mxfp(self) -> bool:
|
||||
# auto-round mxfp data_type is the "mx_fp" family (e.g. "mx_fp",
|
||||
# "mx_fp4e2m1"); match by substring like auto_round.compressors.is_mx_fp.
|
||||
return "mx_fp" in self.data_type
|
||||
|
||||
@property
|
||||
def is_mxfp8(self) -> bool:
|
||||
# MXFP4 and MXFP8 share data_type "mx_fp" and differ only by bit width.
|
||||
return self.is_mxfp and self.weight_bits == self.MXFP8_BITS
|
||||
|
||||
@classmethod
|
||||
def _check_expected(
|
||||
cls,
|
||||
values: dict[str, Any],
|
||||
expected_fields: dict[str, Any],
|
||||
quantization_name: str,
|
||||
) -> None:
|
||||
for field_name, expected_value in expected_fields.items():
|
||||
actual_value = cls.get_from_keys_or(values, [field_name], expected_value)
|
||||
assert actual_value == expected_value, (
|
||||
f"{quantization_name} only supports "
|
||||
f"{field_name}={expected_value!r}, "
|
||||
f"but found {field_name}={actual_value!r}."
|
||||
)
|
||||
|
||||
def _validate_int_quantization(self) -> None:
|
||||
int_packing_formats = {
|
||||
self.DEFAULT_INT_PACKING_FORMAT,
|
||||
"auto_round:auto_awq",
|
||||
}
|
||||
assert isinstance(self.group_size, int), (
|
||||
"INC int quantization requires scalar group_size, "
|
||||
f"but found group_size={self.group_size!r}."
|
||||
)
|
||||
assert self.packing_format in int_packing_formats, (
|
||||
"INC int quantization only supports "
|
||||
f"packing_format in {sorted(int_packing_formats)!r}, "
|
||||
f"but found packing_format={self.packing_format!r}."
|
||||
)
|
||||
|
||||
def _validate_fp8_block_quantization(self) -> None:
|
||||
assert self.weight_bits == self.FP8_BLOCK_BITS, (
|
||||
"INC block-wise FP8 only supports bits=8, "
|
||||
f"but found bits={self.weight_bits}."
|
||||
)
|
||||
assert isinstance(self.group_size, tuple), (
|
||||
"INC block-wise FP8 requires a 2-D group_size, "
|
||||
f"but found group_size={self.group_size!r}."
|
||||
)
|
||||
assert self.sym, "INC block-wise FP8 only supports symmetric weights."
|
||||
assert self.packing_format == self.FP8_BLOCK_PACKING_FORMAT, (
|
||||
"INC block-wise FP8 only supports "
|
||||
f"packing_format={self.FP8_BLOCK_PACKING_FORMAT!r}, "
|
||||
f"but found {self.packing_format!r}."
|
||||
)
|
||||
assert self.backend == "auto", (
|
||||
"INC block-wise FP8 only supports backend='auto', "
|
||||
f"but found backend={self.backend!r}."
|
||||
)
|
||||
|
||||
def _validate_mxfp_quantization(self) -> None:
|
||||
assert self.weight_bits in {4, self.MXFP8_BITS}, (
|
||||
"INC MXFP only supports bits=4 or bits=8, "
|
||||
f"but found bits={self.weight_bits}."
|
||||
)
|
||||
assert self.group_size == self.MXFP8_GROUP_SIZE, (
|
||||
"INC MXFP only supports group_size=32, "
|
||||
f"but found group_size={self.group_size}."
|
||||
)
|
||||
assert self.sym, "INC MXFP only supports symmetric weights."
|
||||
assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
|
||||
"INC MXFP only supports "
|
||||
f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
|
||||
f"but found {self.packing_format!r}."
|
||||
)
|
||||
assert self.backend == "auto", (
|
||||
"INC MXFP only supports backend='auto', "
|
||||
f"but found backend={self.backend!r}."
|
||||
)
|
||||
|
||||
def _validate_supported_quantization(self) -> None:
|
||||
if self.data_type == "int":
|
||||
self._validate_int_quantization()
|
||||
elif self.data_type == self.FP8_BLOCK_DATA_TYPE:
|
||||
self._validate_fp8_block_quantization()
|
||||
elif self.data_type == self.MXFP8_DATA_TYPE:
|
||||
self._validate_mxfp_quantization()
|
||||
|
||||
def _validate_raw_config(self, config: dict[str, Any]) -> None:
|
||||
if self.data_type == self.FP8_BLOCK_DATA_TYPE:
|
||||
assert isinstance(self.group_size, tuple), (
|
||||
"INC block-wise FP8 group_size must be a tuple."
|
||||
)
|
||||
|
||||
expected_fields = {
|
||||
"act_bits": self.FP8_BLOCK_BITS,
|
||||
"act_group_size": self.group_size[0],
|
||||
"act_data_type": self.FP8_BLOCK_DATA_TYPE,
|
||||
"act_sym": True,
|
||||
"act_dynamic": True,
|
||||
"activation_scheme": self.FP8_BLOCK_ACTIVATION_SCHEME,
|
||||
"fmt": self.FP8_BLOCK_FMT,
|
||||
}
|
||||
self._check_expected(
|
||||
config,
|
||||
expected_fields,
|
||||
"INC block-wise FP8",
|
||||
)
|
||||
elif self.is_mxfp8:
|
||||
expected_fields = {
|
||||
"act_bits": self.MXFP8_BITS,
|
||||
"act_data_type": self.MXFP8_DATA_TYPE,
|
||||
"act_group_size": self.MXFP8_GROUP_SIZE,
|
||||
"act_sym": True,
|
||||
"act_dynamic": True,
|
||||
}
|
||||
self._check_expected(
|
||||
config,
|
||||
expected_fields,
|
||||
"INC MXFP8",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "inc"
|
||||
|
||||
@classmethod
|
||||
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
|
||||
return [torch.half, torch.bfloat16]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 60
|
||||
|
||||
@classmethod
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return ["quantization_config.json"]
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
|
||||
data_type = cls._normalize_data_type(
|
||||
cls.get_from_keys_or(config, ["data_type"], "int")
|
||||
)
|
||||
weight_bits = cls.get_from_keys(config, ["bits"])
|
||||
group_size = INCConfigParser._normalize_group_size(
|
||||
cls.get_from_keys(config, ["group_size"])
|
||||
)
|
||||
packing_format = cls.get_from_keys_or(config, ["packing_format"], None)
|
||||
|
||||
if packing_format is None:
|
||||
if (
|
||||
data_type == cls.FP8_BLOCK_DATA_TYPE
|
||||
and weight_bits == cls.FP8_BLOCK_BITS
|
||||
and isinstance(group_size, tuple)
|
||||
):
|
||||
packing_format = cls.FP8_BLOCK_PACKING_FORMAT
|
||||
elif "mx_fp" in data_type:
|
||||
packing_format = cls.MXFP8_PACKING_FORMAT
|
||||
else:
|
||||
packing_format = cls.DEFAULT_INT_PACKING_FORMAT
|
||||
|
||||
quant_config = cls(
|
||||
weight_bits=weight_bits,
|
||||
group_size=group_size,
|
||||
sym=cls.get_from_keys_or(config, ["sym"], True),
|
||||
packing_format=packing_format,
|
||||
block_name_to_quantize=cls.get_from_keys_or(
|
||||
config, ["block_name_to_quantize", "to_quant_block_names"], None
|
||||
),
|
||||
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
|
||||
data_type=data_type,
|
||||
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
|
||||
)
|
||||
quant_config._validate_raw_config(config)
|
||||
return quant_config
|
||||
|
||||
def get_layer_config(self, layer, layer_name: str):
|
||||
return self.config_parser.get_layer_config(layer, layer_name)
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
|
||||
if self.block_name_to_quantize is not None:
|
||||
self.block_name_to_quantize = hf_to_vllm_mapper.apply_list(
|
||||
self.block_name_to_quantize
|
||||
)
|
||||
if self.extra_config is not None:
|
||||
self.extra_config = hf_to_vllm_mapper.apply_dict(self.extra_config)
|
||||
|
||||
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
|
||||
from .schemes.factory import resolve_scheme
|
||||
|
||||
if prefix and self.extra_config:
|
||||
for layer_name in self.extra_config:
|
||||
if (
|
||||
layer_name == prefix or layer_name == f"model.{prefix}"
|
||||
) and self.extra_config[layer_name].get("bits", 16) >= 16:
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
return UnquantizedLinearMethod()
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
|
||||
layer_config = self.config_parser.resolve(layer, prefix)
|
||||
if not layer_config.quantized:
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
return UnquantizedLinearMethod()
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
|
||||
logger.debug(
|
||||
"[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
|
||||
prefix,
|
||||
layer.__class__.__name__,
|
||||
layer_config.bits,
|
||||
layer_config.group_size,
|
||||
layer_config.sym,
|
||||
)
|
||||
|
||||
scheme = resolve_scheme(layer_config)
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
return scheme.get_linear_method(self, layer, prefix, layer_config)
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return scheme.get_moe_method(self, layer, prefix, layer_config)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> "QuantizationMethods | None":
|
||||
"""Override the `auto-round` method to `inc`."""
|
||||
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
|
||||
if is_auto_round_format:
|
||||
return cls.get_name()
|
||||
return None
|
||||
@@ -1,47 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.linear import LinearMethodBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .schemes.inc_scheme import INCLinearScheme
|
||||
|
||||
|
||||
class INCLinearMethod(LinearMethodBase):
|
||||
def __init__(self, scheme: "INCLinearScheme") -> None:
|
||||
self.scheme = scheme
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
return self.scheme.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
return self.scheme.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.scheme.apply_weights(layer, x, bias)
|
||||
@@ -1,19 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .factory import resolve_scheme
|
||||
from .inc_fp8_scheme import INCFp8Scheme
|
||||
from .inc_mxfp4_scheme import INCMxfp4Scheme
|
||||
from .inc_mxfp8_scheme import INCMxfp8Scheme
|
||||
from .inc_scheme import INCLinearScheme, INCScheme
|
||||
from .inc_wna16_scheme import INCWna16Scheme
|
||||
|
||||
__all__ = [
|
||||
"INCScheme",
|
||||
"INCLinearScheme",
|
||||
"INCMxfp8Scheme",
|
||||
"INCFp8Scheme",
|
||||
"INCWna16Scheme",
|
||||
"INCMxfp4Scheme",
|
||||
"resolve_scheme",
|
||||
]
|
||||
@@ -1,28 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..config_parser import INCLayerConfig
|
||||
from .inc_scheme import INCScheme
|
||||
|
||||
|
||||
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
|
||||
from .inc_fp8_scheme import INCFp8Scheme
|
||||
from .inc_mxfp4_scheme import INCMxfp4Scheme
|
||||
from .inc_mxfp8_scheme import INCMxfp8Scheme
|
||||
from .inc_wna16_scheme import INCWna16Scheme
|
||||
|
||||
scheme_list: list[type[INCScheme]] = [
|
||||
INCFp8Scheme,
|
||||
INCMxfp8Scheme,
|
||||
INCWna16Scheme,
|
||||
INCMxfp4Scheme,
|
||||
]
|
||||
|
||||
for scheme_cls in scheme_list:
|
||||
if scheme_cls.can_handle(layer_config):
|
||||
return scheme_cls()
|
||||
|
||||
raise NotImplementedError(f"No INC scheme found for layer config: {layer_config}")
|
||||
@@ -1,123 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_OPS_REGISTERED = False
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_ark_state() -> tuple[bool, str | None, Any | None, Any | None]:
|
||||
"""Return ARK availability, error details, cached module, and QuantLinear."""
|
||||
try:
|
||||
import auto_round_kernel as ark
|
||||
from auto_round_kernel.qlinear import QuantLinear
|
||||
|
||||
logger.info("Successfully imported auto_round_kernel.")
|
||||
except ImportError as error:
|
||||
return False, str(error), None, None
|
||||
|
||||
if getattr(ark, "cpu_lib", None) is None and getattr(ark, "xpu_lib", None) is None:
|
||||
return (
|
||||
False,
|
||||
"No ARK backend library is available.",
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
logger.info("Successfully loaded auto_round_kernel backend library.")
|
||||
return True, None, ark, QuantLinear
|
||||
|
||||
|
||||
def _inc_ark_woq_linear_impl(
|
||||
x: torch.Tensor,
|
||||
qweight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
out_features: int,
|
||||
in_features: int,
|
||||
group_size: int,
|
||||
compute_type: str,
|
||||
weight_type: str,
|
||||
scale_type: str,
|
||||
asym: bool,
|
||||
) -> torch.Tensor:
|
||||
ark = get_ark_state()[2]
|
||||
assert ark is not None
|
||||
|
||||
return ark.woqgemm_linear(
|
||||
x,
|
||||
qweight,
|
||||
bias,
|
||||
out_features,
|
||||
in_features,
|
||||
group_size,
|
||||
compute_type,
|
||||
weight_type,
|
||||
scale_type,
|
||||
asym,
|
||||
)
|
||||
|
||||
|
||||
def _inc_ark_woq_linear_fake(
|
||||
x: torch.Tensor,
|
||||
qweight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
out_features: int,
|
||||
in_features: int,
|
||||
group_size: int,
|
||||
compute_type: str,
|
||||
weight_type: str,
|
||||
scale_type: str,
|
||||
asym: bool,
|
||||
) -> torch.Tensor:
|
||||
del qweight
|
||||
del bias
|
||||
del in_features
|
||||
del group_size
|
||||
del compute_type
|
||||
del weight_type
|
||||
del scale_type
|
||||
del asym
|
||||
return torch.empty(
|
||||
(*x.shape[:-1], out_features),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
|
||||
|
||||
class ark_ops:
|
||||
@staticmethod
|
||||
def register_ops_once() -> None:
|
||||
global _OPS_REGISTERED
|
||||
if _OPS_REGISTERED:
|
||||
return
|
||||
|
||||
is_available, error_str, _, _ = get_ark_state()
|
||||
if not is_available:
|
||||
logger.debug(
|
||||
"Skip registering ark op because ARK is unavailable: %s",
|
||||
error_str or "unknown error",
|
||||
)
|
||||
return
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="inc_ark_woq_linear",
|
||||
op_func=_inc_ark_woq_linear_impl,
|
||||
fake_impl=_inc_ark_woq_linear_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
_OPS_REGISTERED = True
|
||||
|
||||
|
||||
ark_ops.register_ops_once()
|
||||
|
||||
__all__ = ["get_ark_state"]
|
||||
@@ -1,57 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_input_dtype,
|
||||
)
|
||||
|
||||
from .inc_scheme import INCLinearScheme
|
||||
|
||||
|
||||
class INCFp8LinearScheme(INCLinearScheme):
|
||||
def __init__(self, prefix: str, weight_block_size: tuple[int, int]) -> None:
|
||||
self.quant_config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
activation_scheme="dynamic",
|
||||
weight_block_size=list(weight_block_size),
|
||||
)
|
||||
self.linear_method = Fp8LinearMethod(self.quant_config)
|
||||
self.linear_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return Fp8Config.get_min_capability()
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
self.linear_method.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.linear_method.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.linear_method.apply(layer, x, bias)
|
||||
@@ -1,39 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..inc_linear import INCLinearMethod
|
||||
from .inc_scheme import INCScheme
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from ..config_parser import INCLayerConfig
|
||||
from ..inc import INCConfig
|
||||
|
||||
|
||||
class INCFp8Scheme(INCScheme):
|
||||
@staticmethod
|
||||
def can_handle(layer_config: "INCLayerConfig") -> bool:
|
||||
return layer_config.is_fp8_block
|
||||
|
||||
def get_linear_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, layer
|
||||
|
||||
assert isinstance(layer_config.group_size, tuple)
|
||||
|
||||
from .inc_fp8_linear import INCFp8LinearScheme
|
||||
|
||||
return INCLinearMethod(
|
||||
INCFp8LinearScheme(
|
||||
prefix=prefix,
|
||||
weight_block_size=layer_config.group_size,
|
||||
)
|
||||
)
|
||||
@@ -1,97 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
)
|
||||
|
||||
from .inc_scheme import INCLinearScheme
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..config_parser import INCLayerConfig
|
||||
|
||||
|
||||
class INCMxfp4LinearMethod(INCLinearScheme):
|
||||
"""MXFP4 (W4A4) linear method for AutoRound checkpoints.
|
||||
|
||||
E2M1 weights packed two per byte with per-group E8M0 scales
|
||||
(group_size=32, no global scale). The platform kernel is selected by
|
||||
``init_mxfp4_linear_kernel`` (FlashInfer / Marlin on CUDA, ``fp4_gemm``
|
||||
on XPU).
|
||||
"""
|
||||
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
if not isinstance(layer_config.group_size, int):
|
||||
raise ValueError(
|
||||
"INC MXFP4 requires scalar group_size, "
|
||||
f"but found group_size={layer_config.group_size!r}."
|
||||
)
|
||||
self.group_size = layer_config.group_size or 32
|
||||
self.kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic)
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 80
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs: Any,
|
||||
) -> None:
|
||||
del input_size, output_size
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.params_dtype = params_dtype
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // 2,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_packed", weight)
|
||||
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // self.group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight = Parameter(layer.weight_packed.data, requires_grad=False)
|
||||
del layer.weight_packed
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
@@ -1,252 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config.kernel import FLASHINFER_MOE_EP_BACKENDS
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
mxfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsMxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
MarlinExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
B12X_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_moe_fp4_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""W4A4 MXFP4 group MoE for AutoRound ``auto_round:llm_compressor`` exports.
|
||||
|
||||
Registers the packed MXFP4 layout (uint8 ``weight_packed`` + uint8 E8M0
|
||||
``weight_scale``, ``group_size=32``) and dispatches the fused MoE to b12x
|
||||
when requested; otherwise it uses CUTLASS W4A4, XPU W4A4, or Marlin W4A16.
|
||||
The per-expert ``gate_proj`` / ``up_proj`` / ``down_proj`` tensors are
|
||||
folded into the stacked ``w13`` / ``w2`` parameters by
|
||||
``make_expert_params_mapping``.
|
||||
"""
|
||||
|
||||
def __init__(self, moe) -> None:
|
||||
super().__init__(moe)
|
||||
if moe.moe_backend in FLASHINFER_MOE_EP_BACKENDS:
|
||||
raise ValueError(
|
||||
f"moe_backend={moe.moe_backend!r} is not supported for AutoRound "
|
||||
"(INC) MXFP4 MoE checkpoints"
|
||||
)
|
||||
self.group_size = 32
|
||||
# Backend selection must stay consistent with the weight preparation in
|
||||
# process_weights_after_loading / get_fused_moe_quant_config, which use
|
||||
# three layouts: CUTLASS swizzle (true W4A4), b12x and XPU packed
|
||||
# passthrough, and Marlin weight-only. XPU dispatch is deferred to the
|
||||
# shared oracle; b12x dispatch is explicit; every remaining non-CUTLASS
|
||||
# device falls back to Marlin.
|
||||
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None = None
|
||||
if moe.moe_backend == "b12x":
|
||||
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
|
||||
self.use_cutlass_mxfp4 = False
|
||||
elif self.use_cutlass_mxfp4:
|
||||
self.experts_cls = CutlassExpertsMxfp4
|
||||
logger.info_once("Using CutlassExpertsMxfp4 for AutoRound MXFP4 MoE")
|
||||
elif current_platform.is_xpu():
|
||||
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
|
||||
else:
|
||||
self.experts_cls = MarlinExperts
|
||||
logger.info_once(
|
||||
"Using MarlinExperts (weight-only FP4) for AutoRound MXFP4 MoE"
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
layer.num_experts = num_experts
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
# gate + up fused on the output dim; two FP4 packed per input byte.
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // 2,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_packed", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // 2,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_packed", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
# Per-group E8M0 block scales (group_size=32), stored as uint8.
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // self.group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
extra_weight_attrs.update(
|
||||
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
|
||||
)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // self.group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
if self.use_cutlass_mxfp4:
|
||||
# W4A4: both weights and activations quantized to MXFP4.
|
||||
return mxfp4_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
)
|
||||
# b12x selects W4A8 or W4A16; XPU uses W4A4; Marlin uses W4A16.
|
||||
return make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
layer.w13_weight = torch.nn.Parameter(
|
||||
layer.w13_weight_packed.data, requires_grad=False
|
||||
)
|
||||
delattr(layer, "w13_weight_packed")
|
||||
layer.w2_weight = torch.nn.Parameter(
|
||||
layer.w2_weight_packed.data, requires_grad=False
|
||||
)
|
||||
delattr(layer, "w2_weight_packed")
|
||||
|
||||
if self.use_cutlass_mxfp4:
|
||||
# Swizzle weight scales from flat checkpoint layout [E, N, K//32]
|
||||
# to the CUTLASS tiled layout.
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
swizzle_mxfp4_scales,
|
||||
)
|
||||
|
||||
E = layer.w13_weight_scale.shape[0]
|
||||
w13_N = layer.w13_weight_scale.shape[1]
|
||||
w13_scale_K = layer.w13_weight_scale.shape[2]
|
||||
w13_K = w13_scale_K * 32
|
||||
|
||||
w2_M = layer.w2_weight_scale.shape[1]
|
||||
w2_scale_N = layer.w2_weight_scale.shape[2]
|
||||
w2_N = w2_scale_N * 32
|
||||
|
||||
swizzled_w13 = []
|
||||
swizzled_w2 = []
|
||||
for e_idx in range(E):
|
||||
s13 = layer.w13_weight_scale[e_idx]
|
||||
sw13 = swizzle_mxfp4_scales(s13, w13_N, w13_K)
|
||||
swizzled_w13.append(sw13.reshape(w13_N, w13_scale_K))
|
||||
s2 = layer.w2_weight_scale[e_idx]
|
||||
sw2 = swizzle_mxfp4_scales(s2, w2_M, w2_N)
|
||||
swizzled_w2.append(sw2.reshape(w2_M, w2_scale_N))
|
||||
layer.w13_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w13), requires_grad=False
|
||||
)
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w2), requires_grad=False
|
||||
)
|
||||
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
|
||||
pass
|
||||
else:
|
||||
logger.warning_once(
|
||||
"This device lacks native FP4 compute; using weight-only FP4 "
|
||||
"via the Marlin kernel, which may reduce performance for "
|
||||
"compute-heavy workloads."
|
||||
)
|
||||
prepare_moe_fp4_layer_for_marlin(layer)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None:
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
@@ -1,54 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..inc_linear import INCLinearMethod
|
||||
from .inc_scheme import INCScheme
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from ..config_parser import INCLayerConfig
|
||||
from ..inc import INCConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class INCMxfp4Scheme(INCScheme):
|
||||
"""MXFP4 (W4A4) scheme for AutoRound checkpoints.
|
||||
|
||||
Dispatches to :class:`INCMxfp4LinearMethod` for linear layers and
|
||||
:class:`INCMxfp4MoEMethod` for fused MoE layers; see those classes for the
|
||||
per-module weight layout and kernel-selection details.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def can_handle(layer_config: "INCLayerConfig") -> bool:
|
||||
return layer_config.is_mxfp4
|
||||
|
||||
def get_linear_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, layer, prefix
|
||||
from .inc_mxfp4_linear import INCMxfp4LinearMethod
|
||||
|
||||
return INCLinearMethod(INCMxfp4LinearMethod(layer_config))
|
||||
|
||||
def get_moe_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, prefix, layer_config
|
||||
from .inc_mxfp4_moe import INCMxfp4MoEMethod
|
||||
|
||||
return INCMxfp4MoEMethod(layer.moe_config)
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
)
|
||||
|
||||
from .inc_scheme import INCLinearScheme
|
||||
|
||||
|
||||
class INCMxfp8LinearScheme(INCLinearScheme):
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 75
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
del input_size, output_size
|
||||
if input_size_per_partition % MXFP8_BLOCK_SIZE != 0:
|
||||
raise ValueError(
|
||||
"INC MXFP8 requires input_size_per_partition "
|
||||
f"({input_size_per_partition}) to be divisible by "
|
||||
f"{MXFP8_BLOCK_SIZE}."
|
||||
)
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=extra_weight_attrs.get("weight_loader"),
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=extra_weight_attrs.get("weight_loader"),
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
@@ -1,197 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.parameter import ModelWeightParameter
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
|
||||
|
||||
class INCMxfp8MoEMethod(FusedMoEMethodBase):
|
||||
"""W8A8 MXFP8 MoE method for serialized AutoRound checkpoints."""
|
||||
|
||||
def __init__(self, moe) -> None:
|
||||
super().__init__(moe)
|
||||
self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
|
||||
self.mxfp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
assert layer.intermediate_size_per_partition == intermediate_size_per_partition
|
||||
assert layer.hidden_size == hidden_size
|
||||
layer.orig_dtype = params_dtype
|
||||
|
||||
if hidden_size % MXFP8_BLOCK_SIZE != 0:
|
||||
raise ValueError(
|
||||
f"MXFP8 MoE requires hidden_size divisible by {MXFP8_BLOCK_SIZE}, "
|
||||
f"got {hidden_size}."
|
||||
)
|
||||
if intermediate_size_per_partition % MXFP8_BLOCK_SIZE != 0:
|
||||
raise ValueError(
|
||||
"MXFP8 MoE requires intermediate_size_per_partition divisible by "
|
||||
f"{MXFP8_BLOCK_SIZE}, got {intermediate_size_per_partition}."
|
||||
)
|
||||
|
||||
layer.num_experts = num_experts
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
|
||||
w13_weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
|
||||
w2_weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
|
||||
w13_weight_scale = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
|
||||
w2_weight_scale = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
|
||||
set_weight_attrs(
|
||||
layer.w13_weight_scale,
|
||||
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
|
||||
)
|
||||
set_weight_attrs(
|
||||
layer.w2_weight_scale,
|
||||
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
layer=layer,
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
self.moe_kernel = make_fp8_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: RoutedExperts
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
return make_fp8_moe_quant_config(
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..inc_linear import INCLinearMethod
|
||||
from .inc_scheme import INCScheme
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from ..config_parser import INCLayerConfig
|
||||
from ..inc import INCConfig
|
||||
|
||||
|
||||
class INCMxfp8Scheme(INCScheme):
|
||||
@staticmethod
|
||||
def can_handle(layer_config: "INCLayerConfig") -> bool:
|
||||
return layer_config.is_mxfp8
|
||||
|
||||
def get_linear_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, layer, prefix, layer_config
|
||||
from .inc_mxfp8_linear import INCMxfp8LinearScheme
|
||||
|
||||
return INCLinearMethod(INCMxfp8LinearScheme())
|
||||
|
||||
def get_moe_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, prefix, layer_config
|
||||
from .inc_mxfp8_moe import INCMxfp8MoEMethod
|
||||
|
||||
return INCMxfp8MoEMethod(layer.moe_config)
|
||||
@@ -1,88 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
|
||||
from vllm.model_executor.layers.linear import LinearMethodBase
|
||||
|
||||
from ..config_parser import INCLayerConfig
|
||||
from ..inc import INCConfig
|
||||
|
||||
|
||||
class INCScheme(ABC):
|
||||
"""One class per quant type. Single registration point for the factory.
|
||||
|
||||
Each subclass defines:
|
||||
- can_handle(): when does this scheme apply?
|
||||
- get_linear_method(): required — how to quantize Linear layers
|
||||
- get_moe_method(): optional — how to quantize MoE layers
|
||||
|
||||
Schemes that don't support MoE/KVCache inherit the default raise.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def can_handle(layer_config: "INCLayerConfig") -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_linear_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
) -> "LinearMethodBase":
|
||||
raise NotImplementedError
|
||||
|
||||
def get_moe_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
) -> "FusedMoEMethodBase | None":
|
||||
"""Optional. Override if this scheme supports MoE.
|
||||
Default raises NotImplementedError."""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support MoE layers. "
|
||||
f"Layer config: {layer_config}"
|
||||
)
|
||||
|
||||
|
||||
class INCLinearScheme(ABC):
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def create_weights(
|
||||
self,
|
||||
layer: "torch.nn.Module",
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: "torch.dtype",
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: "torch.nn.Module",
|
||||
x: "torch.Tensor",
|
||||
bias: "torch.Tensor | None" = None,
|
||||
) -> "torch.Tensor":
|
||||
raise NotImplementedError
|
||||
@@ -1,112 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from .inc_wna16_linear import INCXPULinearMethod
|
||||
|
||||
|
||||
class INCXPUW4A8LinearMethod(INCXPULinearMethod):
|
||||
"""XPU linear method for INC int4 weights with dynamic int8 activations.
|
||||
|
||||
Uses the same GPTQ-packed "NT" qweight layout as ``INCXPULinearMethod`` —
|
||||
``int4_gemm_w4a8`` and ``int4_gemm_w4a16`` accept bit-identical weights, so
|
||||
no extra repacking is needed. Activations are dynamically quantized
|
||||
per-token to symmetric int8, which keeps the GEMM on the int8 datapath
|
||||
instead of upconverting the weights to the activation dtype. That is a win
|
||||
for compute-bound shapes (large token counts, e.g. diffusion), where w4a16
|
||||
is dominated by dequantization rather than by weight bandwidth.
|
||||
|
||||
Both scale tensors must be float16: the kernel reads them as fp16
|
||||
regardless of the activation dtype, so bf16 scales are silently
|
||||
misinterpreted. The activations themselves may be bf16 or fp16.
|
||||
|
||||
Opt in with ``VLLM_XPU_INC_WNA16_BACKEND=w4a8``. Whether this beats ARK is
|
||||
device-dependent, so it is not the default: it was measured faster on B70
|
||||
(Xe2), where ARK cannot use its XMX int8 path and falls back to fp16.
|
||||
"""
|
||||
|
||||
# int8 activations only pay off once the GEMM is compute-bound. Below this
|
||||
# token count the per-token quantization overhead dominates and w4a16 is
|
||||
# faster, so fall back to it per-call.
|
||||
_MIN_TOKENS_FOR_INT8 = 512
|
||||
|
||||
# int4_gemm_w4a8 requires both GEMM dims to be multiples of 8. Tensor
|
||||
# parallelism shards these, so an aligned model dimension can still yield an
|
||||
# unaligned partition.
|
||||
_DIM_ALIGNMENT = 8
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
unaligned = [
|
||||
(name, size)
|
||||
for name, size in (
|
||||
("input", input_size_per_partition),
|
||||
("output", output_size_per_partition),
|
||||
)
|
||||
if size % self._DIM_ALIGNMENT != 0
|
||||
]
|
||||
if unaligned:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires partitioned in/out "
|
||||
f"sizes that are multiples of {self._DIM_ALIGNMENT}, got "
|
||||
+ ", ".join(f"{name}={size}" for name, size in unaligned)
|
||||
+ f". Partition shape: ({input_size_per_partition}, "
|
||||
f"{output_size_per_partition})."
|
||||
)
|
||||
super().create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
# The kernel reads both scale tensors as fp16; keep an fp16 copy for the
|
||||
# w4a8 path and leave ``scales`` untouched for the w4a16 fallback.
|
||||
layer.scales_fp16 = Parameter(
|
||||
layer.scales.data.to(torch.float16), requires_grad=False
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
if reshaped_x.shape[0] < self._MIN_TOKENS_FOR_INT8:
|
||||
return super().apply_weights(layer, x, bias)
|
||||
|
||||
from vllm._xpu_ops import xpu_ops as ops
|
||||
|
||||
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
|
||||
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
|
||||
reshaped_x, True, 8
|
||||
)
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a8(
|
||||
quant_x,
|
||||
x_scale.to(torch.float16),
|
||||
x_zero,
|
||||
layer.qweight,
|
||||
layer.scales_fp16,
|
||||
layer.qzeros,
|
||||
self.group_size,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
bias,
|
||||
)
|
||||
return out.to(x.dtype).reshape(out_shape)
|
||||
@@ -1,460 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_marlin_supported,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
from .inc_scheme import INCLinearScheme
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..config_parser import INCLayerConfig
|
||||
|
||||
|
||||
class INCWNA16LinearScheme(INCLinearScheme):
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
self.layer_config = layer_config
|
||||
self.linear_method = self._build_inner_method()
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 60
|
||||
|
||||
def _build_inner_method(self):
|
||||
if self.layer_config.is_gptq:
|
||||
return self._build_gptq_method()
|
||||
if self.layer_config.is_awq:
|
||||
return self._build_awq_method()
|
||||
raise NotImplementedError(
|
||||
f"WNA16 linear scheme does not support {self.layer_config}"
|
||||
)
|
||||
|
||||
def _build_gptq_method(self):
|
||||
assert isinstance(self.layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
gptq_type_map = {
|
||||
(4, True): scalar_types.uint4b8,
|
||||
(8, True): scalar_types.uint8b128,
|
||||
}
|
||||
use_marlin = (
|
||||
self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
|
||||
) and (self.layer_config.bits, self.layer_config.sym) in gptq_type_map
|
||||
if use_marlin:
|
||||
use_marlin = check_marlin_supported(
|
||||
gptq_type_map[(self.layer_config.bits, self.layer_config.sym)],
|
||||
self.layer_config.group_size,
|
||||
has_zp=not self.layer_config.sym,
|
||||
)
|
||||
|
||||
if use_marlin:
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQLinearMethod,
|
||||
)
|
||||
|
||||
return AutoGPTQLinearMethod(
|
||||
AutoGPTQConfig(
|
||||
weight_bits=self.layer_config.bits,
|
||||
group_size=self.layer_config.group_size,
|
||||
desc_act=False,
|
||||
is_sym=self.layer_config.sym,
|
||||
lm_head_quantized=False,
|
||||
dynamic={},
|
||||
full_config={},
|
||||
)
|
||||
)
|
||||
|
||||
raise NotImplementedError(
|
||||
f"INC quantization with bits={self.layer_config.bits}, "
|
||||
f"sym={self.layer_config.sym} is not supported. "
|
||||
"Only 4-bit and 8-bit symmetric quantization is supported "
|
||||
"with Marlin kernels."
|
||||
)
|
||||
|
||||
def _build_awq_method(self):
|
||||
assert isinstance(self.layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
awq_type_map = {
|
||||
4: scalar_types.uint4,
|
||||
8: scalar_types.uint8,
|
||||
}
|
||||
use_marlin = (
|
||||
self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
|
||||
) and self.layer_config.bits in awq_type_map
|
||||
if use_marlin:
|
||||
use_marlin = check_marlin_supported(
|
||||
awq_type_map[self.layer_config.bits],
|
||||
self.layer_config.group_size,
|
||||
not self.layer_config.sym,
|
||||
)
|
||||
|
||||
if use_marlin:
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
AutoAWQMarlinLinearMethod,
|
||||
)
|
||||
|
||||
return AutoAWQMarlinLinearMethod(
|
||||
AutoAWQConfig(
|
||||
weight_bits=self.layer_config.bits,
|
||||
group_size=self.layer_config.group_size,
|
||||
zero_point=not self.layer_config.sym,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=[],
|
||||
full_config={},
|
||||
)
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
AutoAWQLinearMethod,
|
||||
)
|
||||
|
||||
return AutoAWQLinearMethod(
|
||||
AutoAWQConfig(
|
||||
weight_bits=self.layer_config.bits,
|
||||
group_size=self.layer_config.group_size,
|
||||
zero_point=not self.layer_config.sym,
|
||||
lm_head_quantized=False,
|
||||
)
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: "torch.nn.Module",
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: "torch.dtype",
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
return self.linear_method.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
|
||||
return self.linear_method.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: "torch.nn.Module",
|
||||
x: "torch.Tensor",
|
||||
bias: "torch.Tensor | None" = None,
|
||||
) -> "torch.Tensor":
|
||||
return self.linear_method.apply(layer, x, bias)
|
||||
|
||||
|
||||
class INCXPULinearBase(INCLinearScheme):
|
||||
# AWQ packs nibbles within each int32 in the order [0, 2, 4, 6, 1, 3, 5, 7];
|
||||
# this permutation undoes that ordering so values can be repacked in
|
||||
# standard sequential (GPTQ) order.
|
||||
_REVERSE_AWQ_PACK_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
|
||||
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
self.weight_bits = layer_config.bits
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"INCXPULinearBase requires integer group_size."
|
||||
)
|
||||
self.group_size = layer_config.group_size
|
||||
|
||||
self.sym = layer_config.sym
|
||||
self.pack_factor = 32 // self.weight_bits
|
||||
self.is_awq_packed = layer_config.is_awq
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 0
|
||||
|
||||
def _create_inc_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Any,
|
||||
) -> None:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
scales_and_zp_size = input_size_per_partition // self.group_size
|
||||
|
||||
if self.is_awq_packed:
|
||||
# AWQ: qweight [in, out // pack_factor] packed along output dim
|
||||
qweight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition,
|
||||
output_size_per_partition // self.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
else:
|
||||
# GPTQ: qweight [in // pack_factor, out] packed along input dim
|
||||
qweight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition // self.pack_factor,
|
||||
output_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=0,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
scales = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
scales_and_zp_size,
|
||||
output_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
# Both AWQ and GPTQ checkpoints store qzeros with this shape; for
|
||||
# symmetric quantization the values are ignored downstream.
|
||||
qzeros = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
scales_and_zp_size,
|
||||
output_size_per_partition // self.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
layer.register_parameter("qweight", qweight)
|
||||
layer.register_parameter("scales", scales)
|
||||
layer.register_parameter("qzeros", qzeros)
|
||||
|
||||
def _convert_awq_qweight_to_gptq(self, qw: torch.Tensor) -> torch.Tensor:
|
||||
"""Convert AWQ qweight [K, N // pf] to GPTQ qweight [K // pf, N].
|
||||
|
||||
AWQ packs along the output dim with a non-standard nibble order; GPTQ
|
||||
packs along the input dim with sequential nibble order. The conversion
|
||||
is lossless — it only reshuffles bits.
|
||||
"""
|
||||
size_bits = self.weight_bits
|
||||
pack_factor = self.pack_factor
|
||||
mask = (1 << size_bits) - 1
|
||||
device = qw.device
|
||||
reverse_order = torch.tensor(
|
||||
self._REVERSE_AWQ_PACK_ORDER, dtype=torch.long, device=device
|
||||
)
|
||||
shifts = torch.arange(0, 32, size_bits, dtype=torch.int32, device=device)
|
||||
|
||||
K, N_packed = qw.shape
|
||||
N = N_packed * pack_factor
|
||||
|
||||
# Unpack int32 → individual values, fix AWQ nibble ordering
|
||||
unpacked = (qw.unsqueeze(-1) >> shifts) & mask # (K, N_packed, pf)
|
||||
unpacked = unpacked[:, :, reverse_order]
|
||||
unpacked = unpacked.reshape(K, N) # (K, N)
|
||||
|
||||
# Repack along input dim (dim 0) in sequential nibble order
|
||||
unpacked = unpacked.reshape(K // pack_factor, pack_factor, N)
|
||||
new_qw = (unpacked.to(torch.int32) << shifts[None, :, None]).sum(
|
||||
dim=1, dtype=torch.int32
|
||||
)
|
||||
return new_qw.contiguous()
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
del input_size, output_size
|
||||
self._create_inc_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
params_dtype=params_dtype,
|
||||
weight_loader=extra_weight_attrs.get("weight_loader"),
|
||||
)
|
||||
|
||||
|
||||
class INCXPULinearMethod(INCXPULinearBase):
|
||||
"""XPU linear method for INC w4a16 quantization (symmetric only).
|
||||
|
||||
Supports both GPTQ-packed (``auto_round:auto_gptq``) and AWQ-packed
|
||||
(``auto_round:auto_awq``) AutoRound checkpoints. AWQ-packed qweights are
|
||||
losslessly repacked into the GPTQ-style nibble layout during
|
||||
``process_weights_after_loading``, before the final oneDNN "NT" transpose
|
||||
that ``torch.ops._xpu_C.int4_gemm_w4a16`` expects.
|
||||
"""
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
device = layer.qweight.data.device
|
||||
|
||||
qweight_data = layer.qweight.data
|
||||
if self.is_awq_packed:
|
||||
# Lossless repack: AWQ [K, N // pf] → GPTQ [K // pf, N]
|
||||
qweight_data = self._convert_awq_qweight_to_gptq(qweight_data)
|
||||
|
||||
qweight_ct = qweight_data.t().contiguous()
|
||||
layer.qweight = Parameter(qweight_ct.t(), requires_grad=False)
|
||||
layer.scales = Parameter(layer.scales.data, requires_grad=False)
|
||||
layer.qzeros = Parameter(
|
||||
torch.tensor([8], dtype=torch.int8, device=device),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a16(
|
||||
reshaped_x,
|
||||
layer.qweight,
|
||||
bias,
|
||||
layer.scales,
|
||||
layer.qzeros,
|
||||
self.group_size,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
)
|
||||
return out.reshape(out_shape)
|
||||
|
||||
|
||||
class INCARKLinearMethod(INCXPULinearBase):
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
super().__init__(layer_config)
|
||||
|
||||
from .inc_ark_ops import get_ark_state
|
||||
|
||||
is_available, error_str, _, quant_linear_cls = get_ark_state()
|
||||
if not is_available or quant_linear_cls is None:
|
||||
reason = error_str or "unknown error"
|
||||
raise ImportError(f"Failed to import auto_round_kernel. {reason}")
|
||||
|
||||
self.quant_linear_cls = quant_linear_cls
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
super().create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
layer.in_features = input_size_per_partition
|
||||
layer.out_features = sum(output_partition_sizes)
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if hasattr(layer, "input_size_per_partition"):
|
||||
in_features = layer.input_size_per_partition
|
||||
elif hasattr(layer, "input_size"):
|
||||
in_features = layer.input_size
|
||||
else:
|
||||
raise AttributeError("Cannot determine in_features for layer.")
|
||||
|
||||
if hasattr(layer, "output_partition_sizes"):
|
||||
out_features = sum(layer.output_partition_sizes)
|
||||
elif hasattr(layer, "output_size_per_partition"):
|
||||
out_features = layer.output_size_per_partition
|
||||
elif hasattr(layer, "output_size"):
|
||||
out_features = layer.output_size
|
||||
else:
|
||||
out_features = layer.scales.shape[-1]
|
||||
|
||||
ark_linear = self.quant_linear_cls(
|
||||
bits=self.weight_bits,
|
||||
group_size=self.group_size,
|
||||
sym=self.sym,
|
||||
in_features=in_features,
|
||||
out_features=out_features,
|
||||
bias=layer.bias is not None,
|
||||
weight_dtype=layer.params_dtype,
|
||||
)
|
||||
ark_linear.to(layer.qweight.device)
|
||||
|
||||
with torch.no_grad():
|
||||
qweight_src = layer.qweight.detach()
|
||||
if self.is_awq_packed:
|
||||
# ARK consumes GPTQ-style packed nibbles; convert AWQ losslessly.
|
||||
qweight_src = self._convert_awq_qweight_to_gptq(qweight_src)
|
||||
ark_linear.qweight.copy_(qweight_src)
|
||||
if hasattr(layer, "qzeros") and layer.qzeros is not None:
|
||||
ark_linear.qzeros.copy_(layer.qzeros.detach())
|
||||
else:
|
||||
ark_linear.qzeros = None
|
||||
ark_linear.scales.copy_(layer.scales.detach())
|
||||
if hasattr(layer, "bias") and layer.bias is not None:
|
||||
ark_linear.bias.copy_(layer.bias.detach())
|
||||
|
||||
ark_linear.post_init()
|
||||
|
||||
layer.qweight = Parameter(ark_linear.qweight.detach(), requires_grad=False)
|
||||
layer.ark_bias = ark_linear.bias
|
||||
layer.ark_compute_type = ark_linear.cdt
|
||||
layer.ark_weight_type = ark_linear.wdt
|
||||
layer.ark_scale_type = ark_linear.sdt
|
||||
|
||||
if hasattr(layer, "qzeros"):
|
||||
del layer.qzeros
|
||||
del layer.scales
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.inc_ark_woq_linear.default(
|
||||
x,
|
||||
layer.qweight,
|
||||
layer.ark_bias,
|
||||
layer.out_features,
|
||||
layer.in_features,
|
||||
self.group_size,
|
||||
layer.ark_compute_type,
|
||||
layer.ark_weight_type,
|
||||
layer.ark_scale_type,
|
||||
not self.sym,
|
||||
)
|
||||
@@ -1,318 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..inc_linear import INCLinearMethod
|
||||
from .inc_scheme import INCScheme
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from ..config_parser import INCLayerConfig
|
||||
from ..inc import INCConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
XPU_WNA16_SUPPORTED_BITS = {2, 4}
|
||||
# On CUDA the Marlin/GPTQ/AWQ kernels only cover 4/8-bit. The remaining
|
||||
# widths (2/3/5/6/7) have no dedicated CUDA kernel, so they are dispatched to
|
||||
# the humming kernel instead. This mirrors compressed-tensors WNA16, whose
|
||||
# 5/6/7-bit biased scalar types also fall back to humming at kernel selection.
|
||||
CUDA_HUMMING_SUPPORTED_BITS = {2, 3, 5, 6, 7}
|
||||
|
||||
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
|
||||
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
|
||||
XPU_ONEDNN_BACKENDS = ("w4a16", "w4a8")
|
||||
|
||||
|
||||
def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> None:
|
||||
"""Raise unless ``int4_gemm_w4a8`` can serve this layer.
|
||||
|
||||
The backend is requested explicitly, so an unusable configuration is an
|
||||
error rather than something to silently fall back from.
|
||||
"""
|
||||
import torch
|
||||
|
||||
if not hasattr(torch.ops._xpu_C, "int4_gemm_w4a8"):
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires the int4_gemm_w4a8 op, "
|
||||
"which this build of vllm-xpu-kernels does not provide. "
|
||||
f"Layer: {prefix}."
|
||||
)
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
|
||||
f"positive multiple of 32, got {layer_config.group_size}. "
|
||||
f"Layer: {prefix}."
|
||||
)
|
||||
|
||||
|
||||
class INCWna16Scheme(INCScheme):
|
||||
@staticmethod
|
||||
def can_handle(layer_config: "INCLayerConfig") -> bool:
|
||||
return layer_config.is_wna16_int
|
||||
|
||||
def get_linear_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, layer
|
||||
if current_platform.is_xpu():
|
||||
if layer_config.bits in XPU_WNA16_SUPPORTED_BITS and layer_config.sym:
|
||||
from .inc_ark_ops import get_ark_state
|
||||
from .inc_w4a8_linear import INCXPUW4A8LinearMethod
|
||||
from .inc_wna16_linear import (
|
||||
INCARKLinearMethod,
|
||||
INCXPULinearMethod,
|
||||
)
|
||||
|
||||
backend = envs.VLLM_XPU_INC_WNA16_BACKEND
|
||||
if backend in XPU_ONEDNN_BACKENDS:
|
||||
if layer_config.bits != 4:
|
||||
raise NotImplementedError(
|
||||
f"VLLM_XPU_INC_WNA16_BACKEND={backend} only supports "
|
||||
f"int4, got int{layer_config.bits}. Layer: {prefix}."
|
||||
)
|
||||
if backend == "w4a8":
|
||||
_check_xpu_w4a8_supported(layer_config, prefix)
|
||||
return INCLinearMethod(INCXPUW4A8LinearMethod(layer_config))
|
||||
return INCLinearMethod(INCXPULinearMethod(layer_config))
|
||||
|
||||
is_ark_available, ark_error, _, _ = get_ark_state()
|
||||
if backend == "ark" and not is_ark_available:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=ark was requested but "
|
||||
f"auto_round_kernel is unavailable: "
|
||||
f"{ark_error or 'unknown error'}. Layer: {prefix}."
|
||||
)
|
||||
if is_ark_available:
|
||||
return INCLinearMethod(INCARKLinearMethod(layer_config))
|
||||
elif layer_config.bits == 2:
|
||||
raise NotImplementedError(
|
||||
"INC int2 on XPU requires the ARK backend. "
|
||||
f"Layer: {prefix}. "
|
||||
f"auto_round_kernel unavailable: "
|
||||
f"{ark_error or 'unknown error'}"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"ARK backend is unavailable for layer %s; "
|
||||
"falling back to the default XPU INC path. Error: %s",
|
||||
prefix,
|
||||
ark_error or "unknown error",
|
||||
)
|
||||
return INCLinearMethod(INCXPULinearMethod(layer_config))
|
||||
raise NotImplementedError(f"INC on XPU: unsupported config {layer_config}")
|
||||
|
||||
if current_platform.is_cpu() and layer_config.is_gptq:
|
||||
if layer_config.bits == 4 and layer_config.sym:
|
||||
from .inc_ark_ops import get_ark_state
|
||||
from .inc_wna16_linear import (
|
||||
INCARKLinearMethod,
|
||||
INCWNA16LinearScheme,
|
||||
)
|
||||
|
||||
is_ark_available, ark_error, _, _ = get_ark_state()
|
||||
if is_ark_available:
|
||||
return INCLinearMethod(INCARKLinearMethod(layer_config))
|
||||
|
||||
logger.debug(
|
||||
"ARK backend is unavailable for layer %s; "
|
||||
"falling back to the default CPU INC path. Error: %s",
|
||||
prefix,
|
||||
ark_error or "unknown error",
|
||||
)
|
||||
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
|
||||
raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
|
||||
|
||||
# CUDA low-bit (2/3/5/6/7): no Marlin/GPTQ/AWQ kernel, route to humming
|
||||
# so a single model can mix 4/8-bit (Marlin) and 2/3/5/6/7-bit (humming)
|
||||
# layers.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
|
||||
):
|
||||
return _build_humming_linear_method(layer_config)
|
||||
|
||||
from .inc_wna16_linear import INCWNA16LinearScheme
|
||||
|
||||
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
|
||||
|
||||
def get_moe_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, prefix
|
||||
# CPU does not support quantized MoE yet.
|
||||
if current_platform.is_cpu():
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
# CUDA low-bit (2/3/5/6/7): route to the humming MoE kernel (see above).
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
|
||||
):
|
||||
return _build_humming_moe_method(layer, layer_config)
|
||||
if layer_config.is_gptq:
|
||||
return _resolve_gptq_moe(layer, layer_config)
|
||||
if layer_config.is_awq:
|
||||
return _resolve_awq_moe(layer, layer_config)
|
||||
raise NotImplementedError(f"WNA16 MoE does not support config {layer_config}")
|
||||
|
||||
|
||||
def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import (
|
||||
MoeWNA16Config,
|
||||
MoeWNA16Method,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
# AutoGPTQMoEMethod selects its fused-MoE backend through the WNA16 oracle
|
||||
# (Marlin on CUDA, XPUExpertsWNA16 on XPU). Gate only on the layer-shape
|
||||
# check like compressed-tensors does; the capability-based
|
||||
# check_marlin_supported is skipped so the XPU path is reachable.
|
||||
use_marlin = (layer_config.bits, layer_config.sym) in {
|
||||
(4, True),
|
||||
(8, True),
|
||||
} and check_moe_marlin_supports_layer(layer, layer_config.group_size)
|
||||
|
||||
if use_marlin:
|
||||
return AutoGPTQMoEMethod(
|
||||
AutoGPTQConfig(
|
||||
weight_bits=layer_config.bits,
|
||||
group_size=layer_config.group_size,
|
||||
desc_act=False,
|
||||
is_sym=layer_config.sym,
|
||||
lm_head_quantized=False,
|
||||
dynamic={},
|
||||
full_config={},
|
||||
),
|
||||
layer.moe_config,
|
||||
)
|
||||
|
||||
moe_config = MoeWNA16Config.from_config(
|
||||
{
|
||||
"quant_method": "gptq",
|
||||
"bits": layer_config.bits,
|
||||
"group_size": layer_config.group_size,
|
||||
"sym": layer_config.sym,
|
||||
"lm_head": False,
|
||||
}
|
||||
)
|
||||
return MoeWNA16Method(moe_config, layer.moe_config)
|
||||
|
||||
|
||||
def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQMoEMethod
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import (
|
||||
MoeWNA16Config,
|
||||
MoeWNA16Method,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
use_marlin = layer_config.bits in (4, 8) and check_moe_marlin_supports_layer(
|
||||
layer, layer_config.group_size
|
||||
)
|
||||
|
||||
if use_marlin:
|
||||
return AutoAWQMoEMethod(
|
||||
AutoAWQConfig(
|
||||
weight_bits=layer_config.bits,
|
||||
group_size=layer_config.group_size,
|
||||
zero_point=not layer_config.sym,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=[],
|
||||
full_config={},
|
||||
),
|
||||
layer.moe_config,
|
||||
)
|
||||
|
||||
moe_config = MoeWNA16Config.from_config(
|
||||
{
|
||||
"quant_method": "awq",
|
||||
"bits": layer_config.bits,
|
||||
"group_size": layer_config.group_size,
|
||||
"zero_point": not layer_config.sym,
|
||||
"lm_head": False,
|
||||
}
|
||||
)
|
||||
return MoeWNA16Method(moe_config, layer.moe_config)
|
||||
|
||||
|
||||
def _humming_weight_config(layer_config: "INCLayerConfig") -> dict:
|
||||
"""Build the humming weight-schema config for a WNA16 int checkpoint."""
|
||||
if layer_config.is_gptq:
|
||||
return {
|
||||
"quant_method": "gptq",
|
||||
"bits": layer_config.bits,
|
||||
"group_size": layer_config.group_size,
|
||||
"desc_act": False,
|
||||
"sym": layer_config.sym,
|
||||
}
|
||||
if layer_config.is_awq:
|
||||
return {
|
||||
"quant_method": "awq",
|
||||
"bits": layer_config.bits,
|
||||
"group_size": layer_config.group_size,
|
||||
"zero_point": not layer_config.sym,
|
||||
}
|
||||
raise NotImplementedError(
|
||||
"INC humming dispatch only supports gptq/awq packed int checkpoints, "
|
||||
f"but found {layer_config}."
|
||||
)
|
||||
|
||||
|
||||
def _build_humming_quant_config(layer_config: "INCLayerConfig"):
|
||||
from vllm.model_executor.layers.quantization.humming import (
|
||||
HummingLayerQuantizationConfig,
|
||||
)
|
||||
from vllm.utils.humming import BaseWeightSchema
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(_humming_weight_config(layer_config))
|
||||
return HummingLayerQuantizationConfig(weight_schema=weight_schema)
|
||||
|
||||
|
||||
def _build_humming_linear_method(layer_config: "INCLayerConfig"):
|
||||
from vllm.model_executor.layers.quantization.humming import HummingLinearMethod
|
||||
|
||||
return HummingLinearMethod(_build_humming_quant_config(layer_config))
|
||||
|
||||
|
||||
def _build_humming_moe_method(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
from vllm.model_executor.layers.quantization.humming import HummingMoEMethod
|
||||
|
||||
return HummingMoEMethod(_build_humming_quant_config(layer_config), layer.moe_config)
|
||||
@@ -19,7 +19,7 @@ from vllm.model_executor.kernels.linear import (
|
||||
init_mxfp8_linear_kernel,
|
||||
init_nvfp4_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEMethodBase,
|
||||
@@ -228,6 +228,8 @@ class ModelOptQuantConfigBase(QuantizationConfig):
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
# handle kv-cache first so we can focus only on weight quantization thereafter
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
|
||||
if isinstance(layer, (Attention, MLAAttention)):
|
||||
return self.KVCacheMethodCls(self)
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,21 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .quark_nvfp4 import QuarkNVFP4
|
||||
from .quark_ocp_mx import QuarkOCP_MX
|
||||
from .quark_scheme import QuarkScheme
|
||||
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
|
||||
from .quark_w4a16_int4 import QuarkW4A16Int4
|
||||
from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
|
||||
from .quark_w8a8_int8 import QuarkW8A8Int8
|
||||
|
||||
__all__ = [
|
||||
"QuarkScheme",
|
||||
"QuarkW8A8Fp8",
|
||||
"QuarkW8A8Fp8PerBlock",
|
||||
"QuarkW8A8Int8",
|
||||
"QuarkOCP_MX",
|
||||
"QuarkW4A16Int4",
|
||||
"QuarkW4A8_MXFP4_FP8",
|
||||
"QuarkNVFP4",
|
||||
]
|
||||
@@ -1,163 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
|
||||
from vllm.model_executor.kernels.linear.nvfp4.emulation import (
|
||||
EmulationNvFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
|
||||
QuarkScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
|
||||
__all__ = ["QuarkNVFP4"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class QuarkNVFP4(QuarkScheme):
|
||||
"""Quark NVFP4 quantization scheme.
|
||||
|
||||
Supports loading NVFP4 checkpoints with the following structure:
|
||||
- weight: uint8, shape [out_features, in_features // 2] (packed FP4)
|
||||
- weight_scale: float8_e4m3fn, shape [out_features, in_features // group_size]
|
||||
- weight_scale_2: bfloat16/float32, scalar (global weight scale)
|
||||
- input_scale_2: bfloat16/float32, scalar (global input scale)
|
||||
"""
|
||||
|
||||
supported_activation_quant_keys = [kNvfp4Dynamic]
|
||||
supported_weight_quant_keys = [kNvfp4Static]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(kNvfp4Static, activation_quant_key)
|
||||
self.kernel = init_nvfp4_linear_kernel()
|
||||
self.group_size = 16
|
||||
|
||||
if not isinstance(self.kernel, EmulationNvFp4LinearKernel):
|
||||
logger.warning_once(
|
||||
"Only EmulationNvFp4LinearKernel NVFP4 dense implementation is "
|
||||
"tested with QuarkNVFP4, got kernel=%s. Correctness is not validated.",
|
||||
type(self.kernel).__name__,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
# FP4 requires Turing (75) or newer
|
||||
return 75
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
|
||||
if input_size_per_partition % self.group_size != 0:
|
||||
raise ValueError(
|
||||
f"Input size per partition ({input_size_per_partition}) must be "
|
||||
f"divisible by group size ({self.group_size})"
|
||||
)
|
||||
|
||||
# Weight: FP4 packed as uint8 (2 FP4 values per uint8)
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // 2,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# Per-group weight scale (FP8 E4M3)
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // self.group_size,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
# Global weight scale (scalar, per partition)
|
||||
weight_scale_2 = PerTensorScaleParameter(
|
||||
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale_2", weight_scale_2)
|
||||
|
||||
# Global input scale (scalar, per partition)
|
||||
input_scale_2 = PerTensorScaleParameter(
|
||||
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("input_scale_2", input_scale_2)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
input_global_scale = layer.input_scale_2.max().to(torch.float32)
|
||||
layer.input_global_scale = Parameter(input_global_scale, requires_grad=False)
|
||||
del layer.input_scale_2
|
||||
|
||||
weight_global_scale = layer.weight_scale_2.to(torch.float32)
|
||||
|
||||
if torch.unique(weight_global_scale).numel() != 1:
|
||||
logger.warning_once(
|
||||
"In NVFP4 linear, the global scale for weight are different"
|
||||
" for parallel layers (e.g. q_proj, k_proj, v_proj). This"
|
||||
" will likely result in reduced accuracy. Please verify the"
|
||||
" model accuracy. Consider using a checkpoint with a shared"
|
||||
" global NVFP4 scale for fused layers."
|
||||
)
|
||||
|
||||
weight_global_scale = weight_global_scale.max()
|
||||
|
||||
layer.weight_global_scale = Parameter(weight_global_scale, requires_grad=False)
|
||||
del layer.weight_scale_2
|
||||
|
||||
layer.alpha = Parameter(
|
||||
layer.input_global_scale * layer.weight_global_scale, requires_grad=False
|
||||
)
|
||||
layer.input_global_scale_inv = Parameter(
|
||||
(1.0 / layer.input_global_scale).to(torch.float32), requires_grad=False
|
||||
)
|
||||
|
||||
# Convert layer to NVFP4 linear kernel format
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.kernel.apply_weights(layer=layer, x=x, bias=bias)
|
||||
@@ -1,264 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MxFp4LinearKernel,
|
||||
MxFp6LinearKernel,
|
||||
Mxfp8LinearKernel,
|
||||
init_mxfp4_linear_kernel,
|
||||
init_mxfp6_linear_kernel,
|
||||
init_mxfp8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
_ACTIVATION_QUANT_KEY_MAP,
|
||||
_WEIGHT_QUANT_KEY_MAP,
|
||||
OCP_MX_BLOCK_SIZE,
|
||||
OCP_MX_UNPACKED_DTYPES,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Static,
|
||||
kMxfp8Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .quark_scheme import QuarkScheme
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class QuarkOCP_MX(QuarkScheme):
|
||||
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel | Mxfp8LinearKernel
|
||||
supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None]
|
||||
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
scale_block_rows: int = 1,
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.scale_block_rows = scale_block_rows
|
||||
self.weight_dtype = next(
|
||||
dtype
|
||||
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
|
||||
if quant_key == weight_quant_key
|
||||
)
|
||||
self.input_dtype = (
|
||||
next(
|
||||
dtype
|
||||
for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items()
|
||||
if quant_key == activation_quant_key
|
||||
)
|
||||
if activation_quant_key is not None
|
||||
else None
|
||||
)
|
||||
|
||||
self.is_unpacked = self.weight_dtype in OCP_MX_UNPACKED_DTYPES
|
||||
|
||||
if self.is_unpacked:
|
||||
self.packed_factor: int | Fraction = 1
|
||||
elif self.weight_dtype == "mxfp4":
|
||||
self.packed_factor = 2
|
||||
else:
|
||||
self.packed_factor = Fraction(numerator=8, denominator=6)
|
||||
|
||||
if self.is_unpacked:
|
||||
# MXFP8 runs on the shared kernel abstraction, which picks a native
|
||||
# backend where one exists and emulates otherwise. The MXFP4/MXFP6
|
||||
# warnings below do not apply.
|
||||
return
|
||||
|
||||
if not current_platform.supports_mx():
|
||||
logger.warning_once(
|
||||
"The current platform does not support native MXFP4/MXFP6 "
|
||||
"computation. Simulated weight dequantization and activation "
|
||||
"QDQ (quantize and dequantize) will be used, with the linear "
|
||||
"layers computed in high precision."
|
||||
)
|
||||
|
||||
if current_platform.supports_mx() and (
|
||||
self.input_dtype != "mxfp4" or self.weight_dtype != "mxfp4"
|
||||
):
|
||||
logger.warning_once(
|
||||
"The current platform supports native MXFP4/MXFP6 "
|
||||
f"computation, but kernels for input_dtype={self.input_dtype} "
|
||||
f"and weight_dtype={self.weight_dtype} are not yet integrated "
|
||||
"in vLLM. Simulated weight dequantization and activation "
|
||||
"QDQ (quantize and dequantize) will be used, with the linear "
|
||||
"layers computed in high precision."
|
||||
)
|
||||
|
||||
def get_packed_dim(self, dim: int, quant_dtype: str):
|
||||
if quant_dtype in OCP_MX_UNPACKED_DTYPES:
|
||||
return dim
|
||||
elif quant_dtype == "mxfp4":
|
||||
assert dim % 2 == 0
|
||||
return dim // 2
|
||||
elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}:
|
||||
# FP6 packs 4 * 6 = 24 bits on 3 bytes.
|
||||
assert (dim * 3) % 4 == 0
|
||||
return (dim * 3) // 4
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"Unsupported quant_dtype in QuarkOCP_MX.get_packed_dim, "
|
||||
f"got quant_dtype={quant_dtype}. Something is wrong, please "
|
||||
"open an issue."
|
||||
)
|
||||
|
||||
def _expanding_scale_loader(self, weight_loader: Callable) -> Callable:
|
||||
"""Expand a 2-D block scale to one row per weight row.
|
||||
|
||||
A ``block_size=[R, 32]`` export carries one e8m0 value per R weight
|
||||
rows; the kernels want one per row. At ``R == 1`` the layouts already
|
||||
coincide, so the loader is passed through untouched.
|
||||
"""
|
||||
if self.scale_block_rows == 1:
|
||||
return weight_loader
|
||||
|
||||
rows = self.scale_block_rows
|
||||
|
||||
def expanding_loader(param, loaded_weight, *args, **kwargs):
|
||||
loaded_weight = loaded_weight.view(torch.uint8).repeat_interleave(
|
||||
rows, dim=0
|
||||
)
|
||||
return weight_loader(param, loaded_weight, *args, **kwargs)
|
||||
|
||||
return expanding_loader
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 70
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
|
||||
|
||||
self.ocp_mx_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
if input_size_per_partition % OCP_MX_BLOCK_SIZE != 0:
|
||||
layer_name = getattr(layer, "prefix", "") or type(layer).__name__
|
||||
raise ValueError(
|
||||
f"OCP MX linear layer {layer_name!r} has an input size per "
|
||||
f"partition of {input_size_per_partition}, which must be "
|
||||
f"divisible by the OCP MX group size {OCP_MX_BLOCK_SIZE}. "
|
||||
"Choose a compatible tensor-parallel size or avoid "
|
||||
"tensor-parallel sharding for this layer."
|
||||
)
|
||||
|
||||
if self.is_unpacked:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // OCP_MX_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=self._expanding_scale_loader(weight_loader),
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
layer.weight_block_size = [1, OCP_MX_BLOCK_SIZE]
|
||||
else:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
|
||||
# WEIGHT
|
||||
weight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_dim=1,
|
||||
packed_factor=self.packed_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# WEIGHT SCALE
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // OCP_MX_BLOCK_SIZE,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
if self.weight_quant_key == kMxfp4Static:
|
||||
self.ocp_mx_linear = init_mxfp4_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
)
|
||||
elif self.weight_quant_key in [kMxfp6E2M3Static, kMxfp6E3M2Static]:
|
||||
self.ocp_mx_linear = init_mxfp6_linear_kernel(
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
)
|
||||
elif self.weight_quant_key == kMxfp8Static:
|
||||
self.ocp_mx_linear = init_mxfp8_linear_kernel(
|
||||
weight_shape=layer.weight.shape
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"No OCP MX linear kernel for weight_quant_key="
|
||||
f"{self.weight_quant_key}. Something is wrong, please open an "
|
||||
"issue."
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.ocp_mx_linear.apply_weights(layer, x, bias)
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
__all__ = ["QuarkScheme"]
|
||||
|
||||
|
||||
class QuarkScheme(ABC):
|
||||
"""Abstract class used to describe the weight creation and forward pass
|
||||
of different quantization schemes supported by Quark.
|
||||
"""
|
||||
|
||||
supported_activation_quant_keys: list[QuantKey | None] = []
|
||||
supported_weight_quant_keys: list[QuantKey] = []
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
if activation_quant_key not in self.supported_activation_quant_keys:
|
||||
raise ValueError(
|
||||
f"Unsupported activation quant key: {activation_quant_key}"
|
||||
)
|
||||
if weight_quant_key not in self.supported_weight_quant_keys:
|
||||
raise ValueError(f"Unsupported weight quant key: {weight_quant_key}")
|
||||
self.activation_quant_key = activation_quant_key
|
||||
self.weight_quant_key = weight_quant_key
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
"""Get minimum device capability."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def create_weights(self, *args, **kwargs):
|
||||
"""Weight creation for the particular scheme. Inputs to this function."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def apply_weights(
|
||||
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
|
||||
):
|
||||
"""Run the forward pass for the particular scheme. This is where
|
||||
scheme-specific dequant/quant steps/kernels should be applied.
|
||||
|
||||
Args:
|
||||
layer: torch.nn.Module with the registered weights and
|
||||
other parameters relevant to the particular scheme.
|
||||
x: input to the layer
|
||||
bias: bias parameter
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
"""Called after weight loading is complete for any cleanup that
|
||||
needs to occur.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -1,212 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MPLinearKernel,
|
||||
MPLinearLayerConfig,
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
_convert_awq_to_standard_format,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
|
||||
QuarkScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
canonicalize_quark_packed_int4,
|
||||
parse_w4a16_int4_weight_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4Static32Asym,
|
||||
kInt4StaticAsym,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
|
||||
class QuarkW4A16Int4(QuarkScheme):
|
||||
"""Quark packed INT4 weight-only linear scheme via MPLinearKernel."""
|
||||
|
||||
supported_activation_quant_keys: list[QuantKey | None] = [None]
|
||||
supported_weight_quant_keys: list[QuantKey] = [
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4StaticAsym,
|
||||
kInt4Static32Asym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
pack_method: str,
|
||||
weight_config: dict[str, Any],
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
|
||||
# group_size is read from the Quark config rather than weight_quant_key:
|
||||
# the int4 QuantKeys only encode group_size 32 or -1, but Quark int4
|
||||
# checkpoints commonly use group_size 128, which the QuantKey cannot
|
||||
# represent. Symmetry is unambiguous, so it comes from the key.
|
||||
self.group_size = parse_w4a16_int4_weight_config(weight_config)[0]
|
||||
self.pack_factor = 8
|
||||
self.pack_reorder = pack_method == "reorder"
|
||||
self.is_symmetric = weight_quant_key.symmetric
|
||||
self.quant_type = (
|
||||
scalar_types.uint4b8 if self.is_symmetric else scalar_types.uint4
|
||||
)
|
||||
self.kernel: MPLinearKernel | None = None
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 70
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
input_size = kwargs["input_size"]
|
||||
output_size = kwargs["output_size"]
|
||||
group_size = (
|
||||
self.group_size if self.group_size != -1 else input_size_per_partition
|
||||
)
|
||||
if input_size_per_partition % group_size != 0:
|
||||
raise ValueError(
|
||||
"The input size is not aligned with the quantized weight shape. "
|
||||
"This can be caused by too large tensor parallel size. "
|
||||
f"input_size_per_partition={input_size_per_partition}, "
|
||||
f"group_size={group_size}."
|
||||
)
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
packed_output_size_per_partition = math.ceil(
|
||||
output_size_per_partition / self.pack_factor
|
||||
)
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.packed_output_size_per_partition = packed_output_size_per_partition
|
||||
|
||||
mp_linear_kernel_config = MPLinearLayerConfig(
|
||||
full_weight_shape=(input_size, output_size),
|
||||
partition_weight_shape=(
|
||||
input_size_per_partition,
|
||||
output_size_per_partition,
|
||||
),
|
||||
weight_type=self.quant_type,
|
||||
act_type=params_dtype,
|
||||
group_size=group_size,
|
||||
zero_points=not self.is_symmetric,
|
||||
)
|
||||
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
|
||||
|
||||
def weight_scale_loader(
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
*args,
|
||||
**loader_kwargs,
|
||||
) -> None:
|
||||
if loaded_weight.shape[1] < param.data.shape[1]:
|
||||
padded_weight = loaded_weight.new_zeros(param.data.shape)
|
||||
padded_weight[:, : loaded_weight.shape[1]] = loaded_weight
|
||||
loaded_weight = padded_weight
|
||||
weight_loader(param, loaded_weight, *args, **loader_kwargs)
|
||||
|
||||
weight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition,
|
||||
packed_output_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
num_groups = input_size_per_partition // group_size
|
||||
weight_zero_point = PackedvLLMParameter(
|
||||
data=torch.zeros(
|
||||
num_groups,
|
||||
packed_output_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
num_groups,
|
||||
packed_output_size_per_partition * self.pack_factor,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
weight_loader=weight_scale_loader,
|
||||
)
|
||||
|
||||
layer.register_parameter("weight", weight)
|
||||
layer.register_parameter("weight_zero_point", weight_zero_point)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_linear_kernel_config,
|
||||
w_q_param_name="weight",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point" if not self.is_symmetric else None,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight.data = canonicalize_quark_packed_int4(
|
||||
layer.weight.data,
|
||||
pack_reorder=self.pack_reorder,
|
||||
is_symmetric=self.is_symmetric,
|
||||
pack_factor=self.pack_factor,
|
||||
)
|
||||
if not self.is_symmetric:
|
||||
layer.weight_zero_point.data = canonicalize_quark_packed_int4(
|
||||
layer.weight_zero_point.data,
|
||||
pack_reorder=self.pack_reorder,
|
||||
is_symmetric=self.is_symmetric,
|
||||
pack_factor=self.pack_factor,
|
||||
)
|
||||
output_size = layer.output_size_per_partition
|
||||
packed_output_size = layer.packed_output_size_per_partition * self.pack_factor
|
||||
if output_size < packed_output_size:
|
||||
layer.weight_scale.data[:, output_size:].zero_()
|
||||
|
||||
_convert_awq_to_standard_format(
|
||||
layer,
|
||||
"weight",
|
||||
"weight_zero_point" if not self.is_symmetric else None,
|
||||
self.quant_type.size_bits,
|
||||
)
|
||||
assert self.kernel is not None
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
):
|
||||
assert self.kernel is not None
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
@@ -1,204 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
get_fp8_min_max,
|
||||
kFp8StaticTensorSym,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .quark_scheme import QuarkScheme
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
__all__ = ["QuarkW4A8_MXFP4_FP8"]
|
||||
|
||||
OCP_MX_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
class QuarkW4A8_MXFP4_FP8(QuarkScheme):
|
||||
"""- Weights: MXFP4 with E8M0 scales per block of 32
|
||||
- Activations: FP8 E4M3 (static per-tensor quantization)
|
||||
|
||||
Uses the AITER Triton kernel and falls back to emulation if AITER not available.
|
||||
"""
|
||||
|
||||
supported_activation_quant_keys = [kFp8StaticTensorSym]
|
||||
supported_weight_quant_keys = [kMxfp4Static]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(kMxfp4Static, activation_quant_key)
|
||||
self.out_dtype = None
|
||||
|
||||
self.weight_dtype = "mxfp4"
|
||||
self.packed_factor: Fraction = Fraction(2, 1) # 2 FP4 values per byte
|
||||
self.weight_block_size = OCP_MX_BLOCK_SIZE
|
||||
|
||||
self.fp8_min, self.fp8_max = get_fp8_min_max()
|
||||
self.fp8_dtype = current_platform.fp8_dtype()
|
||||
|
||||
kernel_supported_gpu = False
|
||||
if current_platform.is_rocm():
|
||||
kernel_supported_gpu = current_platform.supports_mx()
|
||||
|
||||
self.use_aiter_kernel = is_aiter_found_and_supported() and kernel_supported_gpu
|
||||
|
||||
if not self.use_aiter_kernel:
|
||||
logger.warning_once(
|
||||
"[W4A8 MXFP4+FP8] Aiter Triton kernel not found. Using emulation mode."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 70
|
||||
|
||||
def get_packed_dim(self, dim: int) -> int:
|
||||
assert dim % 2 == 0, f"Dimension {dim} must be even for MXFP4 packing"
|
||||
return dim // 2
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
|
||||
# MXFP4 WEIGHT (packed, 2 values per byte)
|
||||
weight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
self.get_packed_dim(input_size_per_partition),
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_dim=1,
|
||||
packed_factor=self.packed_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# WEIGHT SCALE (E8M0 format, per block of 32)
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // self.weight_block_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
# INPUT SCALE (FP8 per-tensor static scale)
|
||||
input_scale = PerTensorScaleParameter(
|
||||
data=torch.empty(
|
||||
len(output_partition_sizes),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
# Initialize to avoid NaN
|
||||
input_scale[:] = torch.finfo(torch.float32).min
|
||||
layer.register_parameter("input_scale", input_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Ensuring weights & scales are non-trainable
|
||||
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
layer.weight_scale.data, requires_grad=False
|
||||
)
|
||||
|
||||
input_scale = layer.input_scale.data
|
||||
# For fused modules (QKV), take the max scale
|
||||
if input_scale.numel() != 1:
|
||||
input_scale = input_scale.max()
|
||||
|
||||
layer.input_scale = torch.nn.Parameter(
|
||||
torch.tensor(input_scale, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.use_aiter_kernel:
|
||||
return self._apply_aiter_kernel(layer, x, bias)
|
||||
else:
|
||||
return self._apply_emulation(layer, x, bias)
|
||||
|
||||
def _apply_aiter_kernel(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
M = x.shape[0]
|
||||
out_dtype = x.dtype if self.out_dtype is None else self.out_dtype
|
||||
|
||||
input_scale = layer.input_scale
|
||||
x_fp8 = (x / input_scale).clamp(self.fp8_min, self.fp8_max).to(self.fp8_dtype)
|
||||
|
||||
# Broadcast per-tensor scale to per-row (M, 1) for Aiter kernel
|
||||
x_scales = input_scale.expand(M, 1).to(dtype=torch.float32, device=x.device)
|
||||
|
||||
y = rocm_aiter_ops.gemm_a8wfp4(
|
||||
x_fp8, layer.weight, x_scales, layer.weight_scale, out_dtype
|
||||
)
|
||||
|
||||
if bias is not None:
|
||||
y = y + bias
|
||||
|
||||
return y
|
||||
|
||||
def _apply_emulation(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
dequant_mxfp4,
|
||||
)
|
||||
|
||||
weight_dq = dequant_mxfp4(
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
x.dtype,
|
||||
)
|
||||
|
||||
input_scale = layer.input_scale
|
||||
x_fp8 = (x / input_scale).clamp(self.fp8_min, self.fp8_max).to(self.fp8_dtype)
|
||||
x_dq = (x_fp8.to(x.dtype) * input_scale).to(x.dtype)
|
||||
|
||||
return F.linear(x_dq, weight_dq, bias)
|
||||
@@ -1,317 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
expose_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
create_fp8_scale_parameter,
|
||||
create_fp8_weight_parameter,
|
||||
validate_fp8_block_shape,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockE8M0Sym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
requantize_with_max_scale,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BlockQuantScaleParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
__all__ = ["QuarkW8A8Fp8", "QuarkW8A8Fp8PerBlock"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class QuarkW8A8Fp8(QuarkScheme):
|
||||
supported_activation_quant_keys: list[QuantKey | None] = [
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticTensorSym,
|
||||
]
|
||||
supported_weight_quant_keys: list[QuantKey] = [
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.weight_qscheme = (
|
||||
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
|
||||
)
|
||||
self.is_static_input_scheme = activation_quant_key == kFp8StaticTensorSym
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
# lovelace and up
|
||||
return 89
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
# If per tensor, when we have a fused module (e.g. QKV) with per
|
||||
# tensor scales (thus N scales being passed to the kernel),
|
||||
# requantize so we can always run per tensor
|
||||
if self.weight_qscheme == "per_tensor":
|
||||
if current_platform.is_fp8_fnuz():
|
||||
input_scale = getattr(layer, "input_scale", None)
|
||||
weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
input_scale=input_scale,
|
||||
)
|
||||
if input_scale is not None:
|
||||
layer.input_scale = Parameter(input_scale, requires_grad=False)
|
||||
else:
|
||||
max_w_scale = layer.weight_scale
|
||||
weight = layer.weight
|
||||
|
||||
max_w_scale, weight = requantize_with_max_scale(
|
||||
weight=weight,
|
||||
weight_scale=max_w_scale,
|
||||
logical_widths=layer.logical_widths,
|
||||
)
|
||||
|
||||
layer.weight = Parameter(weight.t(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
|
||||
|
||||
# If channelwise, scales are already lined up, so just transpose.
|
||||
elif self.weight_qscheme == "per_channel":
|
||||
weight = layer.weight
|
||||
|
||||
if current_platform.is_fp8_fnuz():
|
||||
input_scale = getattr(layer, "input_scale", None)
|
||||
weight, weight_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight=weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
input_scale=input_scale,
|
||||
)
|
||||
if input_scale is not None:
|
||||
layer.input_scale = Parameter(input_scale, requires_grad=False)
|
||||
else:
|
||||
weight_scale = layer.weight_scale.data
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
if self.activation_quant_key.scale.group_shape == GroupShape.PER_TOKEN:
|
||||
weight_scale = weight_scale.view(-1, 1)
|
||||
layer.weight = Parameter(weight.t(), requires_grad=False)
|
||||
# required by torch.compile to be torch.nn.Parameter
|
||||
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown quantization scheme {self.weight_qscheme}")
|
||||
|
||||
# INPUT SCALE
|
||||
if self.is_static_input_scheme:
|
||||
layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False)
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
|
||||
# WEIGHT
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# WEIGHT SCALE
|
||||
# TODO: update create_xxx_parameter functions to return
|
||||
# the newly added parameters
|
||||
if self.weight_qscheme == "per_channel":
|
||||
weight_scale = ChannelQuantScaleParameter(
|
||||
data=torch.empty((sum(output_partition_sizes)), dtype=torch.float32),
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
else:
|
||||
assert self.weight_qscheme == "per_tensor"
|
||||
weight_scale = PerTensorScaleParameter(
|
||||
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
# min requirement for fp8 kernels
|
||||
weight_scale[:] = torch.finfo(torch.float32).min
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
# INPUT SCALE
|
||||
if self.is_static_input_scheme:
|
||||
input_scale = PerTensorScaleParameter(
|
||||
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
input_scale[:] = torch.finfo(torch.float32).min
|
||||
layer.register_parameter("input_scale", input_scale)
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
weight_shape=layer.weight.shape,
|
||||
input_dtype=self.input_dtype,
|
||||
out_dtype=self.out_dtype,
|
||||
module_name=self.__class__.__name__,
|
||||
)
|
||||
|
||||
expose_input_quant_key(layer, self.fp8_linear)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor | QuantizedActivation,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
|
||||
class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
supported_activation_quant_keys = [kFp8Dynamic128Sym]
|
||||
supported_weight_quant_keys = [
|
||||
kFp8Static128BlockSym,
|
||||
kFp8Static128BlockE8M0Sym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.weight_scale_dtype = weight_quant_key.scale.dtype
|
||||
if self.weight_quant_key == kFp8Static128BlockE8M0Sym:
|
||||
# TODO: oracle needs to support kFp8Static128BlockE8M0Sym.
|
||||
self.weight_quant_key = kFp8Static128BlockSym
|
||||
self.weight_block_size = list(weight_quant_key.scale.group_shape)
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return QuarkW8A8Fp8.get_min_capability()
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
# Quark exports the dequant multiplier as ``weight_scale`` (the same
|
||||
# numerical convention as DeepSeek's ``weight_scale_inv``). Kernels
|
||||
# multiply ``weight * scale``; do not invert here.
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
input_size = kwargs.get("input_size", input_size_per_partition)
|
||||
output_size = kwargs.get("output_size", sum(output_partition_sizes))
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
validate_fp8_block_shape(
|
||||
layer,
|
||||
input_size,
|
||||
output_size,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
self.weight_block_size,
|
||||
)
|
||||
|
||||
weight = create_fp8_weight_parameter(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
scale_dtype = (
|
||||
self.weight_scale_dtype
|
||||
if self.weight_scale_dtype != torch.float32
|
||||
else None
|
||||
)
|
||||
weight_scale = create_fp8_scale_parameter(
|
||||
BlockQuantScaleParameter,
|
||||
output_partition_sizes,
|
||||
input_size_per_partition,
|
||||
self.weight_block_size,
|
||||
weight_loader,
|
||||
scale_dtype=scale_dtype,
|
||||
)
|
||||
# Match Quark-exported checkpoints (``.weight_scale``).
|
||||
# ``Fp8LinearMethod.create_weights`` registers weight_scale_inv, which has the
|
||||
# same semantic meaning as Quark's `weight_scale`.
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
weight_shape=layer.weight.shape,
|
||||
input_dtype=self.input_dtype,
|
||||
out_dtype=self.out_dtype,
|
||||
module_name=self.__class__.__name__,
|
||||
)
|
||||
logger.info_once(
|
||||
"Selected %s for QuarkW8A8Fp8PerBlock",
|
||||
type(self.fp8_linear).__name__,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
@@ -1,165 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
init_int8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8StaticTensorSym,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class QuarkW8A8Int8(QuarkScheme):
|
||||
supported_activation_quant_keys = [
|
||||
kInt8StaticTensorSym,
|
||||
kInt8StaticTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8DynamicTokenAsym,
|
||||
]
|
||||
supported_weight_quant_keys = [
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorSym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.qscheme = (
|
||||
"per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor"
|
||||
)
|
||||
|
||||
assert activation_quant_key is not None
|
||||
self.is_static_input_scheme = activation_quant_key.scale.static
|
||||
self.input_symmetric = activation_quant_key.symmetric
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
# turing and up
|
||||
return 75
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
layer.logical_widths = output_partition_sizes
|
||||
|
||||
# Quark stores per-channel weight_scale as 1D [N]; reshape to [N, 1].
|
||||
def _scale_weight_loader(
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
if loaded_weight.dim() == 1:
|
||||
loaded_weight = loaded_weight.unsqueeze(-1)
|
||||
return weight_loader(param, loaded_weight, *args, **kwargs)
|
||||
|
||||
self.kernel = init_int8_linear_kernel(
|
||||
is_channelwise=(self.qscheme == "per_channel"),
|
||||
is_static_input_scheme=(self.is_static_input_scheme is True),
|
||||
input_symmetric=(self.input_symmetric is True),
|
||||
module_name=self.__class__.__name__,
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
sum(output_partition_sizes), input_size_per_partition, dtype=torch.int8
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# WEIGHT SCALE
|
||||
if self.qscheme == "per_channel":
|
||||
weight_scale = ChannelQuantScaleParameter(
|
||||
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32),
|
||||
output_dim=0,
|
||||
weight_loader=_scale_weight_loader,
|
||||
)
|
||||
ChannelQuantZPParameter = ChannelQuantScaleParameter
|
||||
weight_zero_point = ChannelQuantZPParameter(
|
||||
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.int8),
|
||||
output_dim=0,
|
||||
weight_loader=_scale_weight_loader,
|
||||
)
|
||||
else:
|
||||
assert self.qscheme == "per_tensor"
|
||||
weight_scale = PerTensorScaleParameter(
|
||||
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
PerTensorZPParameter = PerTensorScaleParameter
|
||||
weight_zero_point = PerTensorZPParameter(
|
||||
data=torch.empty(len(output_partition_sizes), dtype=torch.int8),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
layer.register_parameter("weight_zero_point", weight_zero_point)
|
||||
|
||||
# INPUT SCALE
|
||||
input_zero_point = None
|
||||
input_scale = None
|
||||
if self.is_static_input_scheme:
|
||||
input_scale = BasevLLMParameter(
|
||||
data=torch.empty(1, dtype=torch.float32), weight_loader=weight_loader
|
||||
)
|
||||
|
||||
input_zero_point = BasevLLMParameter(
|
||||
data=torch.empty(1, dtype=torch.int8), weight_loader=weight_loader
|
||||
)
|
||||
|
||||
layer.register_parameter("input_scale", input_scale)
|
||||
layer.register_parameter("input_zero_point", input_zero_point)
|
||||
if not hasattr(layer, "azp_adj"):
|
||||
layer.register_parameter("azp_adj", None)
|
||||
|
||||
# Checkpoints are serialized in quark format, which is
|
||||
# different from the format the kernel may want. Handle repacking here.
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.register_parameter("weight_zero_point", None)
|
||||
delattr(layer, "weight_zero_point")
|
||||
if self.input_symmetric:
|
||||
layer.register_parameter("input_zero_point", None)
|
||||
delattr(layer, "input_zero_point")
|
||||
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
@@ -1,136 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
)
|
||||
|
||||
QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None
|
||||
|
||||
|
||||
def deep_compare(dict1: Any, dict2: Any) -> bool:
|
||||
if type(dict1) is not type(dict2):
|
||||
return False
|
||||
if isinstance(dict1, dict):
|
||||
if dict1.keys() != dict2.keys():
|
||||
return False
|
||||
return all(deep_compare(dict1[k], dict2[k]) for k in dict1)
|
||||
elif isinstance(dict1, list):
|
||||
# `dict1` may be a list of dict.
|
||||
return all(deep_compare(dict1[i], dict2[i]) for i in range(len(dict1)))
|
||||
else:
|
||||
return dict1 == dict2
|
||||
|
||||
|
||||
def should_ignore_layer(
|
||||
layer_name: str | None,
|
||||
ignore: Iterable[str],
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
*,
|
||||
check_children: bool = False,
|
||||
) -> bool:
|
||||
if layer_name is None:
|
||||
return False
|
||||
|
||||
# MoE layers are currently all-or-nothing: if any child is ignored,
|
||||
# the parent layer must be ignored as well. For example, the
|
||||
# amd/GLM-5.2-MXFP4 config ignores children like
|
||||
# model.layers.78.mlp.experts.*.down_proj, while the layer checked
|
||||
# here is the parent model.layers.N.mlp.experts.
|
||||
# See:
|
||||
# https://huggingface.co/amd/GLM-5.2-MXFP4/blob/main/config.json#L793-L795
|
||||
if check_children and any(
|
||||
target == layer_name or target.startswith(layer_name + ".")
|
||||
for target in ignore
|
||||
if not target.startswith("re:")
|
||||
):
|
||||
return True
|
||||
|
||||
# A direct fused-layer pattern takes precedence over expansion. For
|
||||
# model.layers.0.self_attn.qkv_proj,
|
||||
# ignore=["re:.*qkv_proj.*"] yields [{"re:.*qkv_proj.*"}]. In contrast,
|
||||
# ignore=["re:.*[qkv]_proj"] yields one matching set per expanded shard.
|
||||
per_shard_matches = find_matching_patterns(layer_name, ignore, fused_mapping)
|
||||
shards_ignored = [len(matches) > 0 for matches in per_shard_matches]
|
||||
if any(shards_ignored) and not all(shards_ignored):
|
||||
raise ValueError(
|
||||
f"Found different quantization schemes for the shards of "
|
||||
f"{layer_name}. vLLM requires all to use the same scheme."
|
||||
)
|
||||
return all(shards_ignored)
|
||||
|
||||
|
||||
def parse_w4a16_int4_weight_config(
|
||||
weight_config: Mapping[str, Any],
|
||||
) -> tuple[int, bool]:
|
||||
"""Parse required W4A16 INT4/UINT4 weight fields from Quark config."""
|
||||
if "group_size" not in weight_config:
|
||||
raise ValueError(
|
||||
"Quark W4A16 INT4/UINT4 configs must specify weight.group_size"
|
||||
)
|
||||
if "symmetric" not in weight_config:
|
||||
raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric")
|
||||
|
||||
group_size = weight_config["group_size"]
|
||||
is_symmetric = weight_config["symmetric"]
|
||||
if not isinstance(group_size, int) or group_size <= 0:
|
||||
raise ValueError(
|
||||
f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}"
|
||||
)
|
||||
if not isinstance(is_symmetric, bool):
|
||||
raise ValueError(
|
||||
f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}"
|
||||
)
|
||||
return group_size, is_symmetric
|
||||
|
||||
|
||||
def canonicalize_quark_packed_int4(
|
||||
packed_weight: torch.Tensor,
|
||||
*,
|
||||
pack_reorder: bool,
|
||||
is_symmetric: bool,
|
||||
pack_factor: int = 8,
|
||||
) -> torch.Tensor:
|
||||
"""Convert Quark export nibble layout to AWQ checkpoint layout."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
_REVERSE_AWQ_PACK_ORDER,
|
||||
)
|
||||
|
||||
if pack_reorder:
|
||||
source_order = torch.tensor(
|
||||
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
|
||||
)
|
||||
else:
|
||||
source_order = torch.arange(
|
||||
pack_factor, device=packed_weight.device, dtype=torch.int32
|
||||
)
|
||||
target_order = torch.tensor(
|
||||
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
|
||||
)
|
||||
source_shifts = source_order * 4
|
||||
target_shifts = target_order * 4
|
||||
|
||||
values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF
|
||||
if is_symmetric:
|
||||
values = values ^ 0x8
|
||||
packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1)
|
||||
return packed.to(torch.int32)
|
||||
|
||||
|
||||
# utility for tensor dims > 2 cases
|
||||
def quark_quantize_weight_to_mxfp4(w: torch.Tensor):
|
||||
assert w.dtype == torch.bfloat16, (
|
||||
"Quark dynamic quantization is supported only for fp16 weights and only to MXF4"
|
||||
)
|
||||
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
*dims, d = w.shape
|
||||
w, w_scales = dynamic_mxfp4_quant(w.reshape(-1, d))
|
||||
return w.view(*dims, d // 2), w_scales.view(*dims, d // 32)
|
||||
@@ -20,40 +20,8 @@ def get_quark_ocp_mx_group_size(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
layer_name: str,
|
||||
) -> int | None:
|
||||
"""Return the OCP MX group size for a quantized Quark linear layer."""
|
||||
if quant_config is None:
|
||||
return None
|
||||
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer
|
||||
|
||||
if not isinstance(quant_config, QuarkConfig):
|
||||
return None
|
||||
|
||||
if should_ignore_layer(
|
||||
layer_name,
|
||||
ignore=quant_config.quant_config.get("exclude") or [],
|
||||
fused_mapping=quant_config.packed_modules_mapping,
|
||||
):
|
||||
return None
|
||||
|
||||
layer_quant_config = (
|
||||
quant_config.get_layer_quant_config_from_name(layer_name)
|
||||
or quant_config.quant_config.get("global_quant_config")
|
||||
or {}
|
||||
)
|
||||
weight_quant = layer_quant_config.get("weight")
|
||||
input_quant = layer_quant_config.get("input_tensors")
|
||||
if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant):
|
||||
return None
|
||||
|
||||
assert weight_quant is not None
|
||||
group_size = weight_quant.get("group_size")
|
||||
if group_size is None:
|
||||
block_size = weight_quant.get("block_size")
|
||||
if isinstance(block_size, list | tuple) and len(block_size) >= 2:
|
||||
group_size = block_size[1]
|
||||
return int(group_size) if group_size is not None else None
|
||||
"""Quark support was removed; kept for callers that probe MX group size."""
|
||||
return None
|
||||
|
||||
|
||||
def is_shared_expert_quant_fse_compatible(
|
||||
@@ -75,12 +43,9 @@ def is_shared_expert_quant_fse_compatible(
|
||||
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
|
||||
|
||||
if projection_names is None:
|
||||
projection_names = ["gate_up_proj", "down_proj"]
|
||||
@@ -104,191 +69,12 @@ def is_shared_expert_quant_fse_compatible(
|
||||
shared_expert_targets = None
|
||||
|
||||
if shared_expert_targets is not None:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
ONLINE_SHARED_EXPERT_QUANTIZERS,
|
||||
)
|
||||
|
||||
# NOTE: online shared experts quantization check is only implemented for quark
|
||||
# quant method at the moment. This can be extended here for other quant methods.
|
||||
# Requirement is:
|
||||
# - Retrieve routed weight quant key
|
||||
# - Retrieve routed activation quant key
|
||||
# and ensure the online quantization set on shared expert match them.
|
||||
if isinstance(quant_config, QuarkConfig):
|
||||
routed_weight_key, routed_activation_key, _ = (
|
||||
quant_config.get_quant_method_target(expert_prefix, RoutedExperts)
|
||||
)
|
||||
else:
|
||||
reason = (
|
||||
"online quantization FSE compatibility check is not implemented for "
|
||||
f"{type(quant_config).__name__}"
|
||||
)
|
||||
logger.warning_once(reason)
|
||||
return False, reason
|
||||
|
||||
if routed_weight_key not in ONLINE_SHARED_EXPERT_QUANTIZERS:
|
||||
return (
|
||||
False,
|
||||
"online shared-expert quantization is not supported for routed "
|
||||
f"expert weight key {routed_weight_key}",
|
||||
)
|
||||
|
||||
for shared_expert_target in shared_expert_targets:
|
||||
if shared_expert_target is None:
|
||||
return (
|
||||
False,
|
||||
"online quantization targets only part of the shared expert at "
|
||||
f"{shared_expert_prefix}",
|
||||
)
|
||||
|
||||
_, _, _, shared_quant_spec, shared_method_cls = shared_expert_target
|
||||
if shared_method_cls in (None, UnquantizedLinearMethod):
|
||||
return (
|
||||
False,
|
||||
"online quantization excludes FSE weights at "
|
||||
f"{shared_expert_prefix}",
|
||||
)
|
||||
assert issubclass(shared_method_cls, OnlineLinearBase)
|
||||
|
||||
# NOTE: We can not rely on shared_quant_spec.activation_quant_key
|
||||
# as _ONLINE_SHORTHANDS: dict[str, QuantSpec] in quantization.py
|
||||
# does not define the default activation quant key.
|
||||
# `online_quant_config.resolve_quant_method_cls` should return the
|
||||
# activation quant key as well. Only some online methods define it
|
||||
# on the class, so compare the weight key first.
|
||||
if (
|
||||
shared_quant_spec.weight != routed_weight_key
|
||||
or shared_method_cls.activation_quant_key != routed_activation_key
|
||||
):
|
||||
return (
|
||||
False,
|
||||
"online shared-expert quantization keys do not match the "
|
||||
"routed expert keys",
|
||||
)
|
||||
return True, None
|
||||
|
||||
if isinstance(quant_config, DeepseekV4FP8Config):
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
if quant_config.expert_dtype != "fp4":
|
||||
return False, "DeepSeek-V4 routed experts are not MXFP4"
|
||||
|
||||
hf_config = get_current_vllm_config().model_config.hf_config
|
||||
|
||||
# TODO: This is adapted from former `_shared_experts_are_fp4`, and
|
||||
# needs to be cleaned up this . There should not be Quark-specific
|
||||
# logic in DeepseekV4FP8Config.
|
||||
quantization_config = getattr(hf_config, "quantization_config", None)
|
||||
if quantization_config is None:
|
||||
return False, "DeepSeek-V4 has no quantization configuration"
|
||||
|
||||
if not quant_config._is_quark_mxfp4_ocp(quantization_config):
|
||||
return False, "DeepSeek-v4 FSE is only implemented/tested with Quark MXFP4"
|
||||
|
||||
layer_idx = extract_layer_index(shared_expert_prefix)
|
||||
if layer_idx >= hf_config.num_hidden_layers:
|
||||
shared_expert_prefix = (
|
||||
f"mtp.{layer_idx - hf_config.num_hidden_layers}.ffn.shared_experts"
|
||||
)
|
||||
else:
|
||||
shared_expert_prefix = f"layers.{layer_idx}.ffn.shared_experts"
|
||||
|
||||
if any(
|
||||
entry.startswith(shared_expert_prefix)
|
||||
for entry in quantization_config.get("exclude") or []
|
||||
if isinstance(entry, str)
|
||||
):
|
||||
return (
|
||||
False,
|
||||
f"DeepSeek-V4 excludes shared experts at {shared_expert_prefix}",
|
||||
)
|
||||
|
||||
shared_expert_weight_name = f"{shared_expert_prefix}.w1"
|
||||
layer_quant_config = quantization_config.get("layer_quant_config") or {}
|
||||
layer_config = layer_quant_config.get(shared_expert_weight_name)
|
||||
if layer_config is None:
|
||||
layer_config = next(
|
||||
(
|
||||
config
|
||||
for pattern, config in layer_quant_config.items()
|
||||
if isinstance(pattern, str)
|
||||
and pattern.startswith("re:")
|
||||
and re.fullmatch(
|
||||
pattern.removeprefix("re:"), shared_expert_weight_name
|
||||
)
|
||||
),
|
||||
None,
|
||||
)
|
||||
shared_weight_config = (
|
||||
layer_config or quantization_config.get("global_quant_config") or {}
|
||||
).get("weight") or {}
|
||||
if shared_weight_config.get("dtype") == "fp4":
|
||||
return True, None
|
||||
return (
|
||||
False,
|
||||
f"DeepSeek-V4 shared experts at {shared_expert_prefix} are not MXFP4",
|
||||
)
|
||||
|
||||
if isinstance(quant_config, QuarkConfig):
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
should_ignore_layer,
|
||||
)
|
||||
|
||||
# TODO: layer_type_quant_config is not taken into account here.
|
||||
assert "exclude" in quant_config.quant_config
|
||||
assert "global_quant_config" in quant_config.quant_config
|
||||
|
||||
exclude = quant_config.quant_config["exclude"]
|
||||
|
||||
# should_ignore_layer raises a rightful ValueError in case different shards
|
||||
# have a different ignore policy, as unsupported.
|
||||
is_excluded = any(
|
||||
should_ignore_layer(
|
||||
f"{shared_expert_prefix}.{projection_name}",
|
||||
ignore=exclude,
|
||||
fused_mapping=quant_config.packed_modules_mapping,
|
||||
)
|
||||
for projection_name in projection_names
|
||||
)
|
||||
if is_excluded:
|
||||
return False, f"Quark excludes shared experts at {shared_expert_prefix}"
|
||||
|
||||
global_quant_config = quant_config.quant_config["global_quant_config"]
|
||||
|
||||
def get_projection_quant_configs(layer_name: str) -> list[object]:
|
||||
module_prefix, _, projection_name = layer_name.rpartition(".")
|
||||
packed_projection_names = quant_config.packed_modules_mapping.get(
|
||||
projection_name, [projection_name]
|
||||
)
|
||||
return [
|
||||
quant_config.get_layer_quant_config_from_name(
|
||||
f"{module_prefix}.{packed_projection_name}"
|
||||
)
|
||||
or global_quant_config
|
||||
for packed_projection_name in packed_projection_names
|
||||
]
|
||||
|
||||
expert_quant_config = (
|
||||
quant_config.get_layer_quant_config_from_name(expert_prefix)
|
||||
or global_quant_config
|
||||
)
|
||||
shared_expert_quant_configs = [
|
||||
config
|
||||
for projection_name in projection_names
|
||||
for config in get_projection_quant_configs(
|
||||
f"{shared_expert_prefix}.{projection_name}"
|
||||
)
|
||||
]
|
||||
if all(config == expert_quant_config for config in shared_expert_quant_configs):
|
||||
return True, None
|
||||
return (
|
||||
False,
|
||||
"Quark uses different quantization configurations for routed and "
|
||||
f"shared experts at {shared_expert_prefix}",
|
||||
reason = (
|
||||
"online quantization FSE compatibility check is not implemented for "
|
||||
f"{type(quant_config).__name__}"
|
||||
)
|
||||
logger.warning_once(reason)
|
||||
return False, reason
|
||||
|
||||
if isinstance(quant_config, Fp8Config):
|
||||
if quant_config.store_dtype is not None:
|
||||
|
||||
@@ -52,16 +52,6 @@ def is_fp8(x: torch.dtype | torch.Tensor) -> bool:
|
||||
|
||||
def get_fp8_block_weight_scale(layer: torch.nn.Module) -> torch.Tensor | None:
|
||||
"""Return the block-FP8 weight scale for supported quant methods."""
|
||||
# Local import avoids a circular import: quark_w8a8_fp8 imports this module.
|
||||
from vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8 import (
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
)
|
||||
|
||||
if isinstance(
|
||||
getattr(layer, "scheme", None),
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
):
|
||||
return getattr(layer, "weight_scale", None)
|
||||
return getattr(layer, "weight_scale_inv", None)
|
||||
|
||||
|
||||
|
||||
@@ -549,15 +549,6 @@ def mxfp4_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if current_platform.is_xpu():
|
||||
return xpu_mxfp4_quantize(x)
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
if is_aiter_found_and_supported() and x.dtype == torch.bfloat16:
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
quark_quantize_weight_to_mxfp4,
|
||||
)
|
||||
|
||||
return quark_quantize_weight_to_mxfp4(x)
|
||||
|
||||
quant_tensor, scale, _ = downcast_to_mxfp(x, axis=-1)
|
||||
|
||||
return quant_tensor, scale
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sparse attention indexer layers on aiter's paged MXFP4 kernels, for ROCm.
|
||||
|
||||
With ``indexer_kv_dtype="mxfp4"`` on gfx950, the model writes the indexer K
|
||||
cache in the order aiter's MQA-logits kernel reads (see
|
||||
`rocm_paged_mxfp4_cache_layout`), and the kernel scores the cache in place.
|
||||
Requires `RocmMxfp4IndexerMetadataBuilder` metadata.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import (
|
||||
rocm_mxfp4_sparse_attn_indexer,
|
||||
rocm_mxfp4_sparse_mqa_indexer,
|
||||
)
|
||||
|
||||
|
||||
class RocmSparseAttnIndexer(SparseAttnIndexer):
|
||||
"""`SparseAttnIndexer` whose HIP path walks the paged MXFP4 cache with
|
||||
aiter's kernel. A two-level indexer's source layer also publishes its
|
||||
candidate pool here."""
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
assert isinstance(q_quant, tuple) and self.skip_k_cache_insert, (
|
||||
"the ROCm MXFP4 indexer takes (values, scales) Q and a K cache "
|
||||
"the model already wrote"
|
||||
)
|
||||
q_values, q_scale = q_quant
|
||||
return rocm_mxfp4_sparse_attn_indexer(
|
||||
hidden_states,
|
||||
self.k_cache.prefix,
|
||||
self.k_cache.kv_cache,
|
||||
q_values,
|
||||
q_scale,
|
||||
weights,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.topk_indices_buffer,
|
||||
compress_ratio=self.compress_ratio,
|
||||
candidate_blocks=self.candidate_blocks,
|
||||
candidate_block_size=self.candidate_block_size,
|
||||
candidate_write=self.candidate_write,
|
||||
)
|
||||
|
||||
|
||||
class RocmSparseMQAIndexer(nn.Module):
|
||||
"""DeepSeek-V4.1's candidate-consuming indexer on aiter's paged MXFP4
|
||||
MQA-logits kernel: the consumers score only the candidate pool the source
|
||||
layer published, read straight from the paged cache.
|
||||
|
||||
Built and called as `SparseMQAIndexer` is, so the model can take either.
|
||||
"""
|
||||
|
||||
weights_dtype = torch.float32
|
||||
"""The scorer's weights and logits stay fp32: gfx950 has no packed bf16
|
||||
arithmetic, so its head reduce costs the same in either dtype."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
k_cache,
|
||||
topk_tokens: int,
|
||||
head_dim: int,
|
||||
max_total_seq_len: int,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
candidate_blocks: torch.Tensor,
|
||||
candidate_block_size: int,
|
||||
):
|
||||
super().__init__()
|
||||
self.k_cache = k_cache
|
||||
self.topk_tokens = topk_tokens
|
||||
self.head_dim = head_dim
|
||||
# aiter's kernel bounds a row by the model length in compressed
|
||||
# positions, not by the prefill buffer SparseMQAIndexer sizes.
|
||||
model_config = get_current_vllm_config().model_config
|
||||
self.max_model_len = model_config.max_model_len // k_cache.compress_ratio
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self.candidate_blocks = candidate_blocks
|
||||
self.candidate_block_size = candidate_block_size
|
||||
self.num_candidate_cols = candidate_blocks.shape[1] * candidate_block_size
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
assert k is None, "the model writes the indexer K cache"
|
||||
q_values, q_scale = q_quant
|
||||
return rocm_mxfp4_sparse_mqa_indexer(
|
||||
hidden_states,
|
||||
self.k_cache.prefix,
|
||||
self.k_cache.kv_cache,
|
||||
q_values,
|
||||
q_scale,
|
||||
weights,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.topk_indices_buffer,
|
||||
self.k_cache.compress_ratio,
|
||||
self.candidate_blocks,
|
||||
self.candidate_block_size,
|
||||
self.num_candidate_cols,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,209 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sparse Attention Indexer that scores only the candidate blocks.
|
||||
|
||||
DeepSeek V4.1 two-level selection: the candidate-source indexer publishes
|
||||
the top candidate blocks and later indexers pick their top-k inside them.
|
||||
`SparseAttnIndexer` does that by computing dense logits over the whole
|
||||
context and masking; this layer instead calls DeepGEMM's sparse MQA-logits
|
||||
kernels on the candidate blocks only, so the work is O(candidate blocks)
|
||||
instead of O(context). It requires the `DeepseekV41SparseIndexerBackend`
|
||||
metadata (see ``AttentionConfig.indexer_sparse_logits``).
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
has_deep_select,
|
||||
sparse_mqa_logits_paged_decode,
|
||||
sparse_mqa_logits_prefill_chunk,
|
||||
)
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
_gather_workspace_shapes,
|
||||
kv_cache_as_quant_view,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import has_deep_gemm_sparse_mqa
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerPrefillChunkMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_indexer import (
|
||||
DeepseekV41SparseIndexerMetadata,
|
||||
)
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
|
||||
def _prefill_k_workspaces(
|
||||
total_seq_lens: int, head_dim: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The packed MXFP4 K-gather workspace ``(values, scales)``, shared with
|
||||
the dense indexer layers of the same model."""
|
||||
values_spec, scales_spec = _gather_workspace_shapes(
|
||||
total_seq_lens, head_dim, current_platform.fp8_dtype(), use_fp4_cache=True
|
||||
)
|
||||
k_quant, k_scale = current_workspace_manager().get_simultaneous(
|
||||
values_spec, scales_spec
|
||||
)
|
||||
return k_quant, k_scale
|
||||
|
||||
|
||||
def _gather_prefill_chunk_k(
|
||||
kv_cache: torch.Tensor,
|
||||
k_quant_full: torch.Tensor,
|
||||
k_scale_full: torch.Tensor,
|
||||
chunk: DeepseekV32IndexerPrefillChunkMetadata,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Gather one prefill chunk's paged K into the packed workspace."""
|
||||
assert chunk.local_cu_seq_lens is not None
|
||||
k_quant = k_quant_full[: chunk.max_local_total_seq_lens]
|
||||
k_scale = k_scale_full[: chunk.max_local_total_seq_lens]
|
||||
if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0:
|
||||
ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache,
|
||||
k_quant,
|
||||
k_scale,
|
||||
chunk.block_table,
|
||||
chunk.local_cu_seq_lens,
|
||||
)
|
||||
return k_quant, k_scale
|
||||
|
||||
|
||||
class SparseMQAIndexer(nn.Module):
|
||||
"""Candidate-consuming indexer on DeepGEMM's sparse MQA-logits kernels.
|
||||
|
||||
Only valid for indexer layers that read candidate blocks with the MXFP4
|
||||
indexer cache on SM100. The K cache is written by the model before this
|
||||
runs; ``forward`` takes the same arguments as `SparseAttnIndexer` so the
|
||||
attention layer can call either.
|
||||
"""
|
||||
|
||||
weights_dtype = torch.bfloat16
|
||||
"""Per-head weights dtype the sparse kernels take. The fused Q RoPE-quant
|
||||
kernel writes it directly so no cast runs per step."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
k_cache,
|
||||
topk_tokens: int,
|
||||
head_dim: int,
|
||||
max_total_seq_len: int,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
candidate_blocks: torch.Tensor,
|
||||
candidate_block_size: int,
|
||||
):
|
||||
super().__init__()
|
||||
if not (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and has_deep_gemm_sparse_mqa()
|
||||
):
|
||||
raise ValueError(
|
||||
"SparseMQAIndexer requires an SM100-class GPU and DeepGEMM >= 2.8."
|
||||
)
|
||||
if not has_deep_select():
|
||||
raise ValueError(
|
||||
"SparseMQAIndexer requires the DeepSelect top-k extension "
|
||||
"(vllm._deepselect_C)."
|
||||
)
|
||||
self.k_cache = k_cache
|
||||
self.topk_tokens = topk_tokens
|
||||
self.head_dim = head_dim
|
||||
self.max_total_seq_len = max_total_seq_len
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self.candidate_blocks = candidate_blocks
|
||||
self.candidate_block_size = candidate_block_size
|
||||
|
||||
def _reserve_workspaces(self, device: torch.device) -> None:
|
||||
"""Profiling run: claim the K-gather workspace and the peak sparse
|
||||
logits allocation so the memory estimate covers them."""
|
||||
_prefill_k_workspaces(self.max_total_seq_len, self.head_dim)
|
||||
max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
|
||||
torch.empty(max_logits_bytes, dtype=torch.uint8, device=device)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
self._reserve_workspaces(hidden_states.device)
|
||||
return self.topk_indices_buffer
|
||||
|
||||
metadata = attn_metadata[self.k_cache.prefix]
|
||||
assert isinstance(metadata, DeepseekV41SparseIndexerMetadata), (
|
||||
"SparseMQAIndexer needs DeepseekV41SparseIndexerBackend metadata"
|
||||
)
|
||||
assert k is None, "the model writes the indexer K cache"
|
||||
q_values, q_scale = q_quant
|
||||
kv_cache = self.k_cache.kv_cache
|
||||
topk_tokens = self.topk_tokens
|
||||
topk_indices_buffer = self.topk_indices_buffer
|
||||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||||
sparse_block_kv = metadata.sparse_block_kv
|
||||
|
||||
if metadata.num_prefills > 0:
|
||||
prefill = metadata.prefill
|
||||
sparse_chunks = metadata.sparse_prefill
|
||||
assert prefill is not None and sparse_chunks is not None
|
||||
k_quant_full, k_scale_full = _prefill_k_workspaces(
|
||||
self.max_total_seq_len, self.head_dim
|
||||
)
|
||||
for chunk, rows in zip(prefill.chunks, sparse_chunks):
|
||||
k_quant, k_scale = _gather_prefill_chunk_k(
|
||||
kv_cache, k_quant_full, k_scale_full, chunk
|
||||
)
|
||||
if chunk.local_total_seq_lens == 0:
|
||||
continue # buffer already holds -1
|
||||
start, end = chunk.token_start, chunk.token_end
|
||||
rows.kernel_metadata = sparse_mqa_logits_prefill_chunk(
|
||||
q_values[start:end].view(torch.int8),
|
||||
q_scale[start:end],
|
||||
k_quant.view(torch.int8),
|
||||
k_scale.view(torch.int32).squeeze(-1),
|
||||
weights[start:end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
self.candidate_blocks[start:end],
|
||||
self.candidate_block_size,
|
||||
sparse_block_kv,
|
||||
topk_tokens,
|
||||
topk_indices_buffer[start:end, :topk_tokens],
|
||||
sparse_indices=rows.sparse_indices,
|
||||
end=rows.end,
|
||||
col_indices=rows.col_indices,
|
||||
kernel_metadata=rows.kernel_metadata,
|
||||
)
|
||||
|
||||
if metadata.num_decodes > 0:
|
||||
decode_rows = metadata.sparse_decode
|
||||
assert decode_rows is not None and decode_rows.block_table is not None
|
||||
assert decode_rows.row_indices is not None
|
||||
num_rows = decode_rows.row_ke.shape[0]
|
||||
if num_rows > 0:
|
||||
decode_rows.kernel_metadata = sparse_mqa_logits_paged_decode(
|
||||
q_values[:num_rows].view(torch.int8).unsqueeze(1),
|
||||
q_scale[:num_rows].unsqueeze(1),
|
||||
kv_cache_as_quant_view(kv_cache, self.head_dim, use_fp4_cache=True),
|
||||
weights[:num_rows],
|
||||
decode_rows.row_ke,
|
||||
decode_rows.block_table,
|
||||
decode_rows.row_indices,
|
||||
self.candidate_blocks[:num_rows],
|
||||
self.candidate_block_size,
|
||||
sparse_block_kv,
|
||||
topk_tokens,
|
||||
topk_indices_buffer[:num_rows, :topk_tokens],
|
||||
row_ks=decode_rows.row_ks,
|
||||
sparse_indices=decode_rows.sparse_indices,
|
||||
end=decode_rows.end,
|
||||
col_indices=decode_rows.col_indices,
|
||||
kernel_metadata=decode_rows.kernel_metadata,
|
||||
)
|
||||
return topk_indices_buffer
|
||||
Reference in New Issue
Block a user