[Hy4] support Hy4-preview model (#54160)

Signed-off-by: chengvjiang <chengvjiang@tencent.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: chengvjiang <chengvjiang@tencent.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: russellfeng <russellfeng@tencent.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Cheng Jiang
2026-08-29 12:07:28 +08:00
committed by GitHub
co-authored by chengvjiang Jee Jee Li Bugen Zhao russellfeng Jee Jee Li
parent 6c18a54648
commit b2f685834a
47 changed files with 6260 additions and 217 deletions
@@ -79,12 +79,27 @@ class TrtLlmFp8ExpertsBase:
activation_key,
activation_format,
)
if not supported or moe_config.num_experts <= 2048:
if not supported:
return supported, reason
return False, (
"FlashInfer TRTLLM routing supports at most 2048 experts, "
f"but got {moe_config.num_experts}"
)
# FlashInfer accepts gemm1_alpha/beta/clamp_limit only for MXFP8; the
# DeepSeek-FP8 block-scale kernel rejects them and the per-tensor kernel
# has no clamp at all, so a model-requested SwiGLU clamp cannot be
# honored on those paths.
if (
moe_config.swiglu_limit is not None
or moe_config.swiglu_alpha is not None
or moe_config.swiglu_beta is not None
) and (weight_key, activation_key) != (kMxfp8Static, kMxfp8Dynamic):
return False, (
"the TRTLLM FP8 kernels apply the SwiGLU alpha/beta/clamp "
"parameters only for MXFP8 weights"
)
if moe_config.num_experts > 2048:
return False, (
"FlashInfer TRTLLM routing supports at most 2048 experts, "
f"but got {moe_config.num_experts}"
)
return True, None
def __init__(
self,
@@ -1,11 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.hpc.gated_mla import (
hpc_gated_mla_gemm,
hpc_gated_mla_supported,
)
from vllm.model_executor.layers.hpc.hpc_ihc import (
HpcIHCHead,
HpcIHCPost,
HpcIHCPre,
)
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
from vllm.model_executor.layers.hpc.rope_norm import HpcRopeNorm, QkNormPolicy
__all__ = [
"HpcIHCHead",
"HpcIHCPost",
"HpcIHCPre",
"HpcModule",
"HpcRopeNorm",
"QkNormPolicy",
"hpc_gated_mla_gemm",
"hpc_gated_mla_supported",
]
+113
View File
@@ -0,0 +1,113 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Enablement gate for the HPC fused gated-MLA GEMM.
``hpc.gated_mla_gemm`` fuses the MLA output gating -- the gate projection, the
sigmoid and the elementwise product -- into a single kernel launch, replacing
the eager ``attn_out * sigmoid(hidden_states @ weight.T)`` sequence.
Kernel constraints:
- Requires VLLM_ENABLE_HPC_OPS=1
- Requires the hpc package (.so) built for the current arch
- Only sm100 / sm103 (compute capability 100, 103)
- All three operands must be bfloat16 and contiguous
- The gate width must be a multiple of 256
- Only elementwise gating (headwise has no fused form)
"""
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.utils.hpc import has_hpc
logger = init_logger(__name__)
# hpc.gated_mla_gemm fuses `attn_out * sigmoid(x @ weight.T)` into one launch.
# It is built for sm100/sm103 and needs all three operands bf16 + contiguous,
# with the gate width a multiple of 256.
_HPC_GATED_GEMM_N_ALIGN = 256
_HPC_GATED_GEMM_CAPABILITIES: frozenset[int] = frozenset({100, 103})
def hpc_gated_mla_supported(
gating_type: str | None, attn_output_gate: torch.nn.Module | None
) -> bool:
"""Gate for running the MLA output gating through hpc.gated_mla_gemm."""
if not envs.VLLM_ENABLE_HPC_OPS:
return False
if attn_output_gate is None:
return False
if not has_hpc():
logger.warning_once(
"HPC gated MLA disabled: 'hpc' package is not installed. "
"Please install the HPC library (.so) to enable the fused kernel."
)
return False
from vllm.platforms import current_platform
if not current_platform.is_cuda():
logger.warning_once("HPC gated MLA disabled: only CUDA is supported.")
return False
capability = current_platform.get_device_capability()
if capability is None or capability.to_int() not in _HPC_GATED_GEMM_CAPABILITIES:
logger.warning_once(
"HPC gated MLA disabled: compute capability %s not in %s.",
capability,
_HPC_GATED_GEMM_CAPABILITIES,
)
return False
# Headwise gating broadcasts one scalar per head over v_head_dim; the
# kernel only implements the elementwise product.
if gating_type != "elementwise":
logger.warning_once(
"HPC gated MLA disabled: gating_type '%s' has no fused form.",
gating_type,
)
return False
weight = getattr(attn_output_gate, "weight", None)
if weight is None or weight.dtype != torch.bfloat16:
logger.warning_once(
"HPC gated MLA disabled: gate weight dtype is %s, the kernel needs "
"bfloat16 (keep the layer out of quantization).",
None if weight is None else weight.dtype,
)
return False
if weight.shape[0] % _HPC_GATED_GEMM_N_ALIGN != 0:
logger.warning_once(
"HPC gated MLA disabled: gate width %d is not a multiple of %d.",
weight.shape[0],
_HPC_GATED_GEMM_N_ALIGN,
)
return False
logger.info_once("HPC gated MLA enabled by set VLLM_ENABLE_HPC_OPS.")
return True
def hpc_gated_mla_gemm(
x: torch.Tensor,
weight: torch.Tensor,
atten_output: torch.Tensor,
) -> torch.Tensor:
"""Fused ``atten_output * sigmoid(x @ weight.T)``.
Args:
x: ``[m, k]`` bfloat16, contiguous.
weight: ``[n, k]`` bfloat16, contiguous and row-major (the kernel
transposes it). ``n`` must be a multiple of 256.
atten_output: ``[m, n]`` bfloat16, contiguous.
Returns:
``[m, n]`` bfloat16.
"""
from hpc.gemm import gated_mla_gemm
return gated_mla_gemm(x, weight, atten_output)
+261
View File
@@ -0,0 +1,261 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HPC fused iHC (independent Hyper-Connections) kernels for HY V4.
Replaces the eager HYV4HCPreLayer / HYV4HCPostLayer / HYV4HCHeadLayer bodies
with single-kernel HPC implementations:
pre : rms square sum + w projection + sigmoid gates + hc-dim weighted sum
post : y[n, i, :] = H_post[n, i] * x[n, :] + residual[n, i, :]
head : same as pre but hc_mult projection rows and no H_post output
The eager path issues 20 / 5 / 15 kernels for pre / post / head; each HPC op is
a single kernel.
Constraints:
- Requires VLLM_ENABLE_HPC_OPS=1
- Requires the hpc package (.so) built for the current arch
- Only sm100 / sm103 (compute capability 100, 103)
- Only hc_mult == 4 and hidden_size in {4096, 6144}
- x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint
keeps them out of fp8 quantization via modules_to_not_convert)
The fused modules own no parameters: they hold references to the weights of the
eager layer they replace, bound in ``process_weights_after_loading`` (invoked
generically for every ``HpcModule`` by the model loader).
NOTE: The reference implementation also offers a cross-layer post+pre fusion
(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre.
It requires reworking the decoder-layer forward scheduling and is not ported
here. TODO: add it once the decoder dataflow is restructured for it.
"""
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
from vllm.utils.hpc import has_hpc
logger = init_logger(__name__)
# Shapes the HPC kernels are instantiated for. Other shapes launch nothing,
# so gate on this rather than relying on a silent no-op.
_SUPPORTED_HIDDEN_SIZES: frozenset[int] = frozenset({4096, 6144})
_SUPPORTED_HC_MULTS: frozenset[int] = frozenset({4})
_SUPPORTED_CAPABILITIES: frozenset[int] = frozenset({100, 103})
def _ihc_supported(hc_mult: int, hidden_size: int) -> bool:
"""Shared gate for all three iHC ops."""
if not envs.VLLM_ENABLE_HPC_OPS:
return False
if not has_hpc():
logger.warning_once(
"HPC iHC disabled: 'hpc' package is not installed. "
"Please install the HPC library (.so) to enable fused kernels."
)
return False
from vllm.platforms import current_platform
if not current_platform.is_cuda():
logger.warning_once("HPC iHC disabled: only CUDA is supported.")
return False
capability = current_platform.get_device_capability()
if capability is None or capability.to_int() not in _SUPPORTED_CAPABILITIES:
logger.warning_once(
"HPC iHC disabled: compute capability %s not in %s.",
capability,
_SUPPORTED_CAPABILITIES,
)
return False
if hc_mult not in _SUPPORTED_HC_MULTS:
logger.warning_once(
"HPC iHC disabled: hc_mult=%d not in %s.", hc_mult, _SUPPORTED_HC_MULTS
)
return False
if hidden_size not in _SUPPORTED_HIDDEN_SIZES:
logger.warning_once(
"HPC iHC disabled: hidden_size=%d not in %s.",
hidden_size,
_SUPPORTED_HIDDEN_SIZES,
)
return False
if envs.VLLM_BATCH_INVARIANT:
# Batch-invariant mode relies on torch's deterministic reductions; the
# HPC kernel reduces in a different order, so keep the two regimes
# apart.
logger.warning_once(
"HPC iHC disabled: not supported under VLLM_BATCH_INVARIANT."
)
return False
logger.info_once("HPC iHC enabled by set VLLM_ENABLE_HPC_OPS.")
return True
class HpcIHCPre(HpcModule):
"""Fused iHC pre block.
Computes, in one kernel:
x_flat = x.flatten(1)
r = rsqrt(x_flat.square().mean(-1) + rms_norm_eps)
mixes = (x_flat @ w.T) * r
H_pre = sigmoid(mixes[:, :hc] * hc_scale[0] + hc_base[:hc]) + hc_eps
H_post = magnitude * sigmoid(
mixes[:, hc:] * hc_scale[1] + hc_base[hc:]) + hc_eps
y = sum_i H_pre[:, i] * x[:, i, :]
Args:
hc_mult: HC expand ratio.
hidden_size: Model hidden dimension.
magnitude: H_post multiplier (config.hc_magnitude).
hc_eps: Epsilon added to both gates (config.hc_eps).
norm_eps: Epsilon inside the rsqrt (config.rms_norm_eps).
fallback_op: The eager HYV4HCPreLayer to source weights from.
norm_owner: Optional RMSNorm that immediately follows this pre block.
When given, its weight/eps are folded into the kernel so a single
launch covers pre + RMSNorm (the caller then skips the separate
layernorm). None keeps the original pre-only behaviour.
"""
def __init__(
self,
hc_mult: int,
hidden_size: int,
magnitude: float,
hc_eps: float,
norm_eps: float,
fallback_op: torch.nn.Module,
norm_owner: torch.nn.Module | None = None,
) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
self.magnitude = magnitude
self.hc_eps = hc_eps
self.norm_eps = norm_eps
# Stash the owner outside nn.Module's attribute machinery: callers pass
# their own `self` here, and registering that as a submodule would make
# the module tree cyclic (state_dict() then hits RecursionError).
object.__setattr__(self, "_fallback_op", fallback_op)
# Same anti-cycle idiom for the fused RMSNorm owner (a sibling module).
object.__setattr__(self, "_norm_owner", norm_owner)
self.weight = None
self.hc_scale = None
self.hc_base = None
self.rms_weight = None
self.rms_eps = 0.0
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def process_weights_after_loading(self, model=None):
# hc_fn is a ReplicatedLinear whose weight is [2 * hc_mult, hc_mult * d].
# It stays float32 -- the HPC kernel consumes fp32 weights directly, so
# there is no conversion (and no precision loss) here.
owner = self._fallback_op
self.weight = owner.hc_fn.weight
self.hc_scale = owner.hc_scale
self.hc_base = owner.hc_base
if self._norm_owner is not None:
self.rms_weight = self._norm_owner.weight
self.rms_eps = self._norm_owner.variance_epsilon
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
import hpc
return hpc.fuse_ihc_pre(
x,
self.weight,
self.hc_scale,
self.hc_base,
self.norm_eps,
self.hc_eps,
self.magnitude,
self.rms_weight,
self.rms_eps,
)
class HpcIHCPost(HpcModule):
"""Fused iHC post block: H_post gating plus multi-channel residual add.
Unlike mHC there is no comb matrix, so each output channel only needs its
own residual channel and the whole thing is one fused multiply-add per
element.
"""
def __init__(self, hc_mult: int, hidden_size: int) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def forward(
self, x: torch.Tensor, residual: torch.Tensor, H_post: torch.Tensor
) -> torch.Tensor:
import hpc
return hpc.fuse_ihc_post(x, residual, H_post)
class HpcIHCHead(HpcModule):
"""Fused iHC head block: merge the hc channels back into one hidden state.
Same structure as HpcIHCPre but the projection emits only hc_mult gate
logits and there is no H_post output. Called once per forward, after the
last decoder layer.
"""
def __init__(
self,
hc_mult: int,
hidden_size: int,
hc_eps: float,
norm_eps: float,
fallback_op: torch.nn.Module,
) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
self.hc_eps = hc_eps
self.norm_eps = norm_eps
# See HpcIHCPre: keep the owner out of the module tree.
object.__setattr__(self, "_fallback_op", fallback_op)
self.weight = None
self.hc_scale = None
self.hc_base = None
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def process_weights_after_loading(self, model=None):
# hc_head_fn weight is [hc_mult, hc_mult * d], float32.
owner = self._fallback_op
self.weight = owner.hc_head_fn.weight
self.hc_scale = owner.hc_head_scale
self.hc_base = owner.hc_head_base
def forward(self, x: torch.Tensor) -> torch.Tensor:
import hpc
return hpc.fuse_ihc_head(
x,
self.weight,
self.hc_scale,
self.hc_base,
self.norm_eps,
self.hc_eps,
)
@@ -15,6 +15,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.vocab_parallel_embedding import (
UnquantizedEmbeddingMethod,
VocabParallelEmbedding,
@@ -144,7 +145,12 @@ class LogitsProcessor(PluggableLayer):
lm_head, hidden_states, bias=embedding_bias
)
if not isinstance(lm_head.quant_method, UnquantizedEmbeddingMethod):
# A quant config that excludes lm_head hands out UnquantizedLinearMethod
# rather than UnquantizedEmbeddingMethod, so accept both: either way the
# weight is plain and `lm_head.weight` can be cast directly.
if not isinstance(
lm_head.quant_method, (UnquantizedEmbeddingMethod, UnquantizedLinearMethod)
):
raise ValueError(
"A head_dtype different from the model dtype is only "
"supported for an unquantized lm_head."
+10 -2
View File
@@ -874,12 +874,20 @@ def _try_load_fp8_indexer_wk(
# We have both weight and scale: dequantize FP8 to BF16.
weight_fp8, scale_inv = entry["weight"], entry["scale"]
del buf[layer_prefix]
if scale_inv.dtype in (torch.uint8, torch.float8_e8m0fnu):
# MXFP8 checkpoints store power-of-two E8M0 scales, decode them to float.
scale_inv = scale_inv.view(torch.float8_e8m0fnu).to(torch.float32)
if scale_inv.ndim == 1:
# Per-channel scale: one scale per row of [out, in]
group_shape = GroupShape(1, weight_fp8.shape[1])
else:
block_size = weight_fp8.shape[1] // scale_inv.shape[1]
group_shape = GroupShape(block_size, block_size)
# Derive the block size independently per dim so that per-channel
# ([out, 1]), MX ([out, in / 32]) and 2D block ([out / 128, in / 128])
# scale layouts are all handled.
group_shape = GroupShape(
weight_fp8.shape[0] // scale_inv.shape[0],
weight_fp8.shape[1] // scale_inv.shape[1],
)
weight_bf16 = scaled_dequantize(
weight_fp8,
scale_inv,
+2
View File
@@ -128,6 +128,7 @@ _TEXT_GENERATION_MODELS = {
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
"HYV4ForCausalLM": ("vllm.models.hy_v4", "HYV4ForCausalLM"),
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
@@ -663,6 +664,7 @@ _SPECULATIVE_DECODING_MODELS = {
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
"InternS2MobiusMTP": ("interns2_mobius", "InternS2MobiusMTP"),
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
"HYV4MTPModel": ("vllm.models.hy_v4", "HYV4MTP"),
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
# Temporarily disabled.
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.