Files
MOVZXandHinata e61dcb9143 [Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
2026-10-18 19:19:56 +07:00

230 lines
8.2 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HyperConnection (Gated Residual) utilities — NVIDIA model variant.
Implements the HyperConnection residual scheme proposed in
"HyperConnections" (https://arxiv.org/abs/2409.19606). This NVIDIA variant
delays each HC combine to the following HC mix boundary. HC glue kernels,
including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain
standard vLLM Linear modules.
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
(HC outer, HS inner — checkpoint-native layout).
Typical usage inside a transformer decoder layer::
self.attn_hc = GatedResidual(hc_config)
hidden_states, block_input, injection = self.attn_hc.mix(hidden_states)
attention_output = attention(block_input)
hidden_states, block_input, injection = self.mlp_hc.combine_and_mix(
hidden_states, attention_output, injection
)
"""
import torch
from torch import nn
import vllm.envs as envs
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
ReplicatedLinear,
)
from vllm.model_executor.models.utils import maybe_prefix
from vllm.platforms import current_platform
from ..common.hyperconnection import (
GroupedGemmaRMSNorm,
HyperConnectionConfig,
)
from .ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
hc_silu,
)
# ---------------------------------------------------------------------------
# Gated-residual variant
# ---------------------------------------------------------------------------
class GatedResidual(nn.Module):
"""Gated HyperConnection with learnable low-rank mixing and injection.
``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged
low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean
over the HC streams). When passed a pending block output, it fuses its
residual combine with the RMSNorm. A missing injection selects unit-weight
combine. Final mixers use ``use_combine=False`` and do not produce a new
injection.
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
are vLLM Linear modules (merged replicated linear for down+inject).
Eligible down+inject projections use the fused SiLU GEMM; other projections
use their Linear module's GEMM dispatch.
"""
def __init__(
self,
config: HyperConnectionConfig,
use_combine: bool = True,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
self.lora_rank = config.hc_lowrank
self.hc_count = config.hc_count
self.hidden_size = config.hidden_size
self.use_combine = use_combine
norm_size = (
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
)
group_size = config.hidden_size if config.hc_per_branch_norm else None
# Normalize each H-sized HC stream independently while retaining a
# separate affine weight for every element of the HC*H layout.
self.hc_norm = GroupedGemmaRMSNorm(
norm_size,
eps=config.rms_norm_eps,
group_size=group_size,
dtype=config.params_dtype,
)
# -- vLLM Linear weights --------------------------------------------
# The merged skinny-GEMM shape is physically padded to 16 rows to ensure
# good alignment and performant implementation chosen by CuBLAS heuristics.
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
if use_combine:
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
self.hyper_hidden_size,
[self.lora_rank, self.hc_count]
+ ([self.pad_size] if self.pad_size else []),
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"),
return_bias=False,
disable_tp=True,
)
weight = self.input_mix_weight_down_block_inject.weight
self._use_hc_down_silu = False
self._hc_down_silu_max_m = 0
if (
weight.shape[1] % 8 == 0
and weight.dtype == torch.bfloat16
and current_platform.has_device_capability(90)
):
from .ops.cute_dsl.hc_down_silu import MAX_FUSED_M
self._hc_down_silu_max_m = MAX_FUSED_M
self._use_hc_down_silu = True
else:
self.input_mix_weight_down = ReplicatedLinear(
self.hyper_hidden_size,
self.lora_rank,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_down"),
return_bias=False,
)
self.input_mix_weight_up = ReplicatedLinear(
self.lora_rank,
self.hyper_hidden_size,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_up"),
return_bias=False,
)
def _down_and_inject(
self, xn: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Down projection + SiLU; also returns injection logits if combined."""
if not self.use_combine:
return hc_silu(self.input_mix_weight_down(xn), self.hc_count), None
use_fused = (
self._use_hc_down_silu
and not envs.VLLM_BATCH_INVARIANT
and 1 <= xn.shape[0] <= self._hc_down_silu_max_m
)
if use_fused:
from .ops.cute_dsl.hc_down_silu import hc_down_silu
return hc_down_silu(
xn,
self.input_mix_weight_down_block_inject.weight,
self.lora_rank,
self.hc_count,
)
down_and_injection = self.input_mix_weight_down_block_inject(xn)
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
return hc_silu(lora, self.hc_count), injection
def mix(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
xn = grouped_gemma_rmsnorm(
hidden_states,
self.hc_norm.weight,
self.config.rms_norm_eps,
self.hc_count,
)
lora, injection = self._down_and_inject(xn)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
def combine_and_mix(
self,
hidden_states: torch.Tensor,
prev_block_output: torch.Tensor,
prev_injection: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""Consume a pending combine, then prepare the next block input.
``hidden_states`` is the multi-stream state from before the pending
block's mix. Its combine with ``block_output`` is fused with this
module's input RMSNorm. A missing injection applies the block output
to every stream with unit weight.
"""
hidden_states, xn = hc_combine_norm(
hidden_states,
prev_block_output,
prev_injection,
self.hc_norm.weight,
self.config.rms_norm_eps,
self.hc_count,
)
lora, injection = self._down_and_inject(xn)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
def combine(
self,
hidden_states: torch.Tensor,
block_output: torch.Tensor,
injection: torch.Tensor | None,
) -> torch.Tensor:
return hc_combine(hidden_states, block_output, injection, self.hc_count)
@property
def hyper_hidden_size(self) -> int:
return self.hc_count * self.hidden_size
__all__ = [
"GatedResidual",
"GroupedGemmaRMSNorm",
"HyperConnectionConfig",
]