[Hy4] support Hy4-preview model (#54160)
Signed-off-by: chengvjiang <chengvjiang@tencent.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: russellfeng <russellfeng@tencent.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
chengvjiang
Jee Jee Li
Bugen Zhao
russellfeng
Jee Jee Li
parent
6c18a54648
commit
b2f685834a
@@ -79,12 +79,27 @@ class TrtLlmFp8ExpertsBase:
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
if not supported or moe_config.num_experts <= 2048:
|
||||
if not supported:
|
||||
return supported, reason
|
||||
return False, (
|
||||
"FlashInfer TRTLLM routing supports at most 2048 experts, "
|
||||
f"but got {moe_config.num_experts}"
|
||||
)
|
||||
# FlashInfer accepts gemm1_alpha/beta/clamp_limit only for MXFP8; the
|
||||
# DeepSeek-FP8 block-scale kernel rejects them and the per-tensor kernel
|
||||
# has no clamp at all, so a model-requested SwiGLU clamp cannot be
|
||||
# honored on those paths.
|
||||
if (
|
||||
moe_config.swiglu_limit is not None
|
||||
or moe_config.swiglu_alpha is not None
|
||||
or moe_config.swiglu_beta is not None
|
||||
) and (weight_key, activation_key) != (kMxfp8Static, kMxfp8Dynamic):
|
||||
return False, (
|
||||
"the TRTLLM FP8 kernels apply the SwiGLU alpha/beta/clamp "
|
||||
"parameters only for MXFP8 weights"
|
||||
)
|
||||
if moe_config.num_experts > 2048:
|
||||
return False, (
|
||||
"FlashInfer TRTLLM routing supports at most 2048 experts, "
|
||||
f"but got {moe_config.num_experts}"
|
||||
)
|
||||
return True, None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -1,11 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.hpc.gated_mla import (
|
||||
hpc_gated_mla_gemm,
|
||||
hpc_gated_mla_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.hpc.hpc_ihc import (
|
||||
HpcIHCHead,
|
||||
HpcIHCPost,
|
||||
HpcIHCPre,
|
||||
)
|
||||
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
|
||||
from vllm.model_executor.layers.hpc.rope_norm import HpcRopeNorm, QkNormPolicy
|
||||
|
||||
__all__ = [
|
||||
"HpcIHCHead",
|
||||
"HpcIHCPost",
|
||||
"HpcIHCPre",
|
||||
"HpcModule",
|
||||
"HpcRopeNorm",
|
||||
"QkNormPolicy",
|
||||
"hpc_gated_mla_gemm",
|
||||
"hpc_gated_mla_supported",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Enablement gate for the HPC fused gated-MLA GEMM.
|
||||
|
||||
``hpc.gated_mla_gemm`` fuses the MLA output gating -- the gate projection, the
|
||||
sigmoid and the elementwise product -- into a single kernel launch, replacing
|
||||
the eager ``attn_out * sigmoid(hidden_states @ weight.T)`` sequence.
|
||||
|
||||
Kernel constraints:
|
||||
- Requires VLLM_ENABLE_HPC_OPS=1
|
||||
- Requires the hpc package (.so) built for the current arch
|
||||
- Only sm100 / sm103 (compute capability 100, 103)
|
||||
- All three operands must be bfloat16 and contiguous
|
||||
- The gate width must be a multiple of 256
|
||||
- Only elementwise gating (headwise has no fused form)
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.hpc import has_hpc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# hpc.gated_mla_gemm fuses `attn_out * sigmoid(x @ weight.T)` into one launch.
|
||||
# It is built for sm100/sm103 and needs all three operands bf16 + contiguous,
|
||||
# with the gate width a multiple of 256.
|
||||
_HPC_GATED_GEMM_N_ALIGN = 256
|
||||
_HPC_GATED_GEMM_CAPABILITIES: frozenset[int] = frozenset({100, 103})
|
||||
|
||||
|
||||
def hpc_gated_mla_supported(
|
||||
gating_type: str | None, attn_output_gate: torch.nn.Module | None
|
||||
) -> bool:
|
||||
"""Gate for running the MLA output gating through hpc.gated_mla_gemm."""
|
||||
if not envs.VLLM_ENABLE_HPC_OPS:
|
||||
return False
|
||||
|
||||
if attn_output_gate is None:
|
||||
return False
|
||||
|
||||
if not has_hpc():
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: 'hpc' package is not installed. "
|
||||
"Please install the HPC library (.so) to enable the fused kernel."
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
logger.warning_once("HPC gated MLA disabled: only CUDA is supported.")
|
||||
return False
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.to_int() not in _HPC_GATED_GEMM_CAPABILITIES:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: compute capability %s not in %s.",
|
||||
capability,
|
||||
_HPC_GATED_GEMM_CAPABILITIES,
|
||||
)
|
||||
return False
|
||||
|
||||
# Headwise gating broadcasts one scalar per head over v_head_dim; the
|
||||
# kernel only implements the elementwise product.
|
||||
if gating_type != "elementwise":
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gating_type '%s' has no fused form.",
|
||||
gating_type,
|
||||
)
|
||||
return False
|
||||
|
||||
weight = getattr(attn_output_gate, "weight", None)
|
||||
if weight is None or weight.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gate weight dtype is %s, the kernel needs "
|
||||
"bfloat16 (keep the layer out of quantization).",
|
||||
None if weight is None else weight.dtype,
|
||||
)
|
||||
return False
|
||||
|
||||
if weight.shape[0] % _HPC_GATED_GEMM_N_ALIGN != 0:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gate width %d is not a multiple of %d.",
|
||||
weight.shape[0],
|
||||
_HPC_GATED_GEMM_N_ALIGN,
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info_once("HPC gated MLA enabled by set VLLM_ENABLE_HPC_OPS.")
|
||||
return True
|
||||
|
||||
|
||||
def hpc_gated_mla_gemm(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
atten_output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``atten_output * sigmoid(x @ weight.T)``.
|
||||
|
||||
Args:
|
||||
x: ``[m, k]`` bfloat16, contiguous.
|
||||
weight: ``[n, k]`` bfloat16, contiguous and row-major (the kernel
|
||||
transposes it). ``n`` must be a multiple of 256.
|
||||
atten_output: ``[m, n]`` bfloat16, contiguous.
|
||||
|
||||
Returns:
|
||||
``[m, n]`` bfloat16.
|
||||
"""
|
||||
from hpc.gemm import gated_mla_gemm
|
||||
|
||||
return gated_mla_gemm(x, weight, atten_output)
|
||||
@@ -0,0 +1,261 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HPC fused iHC (independent Hyper-Connections) kernels for HY V4.
|
||||
|
||||
Replaces the eager HYV4HCPreLayer / HYV4HCPostLayer / HYV4HCHeadLayer bodies
|
||||
with single-kernel HPC implementations:
|
||||
|
||||
pre : rms square sum + w projection + sigmoid gates + hc-dim weighted sum
|
||||
post : y[n, i, :] = H_post[n, i] * x[n, :] + residual[n, i, :]
|
||||
head : same as pre but hc_mult projection rows and no H_post output
|
||||
|
||||
The eager path issues 20 / 5 / 15 kernels for pre / post / head; each HPC op is
|
||||
a single kernel.
|
||||
|
||||
Constraints:
|
||||
- Requires VLLM_ENABLE_HPC_OPS=1
|
||||
- Requires the hpc package (.so) built for the current arch
|
||||
- Only sm100 / sm103 (compute capability 100, 103)
|
||||
- Only hc_mult == 4 and hidden_size in {4096, 6144}
|
||||
- x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint
|
||||
keeps them out of fp8 quantization via modules_to_not_convert)
|
||||
|
||||
The fused modules own no parameters: they hold references to the weights of the
|
||||
eager layer they replace, bound in ``process_weights_after_loading`` (invoked
|
||||
generically for every ``HpcModule`` by the model loader).
|
||||
|
||||
NOTE: The reference implementation also offers a cross-layer post+pre fusion
|
||||
(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre.
|
||||
It requires reworking the decoder-layer forward scheduling and is not ported
|
||||
here. TODO: add it once the decoder dataflow is restructured for it.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
|
||||
from vllm.utils.hpc import has_hpc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Shapes the HPC kernels are instantiated for. Other shapes launch nothing,
|
||||
# so gate on this rather than relying on a silent no-op.
|
||||
_SUPPORTED_HIDDEN_SIZES: frozenset[int] = frozenset({4096, 6144})
|
||||
_SUPPORTED_HC_MULTS: frozenset[int] = frozenset({4})
|
||||
_SUPPORTED_CAPABILITIES: frozenset[int] = frozenset({100, 103})
|
||||
|
||||
|
||||
def _ihc_supported(hc_mult: int, hidden_size: int) -> bool:
|
||||
"""Shared gate for all three iHC ops."""
|
||||
if not envs.VLLM_ENABLE_HPC_OPS:
|
||||
return False
|
||||
|
||||
if not has_hpc():
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: 'hpc' package is not installed. "
|
||||
"Please install the HPC library (.so) to enable fused kernels."
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
logger.warning_once("HPC iHC disabled: only CUDA is supported.")
|
||||
return False
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.to_int() not in _SUPPORTED_CAPABILITIES:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: compute capability %s not in %s.",
|
||||
capability,
|
||||
_SUPPORTED_CAPABILITIES,
|
||||
)
|
||||
return False
|
||||
|
||||
if hc_mult not in _SUPPORTED_HC_MULTS:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: hc_mult=%d not in %s.", hc_mult, _SUPPORTED_HC_MULTS
|
||||
)
|
||||
return False
|
||||
|
||||
if hidden_size not in _SUPPORTED_HIDDEN_SIZES:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: hidden_size=%d not in %s.",
|
||||
hidden_size,
|
||||
_SUPPORTED_HIDDEN_SIZES,
|
||||
)
|
||||
return False
|
||||
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
# Batch-invariant mode relies on torch's deterministic reductions; the
|
||||
# HPC kernel reduces in a different order, so keep the two regimes
|
||||
# apart.
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: not supported under VLLM_BATCH_INVARIANT."
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info_once("HPC iHC enabled by set VLLM_ENABLE_HPC_OPS.")
|
||||
return True
|
||||
|
||||
|
||||
class HpcIHCPre(HpcModule):
|
||||
"""Fused iHC pre block.
|
||||
|
||||
Computes, in one kernel:
|
||||
x_flat = x.flatten(1)
|
||||
r = rsqrt(x_flat.square().mean(-1) + rms_norm_eps)
|
||||
mixes = (x_flat @ w.T) * r
|
||||
H_pre = sigmoid(mixes[:, :hc] * hc_scale[0] + hc_base[:hc]) + hc_eps
|
||||
H_post = magnitude * sigmoid(
|
||||
mixes[:, hc:] * hc_scale[1] + hc_base[hc:]) + hc_eps
|
||||
y = sum_i H_pre[:, i] * x[:, i, :]
|
||||
|
||||
Args:
|
||||
hc_mult: HC expand ratio.
|
||||
hidden_size: Model hidden dimension.
|
||||
magnitude: H_post multiplier (config.hc_magnitude).
|
||||
hc_eps: Epsilon added to both gates (config.hc_eps).
|
||||
norm_eps: Epsilon inside the rsqrt (config.rms_norm_eps).
|
||||
fallback_op: The eager HYV4HCPreLayer to source weights from.
|
||||
norm_owner: Optional RMSNorm that immediately follows this pre block.
|
||||
When given, its weight/eps are folded into the kernel so a single
|
||||
launch covers pre + RMSNorm (the caller then skips the separate
|
||||
layernorm). None keeps the original pre-only behaviour.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
magnitude: float,
|
||||
hc_eps: float,
|
||||
norm_eps: float,
|
||||
fallback_op: torch.nn.Module,
|
||||
norm_owner: torch.nn.Module | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
self.magnitude = magnitude
|
||||
self.hc_eps = hc_eps
|
||||
self.norm_eps = norm_eps
|
||||
# Stash the owner outside nn.Module's attribute machinery: callers pass
|
||||
# their own `self` here, and registering that as a submodule would make
|
||||
# the module tree cyclic (state_dict() then hits RecursionError).
|
||||
object.__setattr__(self, "_fallback_op", fallback_op)
|
||||
# Same anti-cycle idiom for the fused RMSNorm owner (a sibling module).
|
||||
object.__setattr__(self, "_norm_owner", norm_owner)
|
||||
self.weight = None
|
||||
self.hc_scale = None
|
||||
self.hc_base = None
|
||||
self.rms_weight = None
|
||||
self.rms_eps = 0.0
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def process_weights_after_loading(self, model=None):
|
||||
# hc_fn is a ReplicatedLinear whose weight is [2 * hc_mult, hc_mult * d].
|
||||
# It stays float32 -- the HPC kernel consumes fp32 weights directly, so
|
||||
# there is no conversion (and no precision loss) here.
|
||||
owner = self._fallback_op
|
||||
self.weight = owner.hc_fn.weight
|
||||
self.hc_scale = owner.hc_scale
|
||||
self.hc_base = owner.hc_base
|
||||
if self._norm_owner is not None:
|
||||
self.rms_weight = self._norm_owner.weight
|
||||
self.rms_eps = self._norm_owner.variance_epsilon
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import hpc
|
||||
|
||||
return hpc.fuse_ihc_pre(
|
||||
x,
|
||||
self.weight,
|
||||
self.hc_scale,
|
||||
self.hc_base,
|
||||
self.norm_eps,
|
||||
self.hc_eps,
|
||||
self.magnitude,
|
||||
self.rms_weight,
|
||||
self.rms_eps,
|
||||
)
|
||||
|
||||
|
||||
class HpcIHCPost(HpcModule):
|
||||
"""Fused iHC post block: H_post gating plus multi-channel residual add.
|
||||
|
||||
Unlike mHC there is no comb matrix, so each output channel only needs its
|
||||
own residual channel and the whole thing is one fused multiply-add per
|
||||
element.
|
||||
"""
|
||||
|
||||
def __init__(self, hc_mult: int, hidden_size: int) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual: torch.Tensor, H_post: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
import hpc
|
||||
|
||||
return hpc.fuse_ihc_post(x, residual, H_post)
|
||||
|
||||
|
||||
class HpcIHCHead(HpcModule):
|
||||
"""Fused iHC head block: merge the hc channels back into one hidden state.
|
||||
|
||||
Same structure as HpcIHCPre but the projection emits only hc_mult gate
|
||||
logits and there is no H_post output. Called once per forward, after the
|
||||
last decoder layer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
hc_eps: float,
|
||||
norm_eps: float,
|
||||
fallback_op: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
self.hc_eps = hc_eps
|
||||
self.norm_eps = norm_eps
|
||||
# See HpcIHCPre: keep the owner out of the module tree.
|
||||
object.__setattr__(self, "_fallback_op", fallback_op)
|
||||
self.weight = None
|
||||
self.hc_scale = None
|
||||
self.hc_base = None
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def process_weights_after_loading(self, model=None):
|
||||
# hc_head_fn weight is [hc_mult, hc_mult * d], float32.
|
||||
owner = self._fallback_op
|
||||
self.weight = owner.hc_head_fn.weight
|
||||
self.hc_scale = owner.hc_head_scale
|
||||
self.hc_base = owner.hc_head_base
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
import hpc
|
||||
|
||||
return hpc.fuse_ihc_head(
|
||||
x,
|
||||
self.weight,
|
||||
self.hc_scale,
|
||||
self.hc_base,
|
||||
self.norm_eps,
|
||||
self.hc_eps,
|
||||
)
|
||||
@@ -15,6 +15,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
@@ -144,7 +145,12 @@ class LogitsProcessor(PluggableLayer):
|
||||
lm_head, hidden_states, bias=embedding_bias
|
||||
)
|
||||
|
||||
if not isinstance(lm_head.quant_method, UnquantizedEmbeddingMethod):
|
||||
# A quant config that excludes lm_head hands out UnquantizedLinearMethod
|
||||
# rather than UnquantizedEmbeddingMethod, so accept both: either way the
|
||||
# weight is plain and `lm_head.weight` can be cast directly.
|
||||
if not isinstance(
|
||||
lm_head.quant_method, (UnquantizedEmbeddingMethod, UnquantizedLinearMethod)
|
||||
):
|
||||
raise ValueError(
|
||||
"A head_dtype different from the model dtype is only "
|
||||
"supported for an unquantized lm_head."
|
||||
|
||||
@@ -874,12 +874,20 @@ def _try_load_fp8_indexer_wk(
|
||||
# We have both weight and scale: dequantize FP8 to BF16.
|
||||
weight_fp8, scale_inv = entry["weight"], entry["scale"]
|
||||
del buf[layer_prefix]
|
||||
if scale_inv.dtype in (torch.uint8, torch.float8_e8m0fnu):
|
||||
# MXFP8 checkpoints store power-of-two E8M0 scales, decode them to float.
|
||||
scale_inv = scale_inv.view(torch.float8_e8m0fnu).to(torch.float32)
|
||||
if scale_inv.ndim == 1:
|
||||
# Per-channel scale: one scale per row of [out, in]
|
||||
group_shape = GroupShape(1, weight_fp8.shape[1])
|
||||
else:
|
||||
block_size = weight_fp8.shape[1] // scale_inv.shape[1]
|
||||
group_shape = GroupShape(block_size, block_size)
|
||||
# Derive the block size independently per dim so that per-channel
|
||||
# ([out, 1]), MX ([out, in / 32]) and 2D block ([out / 128, in / 128])
|
||||
# scale layouts are all handled.
|
||||
group_shape = GroupShape(
|
||||
weight_fp8.shape[0] // scale_inv.shape[0],
|
||||
weight_fp8.shape[1] // scale_inv.shape[1],
|
||||
)
|
||||
weight_bf16 = scaled_dequantize(
|
||||
weight_fp8,
|
||||
scale_inv,
|
||||
|
||||
@@ -128,6 +128,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
|
||||
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
|
||||
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
|
||||
"HYV4ForCausalLM": ("vllm.models.hy_v4", "HYV4ForCausalLM"),
|
||||
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
|
||||
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
|
||||
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
|
||||
@@ -663,6 +664,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
"InternS2MobiusMTP": ("interns2_mobius", "InternS2MobiusMTP"),
|
||||
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
|
||||
"HYV4MTPModel": ("vllm.models.hy_v4", "HYV4MTP"),
|
||||
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
|
||||
# Temporarily disabled.
|
||||
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
|
||||
|
||||
Reference in New Issue
Block a user