[Model][Quantization] Add Ling-3.0-flash-fp8 support (#51265)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3b4c86e489
commit
ba1cdcfcf0
@@ -701,11 +701,32 @@ def test_non_fused_layer_unaffected():
|
|||||||
|
|
||||||
|
|
||||||
def test_substr_match_on_fused_name():
|
def test_substr_match_on_fused_name():
|
||||||
# skip_with_substr=True path: fused-name substring match should also
|
# Substring matching: a fused-name match should also
|
||||||
# short-circuit before shard expansion.
|
# short-circuit before shard expansion.
|
||||||
assert is_layer_skipped(
|
assert is_layer_skipped(
|
||||||
prefix="model.layers.0.self_attn.qkv_proj",
|
prefix="model.layers.0.self_attn.qkv_proj",
|
||||||
ignored_layers=["self_attn.qkv_proj"],
|
ignored_layers=["self_attn.qkv_proj"],
|
||||||
fused_mapping=FUSED_MAPPING,
|
fused_mapping=FUSED_MAPPING,
|
||||||
skip_with_substr=True,
|
match_mode="substring",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("prefix", "ignored_layer", "expected"),
|
||||||
|
[
|
||||||
|
("model.layers.0.self_attn.b_proj", "b_proj", True),
|
||||||
|
("model.layers.0.self_attn.q_b_proj", "b_proj", False),
|
||||||
|
("model.layers.0.self_attn.kv_b_proj", "b_proj", False),
|
||||||
|
("model.layers.5.self_attn.g_proj", "5.self_attn.g_proj", True),
|
||||||
|
("model.layers.6.self_attn.g_proj", "5.self_attn.g_proj", False),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_suffix_match_at_module_boundary(prefix, ignored_layer, expected):
|
||||||
|
assert (
|
||||||
|
is_layer_skipped(
|
||||||
|
prefix=prefix,
|
||||||
|
ignored_layers=[ignored_layer],
|
||||||
|
match_mode="suffix",
|
||||||
|
)
|
||||||
|
is expected
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -292,7 +292,7 @@ class AutoAWQConfig(QuantizationConfig):
|
|||||||
prefix,
|
prefix,
|
||||||
self.modules_to_not_convert,
|
self.modules_to_not_convert,
|
||||||
self.packed_modules_mapping,
|
self.packed_modules_mapping,
|
||||||
skip_with_substr=True,
|
match_mode="substring",
|
||||||
):
|
):
|
||||||
return UnquantizedLinearMethod()
|
return UnquantizedLinearMethod()
|
||||||
|
|
||||||
@@ -335,7 +335,7 @@ class AutoAWQConfig(QuantizationConfig):
|
|||||||
if is_layer_skipped(
|
if is_layer_skipped(
|
||||||
prefix,
|
prefix,
|
||||||
getattr(self, "modules_to_not_convert", []),
|
getattr(self, "modules_to_not_convert", []),
|
||||||
skip_with_substr=True,
|
match_mode="substring",
|
||||||
):
|
):
|
||||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any, Literal
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -108,6 +108,9 @@ class Fp8Config(QuantizationConfig):
|
|||||||
raise ValueError(f"Unsupported activation scheme {activation_scheme}")
|
raise ValueError(f"Unsupported activation scheme {activation_scheme}")
|
||||||
self.activation_scheme = activation_scheme
|
self.activation_scheme = activation_scheme
|
||||||
self.ignored_layers = ignored_layers or []
|
self.ignored_layers = ignored_layers or []
|
||||||
|
self.ignored_layers_match_mode: Literal["exact", "substring", "suffix"] = (
|
||||||
|
"exact"
|
||||||
|
)
|
||||||
self.store_dtype = store_dtype
|
self.store_dtype = store_dtype
|
||||||
if weight_block_size is not None:
|
if weight_block_size is not None:
|
||||||
if not is_checkpoint_fp8_serialized:
|
if not is_checkpoint_fp8_serialized:
|
||||||
@@ -177,6 +180,7 @@ class Fp8Config(QuantizationConfig):
|
|||||||
prefix=prefix,
|
prefix=prefix,
|
||||||
ignored_layers=self.ignored_layers,
|
ignored_layers=self.ignored_layers,
|
||||||
fused_mapping=self.packed_modules_mapping,
|
fused_mapping=self.packed_modules_mapping,
|
||||||
|
match_mode=self.ignored_layers_match_mode,
|
||||||
):
|
):
|
||||||
return UnquantizedLinearMethod()
|
return UnquantizedLinearMethod()
|
||||||
if not self.is_checkpoint_fp8_serialized:
|
if not self.is_checkpoint_fp8_serialized:
|
||||||
@@ -196,6 +200,7 @@ class Fp8Config(QuantizationConfig):
|
|||||||
prefix=prefix,
|
prefix=prefix,
|
||||||
ignored_layers=self.ignored_layers,
|
ignored_layers=self.ignored_layers,
|
||||||
fused_mapping=self.packed_modules_mapping,
|
fused_mapping=self.packed_modules_mapping,
|
||||||
|
match_mode=self.ignored_layers_match_mode,
|
||||||
):
|
):
|
||||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||||
if self.store_dtype == "mxfp4":
|
if self.store_dtype == "mxfp4":
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
from collections.abc import Callable, Mapping
|
from collections.abc import Callable, Mapping
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from types import MappingProxyType
|
from types import MappingProxyType
|
||||||
from typing import TYPE_CHECKING, ClassVar, NamedTuple
|
from typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple
|
||||||
|
|
||||||
import numpy
|
import numpy
|
||||||
import torch
|
import torch
|
||||||
@@ -604,7 +604,7 @@ def is_layer_skipped(
|
|||||||
ignored_layers: list[str],
|
ignored_layers: list[str],
|
||||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||||
*,
|
*,
|
||||||
skip_with_substr: bool = False,
|
match_mode: Literal["exact", "substring", "suffix"] = "exact",
|
||||||
) -> bool:
|
) -> bool:
|
||||||
def prefix_full_match(prefix: str, ignored_layers: list[str]) -> bool:
|
def prefix_full_match(prefix: str, ignored_layers: list[str]) -> bool:
|
||||||
return prefix in ignored_layers
|
return prefix in ignored_layers
|
||||||
@@ -613,7 +613,22 @@ def is_layer_skipped(
|
|||||||
def substr_match(prefix: str, ignored_layers: list[str]) -> bool:
|
def substr_match(prefix: str, ignored_layers: list[str]) -> bool:
|
||||||
return any(layer in prefix for layer in ignored_layers)
|
return any(layer in prefix for layer in ignored_layers)
|
||||||
|
|
||||||
match_func = substr_match if skip_with_substr else prefix_full_match
|
# Match abbreviated module paths at a component boundary. For example,
|
||||||
|
# ``b_proj`` matches ``model.layers.0.self_attn.b_proj`` but not
|
||||||
|
# ``model.layers.0.self_attn.q_b_proj``.
|
||||||
|
def suffix_match(prefix: str, ignored_layers: list[str]) -> bool:
|
||||||
|
return any(
|
||||||
|
prefix == layer or prefix.endswith(f".{layer}") for layer in ignored_layers
|
||||||
|
)
|
||||||
|
|
||||||
|
if match_mode == "exact":
|
||||||
|
match_func = prefix_full_match
|
||||||
|
elif match_mode == "substring":
|
||||||
|
match_func = substr_match
|
||||||
|
elif match_mode == "suffix":
|
||||||
|
match_func = suffix_match
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unsupported layer skip match mode: {match_mode}")
|
||||||
|
|
||||||
# prefix: model.layers.0.self_attn.q_proj
|
# prefix: model.layers.0.self_attn.q_proj
|
||||||
# proj_name: q_proj
|
# proj_name: q_proj
|
||||||
@@ -649,14 +664,11 @@ def is_layer_skipped(
|
|||||||
"are quantized. All shards of fused layers "
|
"are quantized. All shards of fused layers "
|
||||||
"to have the same precision."
|
"to have the same precision."
|
||||||
)
|
)
|
||||||
elif "experts" in prefix and not skip_with_substr:
|
elif "experts" in prefix and match_mode == "exact":
|
||||||
expert_ignore_layers = filter(
|
expert_ignore_layers = filter(
|
||||||
lambda layer_name: "experts" in layer_name, ignored_layers
|
lambda layer_name: "experts" in layer_name, ignored_layers
|
||||||
)
|
)
|
||||||
return any(
|
return any(prefix in layer_name for layer_name in expert_ignore_layers)
|
||||||
prefix in layer_name if not skip_with_substr else layer_name in prefix
|
|
||||||
for layer_name in expert_ignore_layers
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
is_skipped = match_func(prefix, ignored_layers)
|
is_skipped = match_func(prefix, ignored_layers)
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ vLLM's parallel linear layers, MLA kernel, KDA kernel and fused MoE loader.
|
|||||||
|
|
||||||
import copy
|
import copy
|
||||||
from collections.abc import Callable, Iterable
|
from collections.abc import Callable, Iterable
|
||||||
|
from math import lcm
|
||||||
|
from typing import TypeGuard
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -55,6 +57,8 @@ from vllm.model_executor.layers.mla import (
|
|||||||
MultiHeadLatentAttentionWrapper,
|
MultiHeadLatentAttentionWrapper,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||||
|
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||||
|
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
|
||||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
ParallelLMHead,
|
ParallelLMHead,
|
||||||
@@ -79,7 +83,13 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
|||||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||||
|
|
||||||
from .interfaces import HasInnerState, IsHybrid, SupportsPP
|
from .interfaces import HasInnerState, IsHybrid, SupportsPP
|
||||||
from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers, maybe_prefix
|
from .utils import (
|
||||||
|
PPMissingLayer,
|
||||||
|
WeightsMapper,
|
||||||
|
is_pp_missing_parameter,
|
||||||
|
make_layers,
|
||||||
|
maybe_prefix,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def bailing_v3_kda_attention(
|
def bailing_v3_kda_attention(
|
||||||
@@ -193,6 +203,155 @@ def _load_a_log(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
|
|||||||
sharded_weight_loader(2)(param, loaded_weight)
|
sharded_weight_loader(2)(param, loaded_weight)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_block_fp8_config(
|
||||||
|
quant_config: QuantizationConfig | None,
|
||||||
|
) -> TypeGuard[Fp8Config]:
|
||||||
|
return (
|
||||||
|
isinstance(quant_config, Fp8Config)
|
||||||
|
and quant_config.weight_block_size is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configure_ling_fp8_quant_config(
|
||||||
|
quant_config: QuantizationConfig | None,
|
||||||
|
) -> None:
|
||||||
|
if _is_block_fp8_config(quant_config):
|
||||||
|
quant_config.ignored_layers_match_mode = "suffix"
|
||||||
|
|
||||||
|
|
||||||
|
def _is_fp8_module_excluded(
|
||||||
|
quant_config: QuantizationConfig | None,
|
||||||
|
prefix: str,
|
||||||
|
) -> bool:
|
||||||
|
"""Match Ling's abbreviated FP8 exclusions against mapped vLLM prefixes."""
|
||||||
|
if not _is_block_fp8_config(quant_config):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return is_layer_skipped(
|
||||||
|
prefix=prefix,
|
||||||
|
ignored_layers=quant_config.ignored_layers,
|
||||||
|
fused_mapping=quant_config.packed_modules_mapping,
|
||||||
|
match_mode=quant_config.ignored_layers_match_mode,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_block_fp8_mlp_padded_intermediate_size(
|
||||||
|
quant_config: QuantizationConfig | None,
|
||||||
|
intermediate_size: int,
|
||||||
|
prefix: str,
|
||||||
|
) -> int:
|
||||||
|
"""Pad a block-FP8 MLP so each TP shard contains whole quant blocks."""
|
||||||
|
if not _is_block_fp8_config(quant_config):
|
||||||
|
return intermediate_size
|
||||||
|
block_size = quant_config.weight_block_size
|
||||||
|
assert block_size is not None
|
||||||
|
|
||||||
|
alignments: list[int] = []
|
||||||
|
if not _is_fp8_module_excluded(quant_config, f"{prefix}.gate_up_proj"):
|
||||||
|
alignments.append(int(block_size[0]))
|
||||||
|
if not _is_fp8_module_excluded(quant_config, f"{prefix}.down_proj"):
|
||||||
|
alignments.append(int(block_size[1]))
|
||||||
|
if not alignments:
|
||||||
|
return intermediate_size
|
||||||
|
|
||||||
|
tp_size = get_tensor_model_parallel_world_size()
|
||||||
|
tp_alignment = tp_size * lcm(*alignments)
|
||||||
|
return (intermediate_size + tp_alignment - 1) // tp_alignment * tp_alignment
|
||||||
|
|
||||||
|
|
||||||
|
def _pad_block_fp8_mlp_checkpoint_tensor(
|
||||||
|
quant_config: QuantizationConfig,
|
||||||
|
name: str,
|
||||||
|
loaded_weight: torch.Tensor,
|
||||||
|
intermediate_size: int,
|
||||||
|
padded_intermediate_size: int,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""Zero-pad an MLP checkpoint tensor on its intermediate dimension."""
|
||||||
|
if padded_intermediate_size == intermediate_size:
|
||||||
|
return loaded_weight
|
||||||
|
|
||||||
|
block_size = getattr(quant_config, "weight_block_size", None)
|
||||||
|
assert block_size is not None
|
||||||
|
|
||||||
|
if ".down_proj." in name:
|
||||||
|
if name.endswith(".bias"):
|
||||||
|
return loaded_weight
|
||||||
|
dim = 1
|
||||||
|
block = int(block_size[1])
|
||||||
|
logical_shards = 1
|
||||||
|
elif any(
|
||||||
|
projection in name
|
||||||
|
for projection in (".gate_proj.", ".up_proj.", ".gate_up_proj.")
|
||||||
|
):
|
||||||
|
dim = 0
|
||||||
|
block = int(block_size[0])
|
||||||
|
logical_shards = 2 if ".gate_up_proj." in name else 1
|
||||||
|
else:
|
||||||
|
return loaded_weight
|
||||||
|
|
||||||
|
if name.endswith(".weight_scale_inv"):
|
||||||
|
expected_shard_size = (intermediate_size + block - 1) // block
|
||||||
|
target_shard_size = (padded_intermediate_size + block - 1) // block
|
||||||
|
elif name.endswith((".weight", ".bias")):
|
||||||
|
expected_shard_size = intermediate_size
|
||||||
|
target_shard_size = padded_intermediate_size
|
||||||
|
else:
|
||||||
|
return loaded_weight
|
||||||
|
|
||||||
|
current_size = loaded_weight.shape[dim]
|
||||||
|
if current_size % logical_shards != 0:
|
||||||
|
raise ValueError(
|
||||||
|
f"Cannot split {name} dimension {current_size} into "
|
||||||
|
f"{logical_shards} logical shards."
|
||||||
|
)
|
||||||
|
current_shard_size = current_size // logical_shards
|
||||||
|
if current_shard_size == target_shard_size:
|
||||||
|
return loaded_weight
|
||||||
|
if current_shard_size != expected_shard_size:
|
||||||
|
raise ValueError(
|
||||||
|
f"Cannot pad {name}: expected each logical intermediate dimension "
|
||||||
|
f"to be {expected_shard_size}, but got {current_shard_size}."
|
||||||
|
)
|
||||||
|
|
||||||
|
padded_shards: list[torch.Tensor] = []
|
||||||
|
for shard in loaded_weight.split(current_shard_size, dim=dim):
|
||||||
|
pad_shape = list(shard.shape)
|
||||||
|
pad_shape[dim] = target_shard_size - current_shard_size
|
||||||
|
padded_shards.extend([shard, shard.new_zeros(pad_shape)])
|
||||||
|
return torch.cat(padded_shards, dim=dim)
|
||||||
|
|
||||||
|
|
||||||
|
def _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
|
||||||
|
quant_config: QuantizationConfig | None,
|
||||||
|
config: PretrainedConfig,
|
||||||
|
name: str,
|
||||||
|
loaded_weight: torch.Tensor,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
if ".mlp.shared_experts." not in name:
|
||||||
|
return loaded_weight
|
||||||
|
|
||||||
|
shared_prefix = name.split(".shared_experts.", 1)[0] + ".shared_experts"
|
||||||
|
shared_intermediate = (
|
||||||
|
config.moe_shared_expert_intermediate_size * config.num_shared_experts
|
||||||
|
)
|
||||||
|
padded_shared_intermediate = _get_block_fp8_mlp_padded_intermediate_size(
|
||||||
|
quant_config,
|
||||||
|
shared_intermediate,
|
||||||
|
shared_prefix,
|
||||||
|
)
|
||||||
|
if padded_shared_intermediate == shared_intermediate:
|
||||||
|
return loaded_weight
|
||||||
|
|
||||||
|
assert quant_config is not None
|
||||||
|
return _pad_block_fp8_mlp_checkpoint_tensor(
|
||||||
|
quant_config,
|
||||||
|
name,
|
||||||
|
loaded_weight,
|
||||||
|
shared_intermediate,
|
||||||
|
padded_shared_intermediate,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class _IdentityProjection(nn.Module):
|
class _IdentityProjection(nn.Module):
|
||||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]:
|
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]:
|
||||||
return x, None
|
return x, None
|
||||||
@@ -446,13 +605,36 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase):
|
|||||||
|
|
||||||
projection_size = self.head_dim * self.num_heads
|
projection_size = self.head_dim * self.num_heads
|
||||||
self.projection_size_per_partition = projection_size // self.tp_size
|
self.projection_size_per_partition = projection_size // self.tp_size
|
||||||
self.qkvb_proj = MergedColumnParallelLinear(
|
self.separate_b_proj = _is_fp8_module_excluded(quant_config, f"{prefix}.b_proj")
|
||||||
self.hidden_size,
|
self.qkv_proj: MergedColumnParallelLinear | None
|
||||||
[projection_size, projection_size, projection_size, self.num_heads],
|
self.b_proj: ColumnParallelLinear | None
|
||||||
bias=False,
|
self.qkvb_proj: MergedColumnParallelLinear | None
|
||||||
quant_config=quant_config,
|
if self.separate_b_proj:
|
||||||
prefix=f"{prefix}.qkvb_proj",
|
self.qkv_proj = MergedColumnParallelLinear(
|
||||||
)
|
self.hidden_size,
|
||||||
|
[projection_size, projection_size, projection_size],
|
||||||
|
bias=False,
|
||||||
|
quant_config=quant_config,
|
||||||
|
prefix=f"{prefix}.qkv_proj",
|
||||||
|
)
|
||||||
|
self.b_proj = ColumnParallelLinear(
|
||||||
|
self.hidden_size,
|
||||||
|
self.num_heads,
|
||||||
|
bias=False,
|
||||||
|
quant_config=quant_config,
|
||||||
|
prefix=f"{prefix}.b_proj",
|
||||||
|
)
|
||||||
|
self.qkvb_proj = None
|
||||||
|
else:
|
||||||
|
self.qkvb_proj = MergedColumnParallelLinear(
|
||||||
|
self.hidden_size,
|
||||||
|
[projection_size, projection_size, projection_size, self.num_heads],
|
||||||
|
bias=False,
|
||||||
|
quant_config=quant_config,
|
||||||
|
prefix=f"{prefix}.qkvb_proj",
|
||||||
|
)
|
||||||
|
self.qkv_proj = None
|
||||||
|
self.b_proj = None
|
||||||
self.f_proj = ColumnParallelLinear(
|
self.f_proj = ColumnParallelLinear(
|
||||||
self.hidden_size,
|
self.hidden_size,
|
||||||
projection_size,
|
projection_size,
|
||||||
@@ -532,16 +714,23 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase):
|
|||||||
) -> None:
|
) -> None:
|
||||||
del positions
|
del positions
|
||||||
num_tokens = hidden_states.size(0)
|
num_tokens = hidden_states.size(0)
|
||||||
qkvb = self.qkvb_proj(hidden_states)[0]
|
if self.separate_b_proj:
|
||||||
q, k, v, beta_logits = qkvb.split(
|
assert self.qkv_proj is not None and self.b_proj is not None
|
||||||
[
|
qkv = self.qkv_proj(hidden_states)[0]
|
||||||
self.projection_size_per_partition,
|
q, k, v = qkv.split(self.projection_size_per_partition, dim=-1)
|
||||||
self.projection_size_per_partition,
|
beta_logits = self.b_proj(hidden_states)[0]
|
||||||
self.projection_size_per_partition,
|
else:
|
||||||
self.local_num_heads,
|
assert self.qkvb_proj is not None
|
||||||
],
|
qkvb = self.qkvb_proj(hidden_states)[0]
|
||||||
dim=-1,
|
q, k, v, beta_logits = qkvb.split(
|
||||||
)
|
[
|
||||||
|
self.projection_size_per_partition,
|
||||||
|
self.projection_size_per_partition,
|
||||||
|
self.projection_size_per_partition,
|
||||||
|
self.local_num_heads,
|
||||||
|
],
|
||||||
|
dim=-1,
|
||||||
|
)
|
||||||
beta = beta_logits.float().sigmoid().unsqueeze(0)
|
beta = beta_logits.float().sigmoid().unsqueeze(0)
|
||||||
g1 = self.f_proj(hidden_states)[0]
|
g1 = self.f_proj(hidden_states)[0]
|
||||||
g1 = fused_kda_gate(
|
g1 = fused_kda_gate(
|
||||||
@@ -910,7 +1099,6 @@ class BailingMoeV3MoE(nn.Module):
|
|||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.tp_size = get_tensor_model_parallel_world_size()
|
|
||||||
self.num_experts = config.num_experts
|
self.num_experts = config.num_experts
|
||||||
self.top_k = config.num_experts_per_tok
|
self.top_k = config.num_experts_per_tok
|
||||||
self.hidden_size = config.hidden_size
|
self.hidden_size = config.hidden_size
|
||||||
@@ -926,7 +1114,9 @@ class BailingMoeV3MoE(nn.Module):
|
|||||||
config.moe_shared_expert_intermediate_size * config.num_shared_experts
|
config.moe_shared_expert_intermediate_size * config.num_shared_experts
|
||||||
)
|
)
|
||||||
self.shared_experts = BailingMoeV3MLP(
|
self.shared_experts = BailingMoeV3MLP(
|
||||||
intermediate_size=shared_intermediate,
|
intermediate_size=_get_block_fp8_mlp_padded_intermediate_size(
|
||||||
|
quant_config, shared_intermediate, f"{prefix}.shared_experts"
|
||||||
|
),
|
||||||
config=config,
|
config=config,
|
||||||
quant_config=quant_config,
|
quant_config=quant_config,
|
||||||
reduce_results=False,
|
reduce_results=False,
|
||||||
@@ -1149,8 +1339,11 @@ class BailingMoeV3Model(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
|
class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
|
||||||
|
hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={".attention.": ".self_attn."})
|
||||||
|
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping = {
|
||||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||||
|
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||||
"qkvb_proj": ["q_proj", "k_proj", "v_proj", "b_proj"],
|
"qkvb_proj": ["q_proj", "k_proj", "v_proj", "b_proj"],
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1158,6 +1351,7 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
|
|||||||
super().__init__()
|
super().__init__()
|
||||||
config = vllm_config.model_config.hf_config
|
config = vllm_config.model_config.hf_config
|
||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
|
_configure_ling_fp8_quant_config(quant_config)
|
||||||
self.config = config
|
self.config = config
|
||||||
self.quant_config = quant_config
|
self.quant_config = quant_config
|
||||||
self.model = BailingMoeV3Model(
|
self.model = BailingMoeV3Model(
|
||||||
@@ -1226,11 +1420,15 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
|
|||||||
return self.model.get_expert_mapping()
|
return self.model.get_expert_mapping()
|
||||||
|
|
||||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||||
|
weights = self.hf_to_vllm_mapper.apply(weights)
|
||||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||||
loaded_params: set[str] = set()
|
loaded_params: set[str] = set()
|
||||||
stacked_mappings = [
|
stacked_mappings = [
|
||||||
(".gate_up_proj", ".gate_proj", 0),
|
(".gate_up_proj", ".gate_proj", 0),
|
||||||
(".gate_up_proj", ".up_proj", 1),
|
(".gate_up_proj", ".up_proj", 1),
|
||||||
|
(".qkv_proj", ".q_proj", 0),
|
||||||
|
(".qkv_proj", ".k_proj", 1),
|
||||||
|
(".qkv_proj", ".v_proj", 2),
|
||||||
(".qkvb_proj", ".q_proj", 0),
|
(".qkvb_proj", ".q_proj", 0),
|
||||||
(".qkvb_proj", ".k_proj", 1),
|
(".qkvb_proj", ".k_proj", 1),
|
||||||
(".qkvb_proj", ".v_proj", 2),
|
(".qkvb_proj", ".v_proj", 2),
|
||||||
@@ -1263,13 +1461,18 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
|
|||||||
layer_idx = int(name.split("model.layers.")[1].split(".")[0])
|
layer_idx = int(name.split("model.layers.")[1].split(".")[0])
|
||||||
if layer_idx >= self.config.num_hidden_layers:
|
if layer_idx >= self.config.num_hidden_layers:
|
||||||
return None
|
return None
|
||||||
name = name.replace("attention.", "self_attn.")
|
|
||||||
return maybe_remap_kv_scale_name(name, params_dict)
|
return maybe_remap_kv_scale_name(name, params_dict)
|
||||||
|
|
||||||
for orig_name, weight in weights:
|
for orig_name, weight in weights:
|
||||||
name = normalize_name(orig_name)
|
name = normalize_name(orig_name)
|
||||||
if name is None:
|
if name is None:
|
||||||
continue
|
continue
|
||||||
|
weight = _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
|
||||||
|
self.quant_config,
|
||||||
|
self.config,
|
||||||
|
name,
|
||||||
|
weight,
|
||||||
|
)
|
||||||
loaded = False
|
loaded = False
|
||||||
for param_suf, weight_suf, stacked_shard_id in stacked_mappings:
|
for param_suf, weight_suf, stacked_shard_id in stacked_mappings:
|
||||||
if weight_suf in name:
|
if weight_suf in name:
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.fused_moe import (
|
|||||||
fused_moe_make_expert_params_mapping,
|
fused_moe_make_expert_params_mapping,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||||
|
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
ParallelLMHead,
|
ParallelLMHead,
|
||||||
@@ -25,8 +26,11 @@ from vllm.model_executor.model_loader.weight_utils import (
|
|||||||
maybe_remap_kv_scale_name,
|
maybe_remap_kv_scale_name,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.models.bailing_moe_v3 import (
|
from vllm.model_executor.models.bailing_moe_v3 import (
|
||||||
|
BailingMoeV3ForCausalLM,
|
||||||
BailingMoeV3MLAAttention,
|
BailingMoeV3MLAAttention,
|
||||||
BailingMoeV3MoE,
|
BailingMoeV3MoE,
|
||||||
|
_configure_ling_fp8_quant_config,
|
||||||
|
_maybe_pad_block_fp8_shared_expert_checkpoint_tensor,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.models.interfaces import SupportsPP
|
from vllm.model_executor.models.interfaces import SupportsPP
|
||||||
from vllm.model_executor.models.utils import (
|
from vllm.model_executor.models.utils import (
|
||||||
@@ -79,7 +83,14 @@ class BailingMoeV3MultiTokenPredictorLayer(nn.Module):
|
|||||||
self.layer_id = layer_id
|
self.layer_id = layer_id
|
||||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
self.eh_proj = ReplicatedLinear(
|
||||||
|
config.hidden_size * 2,
|
||||||
|
config.hidden_size,
|
||||||
|
bias=False,
|
||||||
|
quant_config=vllm_config.quant_config,
|
||||||
|
prefix=maybe_prefix(prefix, "eh_proj"),
|
||||||
|
return_bias=False,
|
||||||
|
)
|
||||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||||
self.self_attn = BailingMoeV3MLAAttention(
|
self.self_attn = BailingMoeV3MLAAttention(
|
||||||
config,
|
config,
|
||||||
@@ -204,6 +215,8 @@ class BailingMoeV3MultiTokenPredictor(nn.Module):
|
|||||||
|
|
||||||
@support_torch_compile
|
@support_torch_compile
|
||||||
class BailingMoeV3MTPModel(nn.Module, SupportsPP):
|
class BailingMoeV3MTPModel(nn.Module, SupportsPP):
|
||||||
|
hf_to_vllm_mapper = BailingMoeV3ForCausalLM.hf_to_vllm_mapper
|
||||||
|
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping = {
|
||||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||||
"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"],
|
"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"],
|
||||||
@@ -217,6 +230,8 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
|
|||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.config = _get_draft_hf_config(vllm_config)
|
self.config = _get_draft_hf_config(vllm_config)
|
||||||
|
self.quant_config = vllm_config.quant_config
|
||||||
|
_configure_ling_fp8_quant_config(self.quant_config)
|
||||||
self.model = BailingMoeV3MultiTokenPredictor(
|
self.model = BailingMoeV3MultiTokenPredictor(
|
||||||
vllm_config=vllm_config,
|
vllm_config=vllm_config,
|
||||||
prefix=maybe_prefix(prefix, "model"),
|
prefix=maybe_prefix(prefix, "model"),
|
||||||
@@ -265,6 +280,7 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||||
|
weights = self.hf_to_vllm_mapper.apply(weights)
|
||||||
stacked_params_mapping = [
|
stacked_params_mapping = [
|
||||||
(".fused_qkv_a_proj", ".q_a_proj", 0),
|
(".fused_qkv_a_proj", ".q_a_proj", 0),
|
||||||
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
|
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
|
||||||
@@ -320,7 +336,6 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
def normalize_name(name: str) -> str:
|
def normalize_name(name: str) -> str:
|
||||||
name = name.replace(".attention.", ".self_attn.")
|
|
||||||
return name.replace(
|
return name.replace(
|
||||||
"mlp.gate.e_score_correction_bias",
|
"mlp.gate.e_score_correction_bias",
|
||||||
"mlp.gate.expert_bias",
|
"mlp.gate.expert_bias",
|
||||||
@@ -349,6 +364,12 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
|
|||||||
if spec_layer is None:
|
if spec_layer is None:
|
||||||
continue
|
continue
|
||||||
name = normalize_name(name)
|
name = normalize_name(name)
|
||||||
|
loaded_weight = _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
|
||||||
|
self.quant_config,
|
||||||
|
self.config,
|
||||||
|
name,
|
||||||
|
loaded_weight,
|
||||||
|
)
|
||||||
|
|
||||||
loaded = False
|
loaded = False
|
||||||
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
|
for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
|
||||||
|
|||||||
Reference in New Issue
Block a user