[Model] Add GraniteSWA and GraniteMoeSWA via existing Granite (#52706)
Signed-off-by: Davis Wertheimer <davis.wertheimer@ibm.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Harry Mellor
parent
8e46accab2
commit
3130573630
@@ -405,6 +405,8 @@ th {
|
||||
| `GraniteMoeForCausalLM` | Granite 3.0 MoE, PowerMoE | `ibm-granite/granite-3.0-1b-a400m-base`, `ibm-granite/granite-3.0-3b-a800m-instruct`, `ibm/PowerMoE-3b`, etc. | ✅︎ | ✅︎ |
|
||||
| `GraniteMoeHybridForCausalLM` | Granite 4.0 MoE Hybrid | `ibm-granite/granite-4.0-tiny-preview`, etc. | ✅︎ | ✅︎ |
|
||||
| `GraniteMoeSharedForCausalLM` | Granite MoE Shared | `ibm-research/moe-7b-1b-active-shared-experts` (test model) | ✅︎ | ✅︎ |
|
||||
| `GraniteMoeSWAForCausalLM` | Granite MoE SWA | `ibm-granite/granite-swash-3b-a600m` | ✅︎ | ✅︎ |
|
||||
| `GraniteSWAForCausalLM` | Granite SWA | `ibm-granite/granite-swash-2b` | ✅︎ | ✅︎ |
|
||||
| `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
|
||||
| `HrmTextForCausalLM` | HRM-Text | `sapientinc/HRM-Text-1B`, etc. | | |
|
||||
| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ |
|
||||
|
||||
@@ -1,14 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
from transformers import GraniteConfig
|
||||
|
||||
from ...utils import check_logprobs_close
|
||||
from vllm.model_executor.models.granite import granite_layer_attn_params
|
||||
|
||||
MODELS = [
|
||||
from ...utils import check_logprobs_close, check_transformers_version
|
||||
|
||||
# model -> minimum transformers version, or None if unconstrained
|
||||
MODELS = {
|
||||
# TODO(sang): Sliding window should be tested separately.
|
||||
"ibm/PowerLM-3b",
|
||||
"ibm/PowerMoE-3b",
|
||||
]
|
||||
"ibm/PowerLM-3b": None,
|
||||
"ibm/PowerMoE-3b": None,
|
||||
"ibm-granite/granite-swash-2b": "5.15.1",
|
||||
"ibm-granite/granite-swash-3b-a600m": "5.15.1",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@@ -25,6 +31,8 @@ def test_models(
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
check_transformers_version(model, min_transformers_version=MODELS[model])
|
||||
|
||||
with hf_runner(model, dtype=dtype) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
@@ -40,3 +48,34 @@ def test_models(
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
def test_granite_swa_features_are_off_without_swa_config():
|
||||
"""A plain Granite config must not pick up sliding windows or sinks."""
|
||||
config = GraniteConfig(num_hidden_layers=4)
|
||||
theta = config.rope_parameters["rope_theta"]
|
||||
|
||||
assert [granite_layer_attn_params(config, i) for i in range(4)] == [
|
||||
(None, theta, False)
|
||||
] * 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attention_sinks, expected_sink", [(None, True), (False, False)]
|
||||
)
|
||||
def test_granite_swa_features_resolve_per_layer(attention_sinks, expected_sink):
|
||||
"""`layer_types`/`layer_rope_theta` apply per layer; theta 0 means NoPE."""
|
||||
kwargs = {} if attention_sinks is None else {"attention_sinks": attention_sinks}
|
||||
config = GraniteConfig(
|
||||
num_hidden_layers=3,
|
||||
sliding_window=128,
|
||||
layer_types=["full_attention", "sliding_attention", "sliding_attention"],
|
||||
layer_rope_theta=[10000.0, 0.0, 1000000.0],
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
assert [granite_layer_attn_params(config, i) for i in range(3)] == [
|
||||
(None, 10000.0, expected_sink),
|
||||
(128, 0.0, expected_sink),
|
||||
(128, 1000000.0, expected_sink),
|
||||
]
|
||||
|
||||
@@ -320,6 +320,12 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"GraniteMoeSharedForCausalLM": _HfExamplesInfo(
|
||||
"ibm-research/moe-7b-1b-active-shared-experts"
|
||||
),
|
||||
"GraniteMoeSWAForCausalLM": _HfExamplesInfo(
|
||||
"ibm-granite/granite-swash-3b-a600m", min_transformers_version="5.15.1"
|
||||
),
|
||||
"GraniteSWAForCausalLM": _HfExamplesInfo(
|
||||
"ibm-granite/granite-swash-2b", min_transformers_version="5.15.1"
|
||||
),
|
||||
"HrmTextForCausalLM": _HfExamplesInfo(
|
||||
"sapientinc/HRM-Text-1B",
|
||||
min_transformers_version="5.9.0",
|
||||
|
||||
@@ -22,7 +22,13 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only IBM Granite model compatible with HuggingFace weights."""
|
||||
"""Inference-only IBM Granite model compatible with HuggingFace weights.
|
||||
|
||||
Also serves the `granite_swa` checkpoints (`GraniteSWAForCausalLM`), supporting
|
||||
three additional features: per-layer sliding window attention (`layer_types`), a
|
||||
learnable per-head attention sink (`self_attn.sinks`), and a per-layer RoPE base
|
||||
(`layer_rope_theta`, with 0 for NoPE).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
@@ -30,6 +36,7 @@ from itertools import islice
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import GraniteConfig
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -49,6 +56,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
|
||||
@@ -56,11 +65,44 @@ from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
def granite_layer_attn_params(
|
||||
config: PretrainedConfig, layer_idx: int
|
||||
) -> tuple[int | None, float, bool]:
|
||||
"""Resolve one layer's sliding window, RoPE base and sink usage.
|
||||
|
||||
Plain Granite configs carry no SWA fields and fall back to full
|
||||
attention, global RoPE base and no sink. HF SWA checkpoints use
|
||||
sinks without a dedicated flag, so assume true when `layer_types`
|
||||
is used, and allow `attention_sinks` to override that decision.
|
||||
|
||||
Returns:
|
||||
Sliding window size (`None` for full attention), RoPE base theta (`0`
|
||||
for NoPE), and attention sink presence/absence.
|
||||
"""
|
||||
layer_types = getattr(config, "layer_types", None)
|
||||
sliding_window = (
|
||||
config.sliding_window
|
||||
if layer_types is not None and layer_types[layer_idx] == "sliding_attention"
|
||||
else None
|
||||
)
|
||||
|
||||
layer_rope_theta = getattr(config, "layer_rope_theta", None)
|
||||
rope_theta = (
|
||||
layer_rope_theta[layer_idx]
|
||||
if layer_rope_theta is not None
|
||||
else config.rope_parameters["rope_theta"]
|
||||
)
|
||||
|
||||
has_sink = getattr(config, "attention_sinks", layer_types is not None)
|
||||
return sliding_window, rope_theta, has_sink
|
||||
|
||||
|
||||
class GraniteMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -154,11 +196,25 @@ class GraniteAttention(nn.Module):
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
sliding_window, rope_theta, has_sink = granite_layer_attn_params(
|
||||
config, extract_layer_index(prefix)
|
||||
)
|
||||
|
||||
self.use_rope = rope_theta != 0
|
||||
if self.use_rope:
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters={**config.rope_parameters, "rope_theta": rope_theta},
|
||||
)
|
||||
|
||||
# Per-head sink, applied in backend as extra logit in the softmax denom
|
||||
if has_sink:
|
||||
self.sinks = nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
|
||||
set_weight_attrs(self.sinks, {"weight_loader": sharded_weight_loader(0)})
|
||||
else:
|
||||
self.sinks = None
|
||||
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
@@ -166,7 +222,9 @@ class GraniteAttention(nn.Module):
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
per_layer_sliding_window=sliding_window,
|
||||
prefix=f"{prefix}.attn",
|
||||
sinks=self.sinks,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -176,7 +234,8 @@ class GraniteAttention(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
if self.use_rope:
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
@@ -413,8 +472,5 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||
# processed with quantization, LoRA, fine-tuning, etc.
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
|
||||
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=skip_prefixes,
|
||||
)
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -26,10 +26,10 @@
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -39,10 +39,7 @@ from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
@@ -56,15 +53,20 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .granite import granite_layer_attn_params
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import AutoWeightsLoader, is_pp_missing_parameter, make_layers, maybe_prefix
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
class GraniteMoeMoE(nn.Module):
|
||||
@@ -139,11 +141,11 @@ class GraniteMoeMoE(nn.Module):
|
||||
class GraniteMoeAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
max_position: int = 4096 * 32,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
attention_multiplier: float | None = None,
|
||||
@@ -190,12 +192,25 @@ class GraniteMoeAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position,
|
||||
rope_parameters=rope_parameters,
|
||||
is_neox_style=True,
|
||||
sliding_window, rope_theta, has_sink = granite_layer_attn_params(
|
||||
config, extract_layer_index(prefix)
|
||||
)
|
||||
|
||||
self.use_rope = rope_theta != 0
|
||||
if self.use_rope:
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position,
|
||||
rope_parameters={**config.rope_parameters, "rope_theta": rope_theta},
|
||||
is_neox_style=True,
|
||||
)
|
||||
|
||||
if has_sink:
|
||||
self.sinks = nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
|
||||
set_weight_attrs(self.sinks, {"weight_loader": sharded_weight_loader(0)})
|
||||
else:
|
||||
self.sinks = None
|
||||
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
@@ -203,7 +218,9 @@ class GraniteMoeAttention(nn.Module):
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
per_layer_sliding_window=sliding_window,
|
||||
prefix=f"{prefix}.attn",
|
||||
sinks=self.sinks,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -213,7 +230,8 @@ class GraniteMoeAttention(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
if self.use_rope:
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
@@ -234,11 +252,11 @@ class GraniteMoeDecoderLayer(nn.Module):
|
||||
|
||||
self.hidden_size = config.hidden_size
|
||||
self.self_attn = GraniteMoeAttention(
|
||||
config=config,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
max_position=config.max_position_embeddings,
|
||||
num_kv_heads=config.num_key_value_heads,
|
||||
rope_parameters=config.rope_parameters,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
@@ -284,6 +302,23 @@ class GraniteMoeDecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class GraniteMoeModel(nn.Module):
|
||||
hf_to_vllm_mapper: WeightsMapper = WeightsMapper(
|
||||
orig_to_new_suffix={
|
||||
# Legacy names to new names
|
||||
"moe.input_linear.weight": "moe.experts.gate_up_proj",
|
||||
"moe.output_linear.weight": "moe.experts.down_proj",
|
||||
".router.layer.weight": ".gate.weight",
|
||||
# Checkpoint name to vLLM name
|
||||
".router.weight": ".gate.weight",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -339,135 +374,9 @@ class GraniteMoeModel(nn.Module):
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def _load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""
|
||||
This function is copied from `MixtralModel.load_weights`, mainly to
|
||||
decouple from mixtral, avoiding impact on support like BNB
|
||||
quantization.
|
||||
"""
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
]
|
||||
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.num_local_experts,
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if name.endswith("scale"):
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
else:
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if (
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip layers on other devices.
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
new_weights = {}
|
||||
for n, p in weights:
|
||||
if n.endswith(".block_sparse_moe.input_linear.weight"):
|
||||
for e in range(p.size(0)):
|
||||
w1_name = n.replace(
|
||||
".block_sparse_moe.input_linear.weight",
|
||||
f".block_sparse_moe.experts.{e}.w1.weight",
|
||||
)
|
||||
w3_name = n.replace(
|
||||
".block_sparse_moe.input_linear.weight",
|
||||
f".block_sparse_moe.experts.{e}.w3.weight",
|
||||
)
|
||||
w1_param, w3_param = p[e].chunk(2, dim=0)
|
||||
assert w1_name not in new_weights
|
||||
assert w3_name not in new_weights
|
||||
new_weights[w1_name] = w1_param
|
||||
new_weights[w3_name] = w3_param
|
||||
elif n.endswith(".block_sparse_moe.output_linear.weight"):
|
||||
for e in range(p.size(0)):
|
||||
w2_name = n.replace(
|
||||
".block_sparse_moe.output_linear.weight",
|
||||
f".block_sparse_moe.experts.{e}.w2.weight",
|
||||
)
|
||||
w2_param = p[e]
|
||||
assert w2_name not in new_weights
|
||||
new_weights[w2_name] = w2_param
|
||||
elif n.endswith(".block_sparse_moe.router.layer.weight"):
|
||||
gate_name = n.replace(
|
||||
".block_sparse_moe.router.layer.weight",
|
||||
".block_sparse_moe.gate.weight",
|
||||
)
|
||||
assert gate_name not in new_weights
|
||||
new_weights[gate_name] = p
|
||||
else:
|
||||
new_weights[n] = p
|
||||
return self._load_weights(new_weights.items())
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
@@ -543,8 +452,6 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -4,6 +4,10 @@
|
||||
|
||||
The architecture is the same as granitemoe but with the addition of shared
|
||||
experts.
|
||||
|
||||
Also serves the `granitemoe_swa` checkpoints (`GraniteMoeSWAForCausalLM`), which
|
||||
add the same per-layer sliding window, attention sink and per-layer RoPE support
|
||||
as `granite_swa` (see `granite.py`).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
@@ -85,11 +89,11 @@ class GraniteMoeSharedDecoderLayer(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.self_attn = GraniteMoeAttention(
|
||||
config=config,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
max_position=config.max_position_embeddings,
|
||||
num_kv_heads=config.num_key_value_heads,
|
||||
rope_parameters=config.rope_parameters,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
@@ -207,43 +211,8 @@ class GraniteMoeSharedModel(nn.Module):
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
new_weights = {}
|
||||
for n, p in weights:
|
||||
if n.endswith(".block_sparse_moe.input_linear.weight"):
|
||||
for e in range(p.size(0)):
|
||||
w1_name = n.replace(
|
||||
".block_sparse_moe.input_linear.weight",
|
||||
f".block_sparse_moe.experts.{e}.w1.weight",
|
||||
)
|
||||
w3_name = n.replace(
|
||||
".block_sparse_moe.input_linear.weight",
|
||||
f".block_sparse_moe.experts.{e}.w3.weight",
|
||||
)
|
||||
w1_param, w3_param = p[e].chunk(2, dim=0)
|
||||
assert w1_name not in new_weights
|
||||
assert w3_name not in new_weights
|
||||
new_weights[w1_name] = w1_param
|
||||
new_weights[w3_name] = w3_param
|
||||
elif n.endswith(".block_sparse_moe.output_linear.weight"):
|
||||
for e in range(p.size(0)):
|
||||
w2_name = n.replace(
|
||||
".block_sparse_moe.output_linear.weight",
|
||||
f".block_sparse_moe.experts.{e}.w2.weight",
|
||||
)
|
||||
w2_param = p[e]
|
||||
assert w2_name not in new_weights
|
||||
new_weights[w2_name] = w2_param
|
||||
elif n.endswith(".block_sparse_moe.router.layer.weight"):
|
||||
gate_name = n.replace(
|
||||
".block_sparse_moe.router.layer.weight",
|
||||
".block_sparse_moe.gate.weight",
|
||||
)
|
||||
assert gate_name not in new_weights
|
||||
new_weights[gate_name] = p
|
||||
else:
|
||||
new_weights[n] = p
|
||||
return GraniteMoeModel._load_weights(self, new_weights.items())
|
||||
hf_to_vllm_mapper = GraniteMoeModel.hf_to_vllm_mapper
|
||||
load_weights = GraniteMoeModel.load_weights
|
||||
|
||||
|
||||
class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
@@ -320,8 +289,6 @@ class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -124,6 +124,8 @@ _TEXT_GENERATION_MODELS = {
|
||||
"GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
|
||||
"GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
|
||||
"GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
|
||||
"GraniteMoeSWAForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
|
||||
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
|
||||
"GritLM": ("gritlm", "GritLM"),
|
||||
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
|
||||
"HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
|
||||
|
||||
Reference in New Issue
Block a user