[Model] Add GraniteSWA and GraniteMoeSWA via existing Granite (#52706)

Signed-off-by: Davis Wertheimer <davis.wertheimer@ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Davis Wertheimer
2026-08-18 20:32:46 -07:00
committed by GitHub
co-authored by Claude Harry Mellor
parent 8e46accab2
commit 3130573630
7 changed files with 186 additions and 207 deletions
+2
View File
@@ -405,6 +405,8 @@ th {
| `GraniteMoeForCausalLM` | Granite 3.0 MoE, PowerMoE | `ibm-granite/granite-3.0-1b-a400m-base`, `ibm-granite/granite-3.0-3b-a800m-instruct`, `ibm/PowerMoE-3b`, etc. | ✅︎ | ✅︎ |
| `GraniteMoeHybridForCausalLM` | Granite 4.0 MoE Hybrid | `ibm-granite/granite-4.0-tiny-preview`, etc. | ✅︎ | ✅︎ |
| `GraniteMoeSharedForCausalLM` | Granite MoE Shared | `ibm-research/moe-7b-1b-active-shared-experts` (test model) | ✅︎ | ✅︎ |
| `GraniteMoeSWAForCausalLM` | Granite MoE SWA | `ibm-granite/granite-swash-3b-a600m` | ✅︎ | ✅︎ |
| `GraniteSWAForCausalLM` | Granite SWA | `ibm-granite/granite-swash-2b` | ✅︎ | ✅︎ |
| `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
| `HrmTextForCausalLM` | HRM-Text | `sapientinc/HRM-Text-1B`, etc. | | |
| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ |
@@ -1,14 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import GraniteConfig
from ...utils import check_logprobs_close
from vllm.model_executor.models.granite import granite_layer_attn_params
MODELS = [
from ...utils import check_logprobs_close, check_transformers_version
# model -> minimum transformers version, or None if unconstrained
MODELS = {
# TODO(sang): Sliding window should be tested separately.
"ibm/PowerLM-3b",
"ibm/PowerMoE-3b",
]
"ibm/PowerLM-3b": None,
"ibm/PowerMoE-3b": None,
"ibm-granite/granite-swash-2b": "5.15.1",
"ibm-granite/granite-swash-3b-a600m": "5.15.1",
}
@pytest.mark.parametrize("model", MODELS)
@@ -25,6 +31,8 @@ def test_models(
max_tokens: int,
num_logprobs: int,
) -> None:
check_transformers_version(model, min_transformers_version=MODELS[model])
with hf_runner(model, dtype=dtype) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
@@ -40,3 +48,34 @@ def test_models(
name_0="hf",
name_1="vllm",
)
def test_granite_swa_features_are_off_without_swa_config():
"""A plain Granite config must not pick up sliding windows or sinks."""
config = GraniteConfig(num_hidden_layers=4)
theta = config.rope_parameters["rope_theta"]
assert [granite_layer_attn_params(config, i) for i in range(4)] == [
(None, theta, False)
] * 4
@pytest.mark.parametrize(
"attention_sinks, expected_sink", [(None, True), (False, False)]
)
def test_granite_swa_features_resolve_per_layer(attention_sinks, expected_sink):
"""`layer_types`/`layer_rope_theta` apply per layer; theta 0 means NoPE."""
kwargs = {} if attention_sinks is None else {"attention_sinks": attention_sinks}
config = GraniteConfig(
num_hidden_layers=3,
sliding_window=128,
layer_types=["full_attention", "sliding_attention", "sliding_attention"],
layer_rope_theta=[10000.0, 0.0, 1000000.0],
**kwargs,
)
assert [granite_layer_attn_params(config, i) for i in range(3)] == [
(None, 10000.0, expected_sink),
(128, 0.0, expected_sink),
(128, 1000000.0, expected_sink),
]
+6
View File
@@ -320,6 +320,12 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"GraniteMoeSharedForCausalLM": _HfExamplesInfo(
"ibm-research/moe-7b-1b-active-shared-experts"
),
"GraniteMoeSWAForCausalLM": _HfExamplesInfo(
"ibm-granite/granite-swash-3b-a600m", min_transformers_version="5.15.1"
),
"GraniteSWAForCausalLM": _HfExamplesInfo(
"ibm-granite/granite-swash-2b", min_transformers_version="5.15.1"
),
"HrmTextForCausalLM": _HfExamplesInfo(
"sapientinc/HRM-Text-1B",
min_transformers_version="5.9.0",
+66 -10
View File
@@ -22,7 +22,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only IBM Granite model compatible with HuggingFace weights."""
"""Inference-only IBM Granite model compatible with HuggingFace weights.
Also serves the `granite_swa` checkpoints (`GraniteSWAForCausalLM`), supporting
three additional features: per-layer sliding window attention (`layer_types`), a
learnable per-head attention sink (`self_attn.sinks`), and a per-layer RoPE base
(`layer_rope_theta`, with 0 for NoPE).
"""
from collections.abc import Iterable
from itertools import islice
@@ -30,6 +36,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import GraniteConfig
from transformers.configuration_utils import PretrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -49,6 +56,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
@@ -56,11 +65,44 @@ from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
make_layers,
maybe_prefix,
)
def granite_layer_attn_params(
config: PretrainedConfig, layer_idx: int
) -> tuple[int | None, float, bool]:
"""Resolve one layer's sliding window, RoPE base and sink usage.
Plain Granite configs carry no SWA fields and fall back to full
attention, global RoPE base and no sink. HF SWA checkpoints use
sinks without a dedicated flag, so assume true when `layer_types`
is used, and allow `attention_sinks` to override that decision.
Returns:
Sliding window size (`None` for full attention), RoPE base theta (`0`
for NoPE), and attention sink presence/absence.
"""
layer_types = getattr(config, "layer_types", None)
sliding_window = (
config.sliding_window
if layer_types is not None and layer_types[layer_idx] == "sliding_attention"
else None
)
layer_rope_theta = getattr(config, "layer_rope_theta", None)
rope_theta = (
layer_rope_theta[layer_idx]
if layer_rope_theta is not None
else config.rope_parameters["rope_theta"]
)
has_sink = getattr(config, "attention_sinks", layer_types is not None)
return sliding_window, rope_theta, has_sink
class GraniteMLP(nn.Module):
def __init__(
self,
@@ -154,11 +196,25 @@ class GraniteAttention(nn.Module):
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
sliding_window, rope_theta, has_sink = granite_layer_attn_params(
config, extract_layer_index(prefix)
)
self.use_rope = rope_theta != 0
if self.use_rope:
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
rope_parameters={**config.rope_parameters, "rope_theta": rope_theta},
)
# Per-head sink, applied in backend as extra logit in the softmax denom
if has_sink:
self.sinks = nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
set_weight_attrs(self.sinks, {"weight_loader": sharded_weight_loader(0)})
else:
self.sinks = None
self.attn = Attention(
self.num_heads,
self.head_dim,
@@ -166,7 +222,9 @@ class GraniteAttention(nn.Module):
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
per_layer_sliding_window=sliding_window,
prefix=f"{prefix}.attn",
sinks=self.sinks,
)
def forward(
@@ -176,7 +234,8 @@ class GraniteAttention(nn.Module):
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
if self.use_rope:
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.o_proj(attn_output)
return output
@@ -413,8 +472,5 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
# processed with quantization, LoRA, fine-tuning, etc.
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
loader = AutoWeightsLoader(
self,
skip_prefixes=skip_prefixes,
)
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
return loader.load_weights(weights)
+57 -150
View File
@@ -26,10 +26,10 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any
import torch
from torch import nn
from transformers.configuration_utils import PretrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -39,10 +39,7 @@ from vllm.distributed import (
tensor_model_parallel_all_gather,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -56,15 +53,20 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.model_executor.utils import set_weight_attrs
from vllm.sequence import IntermediateTensors
from .granite import granite_layer_attn_params
from .interfaces import SupportsLoRA, SupportsPP
from .utils import AutoWeightsLoader, is_pp_missing_parameter, make_layers, maybe_prefix
from .utils import (
AutoWeightsLoader,
WeightsMapper,
extract_layer_index,
make_layers,
maybe_prefix,
)
class GraniteMoeMoE(nn.Module):
@@ -139,11 +141,11 @@ class GraniteMoeMoE(nn.Module):
class GraniteMoeAttention(nn.Module):
def __init__(
self,
config: PretrainedConfig,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
max_position: int = 4096 * 32,
rope_parameters: dict[str, Any] | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
attention_multiplier: float | None = None,
@@ -190,12 +192,25 @@ class GraniteMoeAttention(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position,
rope_parameters=rope_parameters,
is_neox_style=True,
sliding_window, rope_theta, has_sink = granite_layer_attn_params(
config, extract_layer_index(prefix)
)
self.use_rope = rope_theta != 0
if self.use_rope:
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position,
rope_parameters={**config.rope_parameters, "rope_theta": rope_theta},
is_neox_style=True,
)
if has_sink:
self.sinks = nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
set_weight_attrs(self.sinks, {"weight_loader": sharded_weight_loader(0)})
else:
self.sinks = None
self.attn = Attention(
self.num_heads,
self.head_dim,
@@ -203,7 +218,9 @@ class GraniteMoeAttention(nn.Module):
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
per_layer_sliding_window=sliding_window,
prefix=f"{prefix}.attn",
sinks=self.sinks,
)
def forward(
@@ -213,7 +230,8 @@ class GraniteMoeAttention(nn.Module):
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
if self.use_rope:
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.o_proj(attn_output)
return output
@@ -234,11 +252,11 @@ class GraniteMoeDecoderLayer(nn.Module):
self.hidden_size = config.hidden_size
self.self_attn = GraniteMoeAttention(
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
max_position=config.max_position_embeddings,
num_kv_heads=config.num_key_value_heads,
rope_parameters=config.rope_parameters,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
@@ -284,6 +302,23 @@ class GraniteMoeDecoderLayer(nn.Module):
@support_torch_compile
class GraniteMoeModel(nn.Module):
hf_to_vllm_mapper: WeightsMapper = WeightsMapper(
orig_to_new_suffix={
# Legacy names to new names
"moe.input_linear.weight": "moe.experts.gate_up_proj",
"moe.output_linear.weight": "moe.experts.down_proj",
".router.layer.weight": ".gate.weight",
# Checkpoint name to vLLM name
".router.weight": ".gate.weight",
},
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
},
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -339,135 +374,9 @@ class GraniteMoeModel(nn.Module):
hidden_states = self.norm(hidden_states)
return hidden_states
def _load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""
This function is copied from `MixtralModel.load_weights`, mainly to
decouple from mixtral, avoiding impact on support like BNB
quantization.
"""
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.num_local_experts,
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name.endswith("scale"):
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
new_weights = {}
for n, p in weights:
if n.endswith(".block_sparse_moe.input_linear.weight"):
for e in range(p.size(0)):
w1_name = n.replace(
".block_sparse_moe.input_linear.weight",
f".block_sparse_moe.experts.{e}.w1.weight",
)
w3_name = n.replace(
".block_sparse_moe.input_linear.weight",
f".block_sparse_moe.experts.{e}.w3.weight",
)
w1_param, w3_param = p[e].chunk(2, dim=0)
assert w1_name not in new_weights
assert w3_name not in new_weights
new_weights[w1_name] = w1_param
new_weights[w3_name] = w3_param
elif n.endswith(".block_sparse_moe.output_linear.weight"):
for e in range(p.size(0)):
w2_name = n.replace(
".block_sparse_moe.output_linear.weight",
f".block_sparse_moe.experts.{e}.w2.weight",
)
w2_param = p[e]
assert w2_name not in new_weights
new_weights[w2_name] = w2_param
elif n.endswith(".block_sparse_moe.router.layer.weight"):
gate_name = n.replace(
".block_sparse_moe.router.layer.weight",
".block_sparse_moe.gate.weight",
)
assert gate_name not in new_weights
new_weights[gate_name] = p
else:
new_weights[n] = p
return self._load_weights(new_weights.items())
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
@@ -543,8 +452,6 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
return loader.load_weights(weights)
+9 -42
View File
@@ -4,6 +4,10 @@
The architecture is the same as granitemoe but with the addition of shared
experts.
Also serves the `granitemoe_swa` checkpoints (`GraniteMoeSWAForCausalLM`), which
add the same per-layer sliding window, attention sink and per-layer RoPE support
as `granite_swa` (see `granite.py`).
"""
from collections.abc import Iterable
@@ -85,11 +89,11 @@ class GraniteMoeSharedDecoderLayer(nn.Module):
super().__init__()
self.hidden_size = config.hidden_size
self.self_attn = GraniteMoeAttention(
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
max_position=config.max_position_embeddings,
num_kv_heads=config.num_key_value_heads,
rope_parameters=config.rope_parameters,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
@@ -207,43 +211,8 @@ class GraniteMoeSharedModel(nn.Module):
hidden_states = self.norm(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
new_weights = {}
for n, p in weights:
if n.endswith(".block_sparse_moe.input_linear.weight"):
for e in range(p.size(0)):
w1_name = n.replace(
".block_sparse_moe.input_linear.weight",
f".block_sparse_moe.experts.{e}.w1.weight",
)
w3_name = n.replace(
".block_sparse_moe.input_linear.weight",
f".block_sparse_moe.experts.{e}.w3.weight",
)
w1_param, w3_param = p[e].chunk(2, dim=0)
assert w1_name not in new_weights
assert w3_name not in new_weights
new_weights[w1_name] = w1_param
new_weights[w3_name] = w3_param
elif n.endswith(".block_sparse_moe.output_linear.weight"):
for e in range(p.size(0)):
w2_name = n.replace(
".block_sparse_moe.output_linear.weight",
f".block_sparse_moe.experts.{e}.w2.weight",
)
w2_param = p[e]
assert w2_name not in new_weights
new_weights[w2_name] = w2_param
elif n.endswith(".block_sparse_moe.router.layer.weight"):
gate_name = n.replace(
".block_sparse_moe.router.layer.weight",
".block_sparse_moe.gate.weight",
)
assert gate_name not in new_weights
new_weights[gate_name] = p
else:
new_weights[n] = p
return GraniteMoeModel._load_weights(self, new_weights.items())
hf_to_vllm_mapper = GraniteMoeModel.hf_to_vllm_mapper
load_weights = GraniteMoeModel.load_weights
class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
@@ -320,8 +289,6 @@ class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
return loader.load_weights(weights)
+2
View File
@@ -124,6 +124,8 @@ _TEXT_GENERATION_MODELS = {
"GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
"GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
"GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
"GraniteMoeSWAForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"),
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
"GritLM": ("gritlm", "GritLM"),
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
"HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),