[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -250,8 +250,8 @@ def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
return BatchFeature(data=inputs, tensor_type="pt")
|
||||
|
||||
hf_model.processor = processor
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.language.model.embed_tokens
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.language.model.embed_tokens
|
||||
)
|
||||
return hf_model
|
||||
|
||||
@@ -353,8 +353,8 @@ def glm4v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
)
|
||||
|
||||
hf_model.processor = processor
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.transformer.output_layer
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.transformer.output_layer
|
||||
)
|
||||
return hf_model
|
||||
|
||||
@@ -442,8 +442,8 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
hf_model.model.img_context_token_id = img_context_token_id
|
||||
hf_model.processor = H2OVLProcessor(hf_model)
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.language_model.get_output_embeddings()
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.language_model.get_output_embeddings()
|
||||
)
|
||||
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
|
||||
return hf_model
|
||||
@@ -678,8 +678,8 @@ def skyworkr1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
hf_model.model.img_context_token_id = img_context_token_id
|
||||
hf_model.processor = SkyworkR1VProcessor(hf_model)
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.language_model.get_output_embeddings()
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.language_model.get_output_embeddings()
|
||||
)
|
||||
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
|
||||
return hf_model
|
||||
@@ -787,8 +787,8 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
hf_model.model.img_context_token_id = img_context_token_id
|
||||
hf_model.processor = InternVLProcessor(hf_model)
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.language_model.get_output_embeddings()
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.language_model.get_output_embeddings()
|
||||
)
|
||||
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
|
||||
return hf_model
|
||||
@@ -941,8 +941,8 @@ def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
|
||||
def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
"""Patches and returns an instance of the HfRunner to use for Ovis2."""
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.llm.get_output_embeddings()
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.llm.get_output_embeddings()
|
||||
)
|
||||
|
||||
def processor(*args, text="", images=None, **kwargs):
|
||||
@@ -977,8 +977,8 @@ def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
|
||||
def ovis2_5_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
"""Patches and returns an instance of the HfRunner to use for Ovis2."""
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.llm.get_output_embeddings()
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.llm.get_output_embeddings()
|
||||
)
|
||||
|
||||
def processor(*args, text="", images=None, videos=None, **kwargs):
|
||||
@@ -1209,221 +1209,3 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
|
||||
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign]
|
||||
hf_model.model.generate = patched_generate # type: ignore[method-assign]
|
||||
return hf_model
|
||||
|
||||
|
||||
def moondream3_processor(model: str):
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
return Moondream3Processor.from_pretrained(model, trust_remote_code=True)
|
||||
|
||||
|
||||
def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
"""Patch HfRunner for Moondream3."""
|
||||
moondream_processor = hf_model.processor
|
||||
|
||||
def processor(*args, text="", images=None, **kwargs):
|
||||
if images is None:
|
||||
return moondream_processor(text=text, **kwargs)
|
||||
|
||||
images_list = [images] if isinstance(images, Image) else images
|
||||
return moondream_processor(images=images_list, text=text, **kwargs)
|
||||
|
||||
hf_model.processor = processor
|
||||
|
||||
# Expose the LM head for logprob extraction.
|
||||
hf_model.model.get_output_embeddings = lambda: hf_model.model.model.text.lm_head
|
||||
|
||||
native_model = hf_model.model.model # MoondreamModel instance
|
||||
|
||||
from torch.nn import functional as F
|
||||
|
||||
from vllm.model_executor.models.moondream3 import reconstruct_from_crops
|
||||
|
||||
# Resolve the placeholder tokens from the tokenizer instead of hard-coding.
|
||||
image_placeholder_ids = moondream_processor.tokenizer.encode(
|
||||
"<image>", add_special_tokens=False
|
||||
)
|
||||
|
||||
def _normalize_tiling(tilings):
|
||||
"""Extract (h, w) tuple from various tiling container formats."""
|
||||
tiling = tilings
|
||||
if isinstance(tiling, torch.Tensor):
|
||||
tiling = tuple(tiling.squeeze().tolist())
|
||||
elif isinstance(tiling, (list, tuple)):
|
||||
t0 = tiling[0]
|
||||
if isinstance(t0, torch.Tensor):
|
||||
tiling = tuple(t0.tolist())
|
||||
elif isinstance(t0, (list, tuple)):
|
||||
tiling = tuple(t0)
|
||||
return tiling
|
||||
|
||||
def _encode_vision(pixel_values, tilings):
|
||||
"""Run preprocessed crops through vision encoder + projection."""
|
||||
device = native_model.device
|
||||
dtype = native_model.vision.pos_emb.dtype
|
||||
config = native_model.config
|
||||
|
||||
pv = pixel_values
|
||||
while pv.dim() > 4:
|
||||
pv = pv.squeeze(0)
|
||||
pv = pv.to(device=device, dtype=dtype)
|
||||
|
||||
features = native_model._vis_enc(pv)
|
||||
grid_size = config.vision.crop_size // config.vision.enc_patch_size
|
||||
global_feat = features[0]
|
||||
|
||||
if features.shape[0] > 1 and tilings is not None:
|
||||
tiling = _normalize_tiling(tilings)
|
||||
local = features[1:].view(-1, grid_size, grid_size, config.vision.enc_dim)
|
||||
reconstructed = reconstruct_from_crops(
|
||||
local,
|
||||
tiling,
|
||||
config.vision.overlap_margin,
|
||||
patch_size=1,
|
||||
)
|
||||
else:
|
||||
reconstructed = global_feat.view(
|
||||
grid_size, grid_size, config.vision.enc_dim
|
||||
)
|
||||
|
||||
return native_model._vis_proj(global_feat, reconstructed)
|
||||
|
||||
def _find_subsequence(seq, subseq):
|
||||
"""Find start index of subseq in seq, or None."""
|
||||
n = len(subseq)
|
||||
for i in range(len(seq) - n + 1):
|
||||
if seq[i : i + n] == subseq:
|
||||
return i
|
||||
return None
|
||||
|
||||
def _generate(
|
||||
self,
|
||||
input_ids=None,
|
||||
pixel_values=None,
|
||||
tilings=None,
|
||||
attention_mask=None,
|
||||
**kwargs,
|
||||
):
|
||||
max_new_tokens = kwargs.get("max_new_tokens", 128)
|
||||
return_dict = kwargs.get("return_dict_in_generate", False)
|
||||
output_hs = kwargs.get("output_hidden_states", False)
|
||||
|
||||
if pixel_values is None:
|
||||
sequences = input_ids
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=() if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
# Processor may return lists; extract the single element.
|
||||
if isinstance(pixel_values, (list, tuple)):
|
||||
pixel_values = pixel_values[0]
|
||||
if (
|
||||
isinstance(tilings, (list, tuple))
|
||||
and tilings
|
||||
and not isinstance(tilings[0], int)
|
||||
):
|
||||
tilings = tilings[0]
|
||||
|
||||
hf_model.model._setup_caches()
|
||||
native_model.use_flex_decoding = False
|
||||
|
||||
device = native_model.device
|
||||
config = native_model.config
|
||||
|
||||
with torch.inference_mode():
|
||||
for block in native_model.text.blocks:
|
||||
block.kv_cache.k_cache.zero_()
|
||||
block.kv_cache.v_cache.zero_()
|
||||
|
||||
img_emb = _encode_vision(pixel_values, tilings)
|
||||
|
||||
bos_emb = F.embedding(
|
||||
torch.tensor([[config.tokenizer.bos_id]], device=device),
|
||||
native_model.text.wte,
|
||||
)
|
||||
img_input = torch.cat([bos_emb, img_emb.unsqueeze(0)], dim=1)
|
||||
prefix_len = img_input.size(1)
|
||||
|
||||
mask = native_model.attn_mask[:, :, :prefix_len, :]
|
||||
pos_ids = torch.arange(prefix_len, dtype=torch.long, device=device)
|
||||
native_model._prefill(img_input, mask, pos_ids, None)
|
||||
|
||||
ids = input_ids.squeeze(0).tolist()
|
||||
img_start = _find_subsequence(ids, image_placeholder_ids)
|
||||
|
||||
if img_start is None:
|
||||
sequences = input_ids
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=() if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
prompt_tokens = ids[img_start + len(image_placeholder_ids) :]
|
||||
|
||||
if not prompt_tokens:
|
||||
sequences = input_ids
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=() if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
prompt_tensor = torch.tensor([prompt_tokens], device=device)
|
||||
prompt_emb = F.embedding(prompt_tensor, native_model.text.wte)
|
||||
prompt_len = prompt_emb.size(1)
|
||||
|
||||
mask = native_model.attn_mask[:, :, prefix_len : prefix_len + prompt_len, :]
|
||||
pos_ids = torch.arange(
|
||||
prefix_len,
|
||||
prefix_len + prompt_len,
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
)
|
||||
hidden = native_model._prefill(prompt_emb, mask, pos_ids, None)
|
||||
pos = prefix_len + prompt_len
|
||||
|
||||
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
|
||||
logits = native_model.text.lm_head(hidden_last.squeeze(1))
|
||||
|
||||
generated = []
|
||||
all_hidden_states = []
|
||||
# Record the hidden state that predicted each generated token.
|
||||
prev_hs = hidden_last
|
||||
for _ in range(max_new_tokens):
|
||||
next_token = logits.argmax(dim=-1).item()
|
||||
if next_token == 0:
|
||||
break
|
||||
generated.append(next_token)
|
||||
if output_hs:
|
||||
all_hidden_states.append((prev_hs,))
|
||||
|
||||
next_emb = F.embedding(
|
||||
torch.tensor([[next_token]], device=device),
|
||||
native_model.text.wte,
|
||||
)
|
||||
mask = native_model.attn_mask[:, :, pos : pos + 1, :]
|
||||
pos_ids_step = torch.tensor([pos], dtype=torch.long, device=device)
|
||||
hidden = native_model._prefill(next_emb, mask, pos_ids_step, None)
|
||||
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
|
||||
prev_hs = hidden_last
|
||||
logits = native_model.text.lm_head(hidden_last.squeeze(1))
|
||||
pos += 1
|
||||
|
||||
result_ids = ids + generated
|
||||
sequences = torch.tensor([result_ids], device=device)
|
||||
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=tuple(all_hidden_states) if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
|
||||
return hf_model
|
||||
|
||||
@@ -15,7 +15,6 @@ from safetensors import safe_open
|
||||
from safetensors.torch import load, load_file, save_file
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
@@ -31,10 +30,6 @@ from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
from vllm.models.qwen4_exp.amd import ple_layer as amd_ple_layer
|
||||
from vllm.models.qwen4_exp.amd.ple_layer import (
|
||||
Qwen4ExpPLELayer as Qwen4ExpPLELayerAMD,
|
||||
)
|
||||
from vllm.models.qwen4_exp.common.ple import (
|
||||
PLEShardOverlap,
|
||||
compute_ple_shard_overlap,
|
||||
@@ -51,7 +46,6 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import weak_ref_tensor
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionMetadata,
|
||||
@@ -732,10 +726,6 @@ def test_ple_embedding_respects_modelopt_exclusion() -> None:
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||
@@ -2289,162 +2279,6 @@ def _make_conv_metadata(
|
||||
return metadata, token_offset
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused conv needs CUDA")
|
||||
@pytest.mark.parametrize("state_layout", ["SD", "DS"])
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
pytest.param(
|
||||
_ConvBatchCase(num_decodes=1, channels=512),
|
||||
id="decode-single",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(
|
||||
num_decodes=33,
|
||||
channels=2048,
|
||||
kernel_size=5,
|
||||
dilation=1,
|
||||
),
|
||||
id="decode-padded",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(prefill_query_lens=(37, 0, 5, 128)),
|
||||
id="prefill",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(
|
||||
prefill_query_lens=(16, 676),
|
||||
channels=10240,
|
||||
state_index_stride=4,
|
||||
include_null_state=False,
|
||||
),
|
||||
id="prefill-strided-state-indices",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(
|
||||
spec_query_lens=(1, 4, 4, 0),
|
||||
num_accepted=(1, 2, 4, 1),
|
||||
spec_query_len=4,
|
||||
graph_padding=4,
|
||||
),
|
||||
id="spec-graph-padded",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(
|
||||
spec_query_lens=(3, 4),
|
||||
num_accepted=(2, 4),
|
||||
num_decodes=1,
|
||||
prefill_query_lens=(5,),
|
||||
spec_query_len=4,
|
||||
),
|
||||
id="mixed",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(
|
||||
spec_query_lens=(1, 2),
|
||||
num_accepted=(0, 1),
|
||||
num_decodes=2,
|
||||
prefill_query_lens=(0, 8, 3),
|
||||
channels=640,
|
||||
kernel_size=3,
|
||||
dilation=2,
|
||||
spec_query_len=3,
|
||||
),
|
||||
id="mixed-varied",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("layer_cls", "fuse_residual"),
|
||||
[
|
||||
pytest.param(Qwen4ExpPLELayer, True, id="nvidia-fused"),
|
||||
pytest.param(Qwen4ExpPLELayer, False, id="nvidia-unfused"),
|
||||
pytest.param(Qwen4ExpPLELayerAMD, True, id="amd-fused"),
|
||||
],
|
||||
)
|
||||
def test_fused_conv_correctness(
|
||||
case: _ConvBatchCase,
|
||||
state_layout: str,
|
||||
layer_cls: type[nn.Module],
|
||||
fuse_residual: bool,
|
||||
) -> None:
|
||||
device = torch.device("cuda")
|
||||
metadata, num_real_tokens = _make_conv_metadata(case, device)
|
||||
if case.state_index_stride > 1:
|
||||
assert metadata.state_indices_tensor.stride(0) == case.state_index_stride
|
||||
# layer_cls is type[nn.Module], so layer_cls.__new__ is type.__new__.
|
||||
module = object.__new__(layer_cls)
|
||||
nn.Module.__init__(module)
|
||||
module.conv_state_len = (case.kernel_size - 1) * case.dilation
|
||||
module.short_conv_dilation = case.dilation
|
||||
|
||||
rng, state_reference, conv_state, weights = _make_conv_case(
|
||||
device,
|
||||
seed=num_real_tokens + case.channels,
|
||||
channels=case.channels,
|
||||
kernel_size=case.kernel_size,
|
||||
dilation=case.dilation,
|
||||
state_layout=state_layout,
|
||||
spec_query_len=case.spec_query_len,
|
||||
)
|
||||
inputs = torch.randn(
|
||||
metadata.num_actual_tokens,
|
||||
case.channels,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
residual = torch.randn(
|
||||
inputs.shape,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
outer_residual = torch.randn(
|
||||
inputs.shape,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
null_state = conv_state[NULL_BLOCK_ID].clone()
|
||||
residual_kernel = residual.clone()
|
||||
# The unfused kernel must overwrite the buffer without reading its contents.
|
||||
residual_reference = (
|
||||
residual.clone() if fuse_residual else torch.zeros_like(residual)
|
||||
)
|
||||
|
||||
module._short_conv_dilated_dispatch(
|
||||
inputs=inputs,
|
||||
residual=residual_kernel,
|
||||
outer_residual=outer_residual if fuse_residual else None,
|
||||
metadata=metadata,
|
||||
conv_state=conv_state,
|
||||
conv_weights=weights,
|
||||
)
|
||||
_short_conv_dilated_dispatch_pytorch(
|
||||
inputs=inputs,
|
||||
residual=residual_reference,
|
||||
metadata=metadata,
|
||||
conv_state=state_reference,
|
||||
conv_weights=weights,
|
||||
conv_state_len=module.conv_state_len,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
if fuse_residual:
|
||||
residual_reference = outer_residual + residual_reference
|
||||
|
||||
assert torch.equal(
|
||||
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
|
||||
)
|
||||
assert torch.equal(conv_state, state_reference)
|
||||
assert torch.equal(conv_state[NULL_BLOCK_ID], null_state)
|
||||
if case.graph_padding:
|
||||
assert torch.equal(
|
||||
residual_kernel[num_real_tokens:],
|
||||
residual_reference[num_real_tokens:],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused gate needs CUDA")
|
||||
@pytest.mark.parametrize(("num_tokens", "strided_kv"), [(1, False), (64, True)])
|
||||
def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None:
|
||||
@@ -2509,273 +2343,3 @@ def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None:
|
||||
expected_normed = grouped_norm(expected_gated, norm_conv)
|
||||
assert torch.equal(gated, expected_gated)
|
||||
torch.testing.assert_close(normed, expected_normed, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def _build_amd_ngram_embedding(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
device: str,
|
||||
cpu_offload: bool,
|
||||
fp8_checkpoint: bool,
|
||||
) -> tuple[amd_ple_layer.Qwen4ExpNGramEmbedding, torch.Tensor]:
|
||||
"""Construct and load a small AMD embedding through its checkpoint loader."""
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
amd_ple_layer,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(engram_config=SimpleNamespace(cpu_offload=cpu_offload)),
|
||||
)
|
||||
config = Qwen4ExpTextConfig(
|
||||
ngram_size=3,
|
||||
heads_per_ngram=1,
|
||||
ngram_vocab_size_base=5,
|
||||
make_ngram_vocab_size_divisible_by=8,
|
||||
split_ngram_parts=2,
|
||||
ple_embed_dim=6,
|
||||
eos_token_id=0,
|
||||
vocab_size=64,
|
||||
)
|
||||
with torch.device(device):
|
||||
module = amd_ple_layer.Qwen4ExpNGramEmbedding(
|
||||
config,
|
||||
embedding_dim=config.ple_embed_dim,
|
||||
ple_dense_layer_id=0,
|
||||
max_total_tokens=4,
|
||||
prefix="test.ple_embedding",
|
||||
layer_name="test.ple",
|
||||
quant_config=(
|
||||
Fp8Config(is_checkpoint_fp8_serialized=True) if fp8_checkpoint else None
|
||||
),
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
# Only the outer owner is a stub; the embedding constructor and loader run.
|
||||
layer = Qwen4ExpPLELayerAMD.__new__(Qwen4ExpPLELayerAMD)
|
||||
nn.Module.__init__(layer)
|
||||
layer.ple_embedding = module
|
||||
|
||||
monkeypatch.setattr(
|
||||
amd_ple_layer,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(no_compile_layers={module.layer_name: layer}),
|
||||
)
|
||||
|
||||
embedding = module.ngram_embedding
|
||||
loaded_weight = (
|
||||
torch.arange(embedding.org_vocab_size * embedding.embedding_dim)
|
||||
.reshape(embedding.org_vocab_size, embedding.embedding_dim)
|
||||
.to(embedding.weight.dtype)
|
||||
)
|
||||
shard_size = (embedding.org_vocab_size + 1) // 2
|
||||
weights = [
|
||||
(f"ngram_embedding.shard_{i}.weight", shard)
|
||||
for i, shard in enumerate(loaded_weight.split(shard_size))
|
||||
]
|
||||
expected_loaded = {"ngram_embedding.weight"}
|
||||
if fp8_checkpoint:
|
||||
weights.append(("ngram_embedding.weight_scale", torch.tensor([0.25])))
|
||||
expected_loaded.add("ngram_embedding.weight_scale")
|
||||
assert module.load_weights(weights) == expected_loaded
|
||||
embedding.quant_method.process_weights_after_loading(embedding)
|
||||
return module, loaded_weight
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"device,cpu_offload",
|
||||
[
|
||||
pytest.param("cpu", False, id="cpu-resident"),
|
||||
pytest.param(
|
||||
"cuda:0",
|
||||
False,
|
||||
id="gpu-resident",
|
||||
marks=pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA or ROCm"
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
"cuda:0",
|
||||
True,
|
||||
id="gpu-offloaded",
|
||||
marks=pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA or ROCm"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
device: str,
|
||||
cpu_offload: bool,
|
||||
) -> None:
|
||||
"""AMD checkpoint loading must accept and apply the global FP8 scale."""
|
||||
module, loaded_weight = _build_amd_ngram_embedding(
|
||||
monkeypatch, device=device, cpu_offload=cpu_offload, fp8_checkpoint=True
|
||||
)
|
||||
embedding = module.ngram_embedding
|
||||
assert embedding.weight.dtype == torch.float8_e4m3fn
|
||||
assert embedding.weight.device == torch.device("cpu" if cpu_offload else device)
|
||||
if cpu_offload:
|
||||
assert embedding.weight.is_pinned()
|
||||
assert embedding.weight_scale.device == torch.device(device)
|
||||
torch.testing.assert_close(embedding.weight.float().cpu(), loaded_weight.float())
|
||||
dequantized = embedding.dequantize(embedding.weight.to(device), torch.bfloat16)
|
||||
torch.testing.assert_close(
|
||||
dequantized.cpu(), loaded_weight.to(torch.bfloat16) * 0.25, rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
|
||||
def test_amd_pinned_embedding_prefetch_machinery_is_lazy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The pinned embedding must not allocate the prefetch stream/buffer in init.
|
||||
|
||||
Paths that never prefetch (AMD uses the synchronous pinned-lookup op)
|
||||
keep the stream/buffer unallocated for the process lifetime. NVIDIA's
|
||||
first ``start_prefetch`` — the eager profile run, always before cudagraph
|
||||
capture — allocates them lazily.
|
||||
"""
|
||||
module, _ = _build_amd_ngram_embedding(
|
||||
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=False
|
||||
)
|
||||
embedding = module.ngram_embedding
|
||||
assert embedding._prefetch_buffer is None
|
||||
assert embedding._prefetch_stream is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
|
||||
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
|
||||
def test_amd_pinned_embedding_output_written_under_compile(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_checkpoint: bool,
|
||||
) -> None:
|
||||
"""The AMD pinned lookup op must survive aot_autograd/Inductor.
|
||||
|
||||
The op writes via the declared-mutated ``output``. Without
|
||||
``mutates_args=["output"]`` the call is dead-code-eliminated, the UVA lookup
|
||||
never runs, and the returned ``output`` stays untouched.
|
||||
"""
|
||||
module, loaded_weight = _build_amd_ngram_embedding(
|
||||
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint
|
||||
)
|
||||
|
||||
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
output = torch.zeros(
|
||||
ngram_ids.shape[0],
|
||||
module.embedding_dim,
|
||||
dtype=module.ngram_embedding.weight.dtype,
|
||||
device="cuda:0",
|
||||
)
|
||||
|
||||
def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
|
||||
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned(
|
||||
ids, out, module.layer_name
|
||||
)
|
||||
return out
|
||||
|
||||
torch.compile(lookup, fullgraph=True)(ngram_ids, output)
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
|
||||
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
|
||||
def test_amd_pinned_embedding_output_written_under_cudagraph_capture(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_checkpoint: bool,
|
||||
) -> None:
|
||||
"""The AMD pinned lookup op must be capture-safe.
|
||||
|
||||
It launches a single kernel on the current stream, so a cudagraph capture
|
||||
reproduces the lookup into ``output`` without cross-stream work.
|
||||
"""
|
||||
module, loaded_weight = _build_amd_ngram_embedding(
|
||||
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint
|
||||
)
|
||||
|
||||
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
output = torch.zeros(
|
||||
ngram_ids.shape[0],
|
||||
module.embedding_dim,
|
||||
dtype=module.ngram_embedding.weight.dtype,
|
||||
device="cuda:0",
|
||||
)
|
||||
|
||||
def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
|
||||
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned(
|
||||
ids, out, module.layer_name
|
||||
)
|
||||
return out
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
warmup_stream = torch.cuda.Stream(device="cuda:0")
|
||||
warmup_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(warmup_stream):
|
||||
for _ in range(3):
|
||||
lookup(ngram_ids, output)
|
||||
torch.cuda.current_stream().wait_stream(warmup_stream)
|
||||
|
||||
with torch.cuda.graph(graph):
|
||||
lookup(ngram_ids, output)
|
||||
for _ in range(2):
|
||||
ngram_ids.copy_(ngram_ids.flip(-1) + 3)
|
||||
output.fill_(float("nan"))
|
||||
graph.replay()
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="only the ROCm backend runs the n-gram embedding under torch.compile",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("query_lens", "contexts"),
|
||||
[([3, 1], [[5, 6], [0, 7]]), ([1, 0, 2], [[1, 2], [3, 4], [0, 0]])],
|
||||
ids=["two-requests", "empty-request"],
|
||||
)
|
||||
def test_amd_ngram_embedding_matches_reference_under_compile(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
query_lens: list[int],
|
||||
contexts: list[list[int]],
|
||||
) -> None:
|
||||
"""The AMD n-gram ID op must run under Dynamo with a symbolic request count."""
|
||||
module, loaded_weight = _build_amd_ngram_embedding(
|
||||
monkeypatch, device="cuda:0", cpu_offload=False, fp8_checkpoint=False
|
||||
)
|
||||
device = torch.device("cuda:0")
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *accumulate(query_lens)], dtype=torch.int32, device=device
|
||||
)
|
||||
input_ids = torch.tensor(
|
||||
[9, 0, 11, 12][: sum(query_lens)], dtype=torch.int32, device=device
|
||||
)
|
||||
ngram_context = torch.tensor(contexts, dtype=torch.int32, device=device)
|
||||
|
||||
compiled = torch.compile(module, fullgraph=True, dynamic=True)
|
||||
output = compiled(input_ids, query_start_loc, ngram_context)
|
||||
|
||||
ngram_ids = _reference_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
module.layer_multipliers,
|
||||
module.ngram_heads_vocab_sizes,
|
||||
module.ngram_heads_offsets,
|
||||
module.eos_token_id,
|
||||
module.heads_per_ngram,
|
||||
)
|
||||
expected = loaded_weight[ngram_ids.cpu()].flatten(-2).to(device)
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,181 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Transformers modeling backend's MLA fuser.
|
||||
|
||||
The fuser must discover every MLA submodule structurally (never by assuming the
|
||||
Transformers attribute names) and, when the query is low-rank, merge the checkpoint's
|
||||
separate `q_a_proj`/`kv_a_proj_with_mqa` weights into the single fused down-projection.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers import MLAFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import trace
|
||||
|
||||
_FUSED_QKV_A_PROJ = MLAFuser.merged_name
|
||||
|
||||
|
||||
def _attention(q_lora_rank: int | None):
|
||||
"""A meta `DeepseekV2Attention` in the requested q_lora configuration."""
|
||||
pytest.importorskip("transformers.models.deepseek_v2.modeling_deepseek_v2")
|
||||
from transformers.models.deepseek_v2.configuration_deepseek_v2 import (
|
||||
DeepseekV2Config,
|
||||
)
|
||||
from transformers.models.deepseek_v2.modeling_deepseek_v2 import (
|
||||
DeepseekV2Attention,
|
||||
)
|
||||
|
||||
cfg = DeepseekV2Config(
|
||||
hidden_size=256,
|
||||
num_attention_heads=16,
|
||||
kv_lora_rank=128,
|
||||
qk_rope_head_dim=32,
|
||||
qk_nope_head_dim=32,
|
||||
v_head_dim=32,
|
||||
q_lora_rank=q_lora_rank,
|
||||
num_hidden_layers=1,
|
||||
)
|
||||
with torch.device("meta"):
|
||||
return DeepseekV2Attention(cfg, layer_idx=0)
|
||||
|
||||
|
||||
def _match(q_lora_rank: int | None) -> MLAFuser | None:
|
||||
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
|
||||
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
|
||||
attn = _attention(q_lora_rank)
|
||||
return MLAFuser.match(trace(attn), attn)
|
||||
|
||||
|
||||
def test_discovers_modules_without_q_lora():
|
||||
fuser = _match(q_lora_rank=None)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
assert fuser.q_proj_name == "q_proj"
|
||||
assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa"
|
||||
assert fuser.kv_a_layernorm_name == "kv_a_layernorm"
|
||||
assert fuser.kv_b_proj_name == "kv_b_proj"
|
||||
assert fuser.o_proj_name == "o_proj"
|
||||
assert fuser.q_a_proj_name is None
|
||||
# Nothing is stacked without a query LoRA.
|
||||
assert fuser.packed_modules_mapping == {}
|
||||
assert fuser.orig_to_new_stacked("model.layers.0.self_attn") == {}
|
||||
|
||||
|
||||
def test_discovers_modules_with_q_lora():
|
||||
fuser = _match(q_lora_rank=64)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
assert fuser.q_a_proj_name == "q_a_proj"
|
||||
assert fuser.q_a_layernorm_name == "q_a_layernorm"
|
||||
assert fuser.q_b_proj_name == "q_b_proj"
|
||||
assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa"
|
||||
assert fuser.kv_a_layernorm_name == "kv_a_layernorm"
|
||||
assert fuser.kv_b_proj_name == "kv_b_proj"
|
||||
assert fuser.o_proj_name == "o_proj"
|
||||
assert fuser.q_proj_name is None
|
||||
|
||||
|
||||
def test_q_lora_stacks_qkv_a_proj():
|
||||
"""The MLA layer reads `q_a_proj` and `kv_a_proj_with_mqa` fused into one
|
||||
down-projection, so both checkpoint weights must remap into it."""
|
||||
fuser = _match(q_lora_rank=64)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
prefix = "model.layers.0.self_attn"
|
||||
merged = f"{prefix}.{_FUSED_QKV_A_PROJ}"
|
||||
assert fuser.packed_modules_mapping == {
|
||||
_FUSED_QKV_A_PROJ: ["q_a_proj", "kv_a_proj_with_mqa"]
|
||||
}
|
||||
assert fuser.orig_to_new_stacked(prefix) == {
|
||||
f"{prefix}.q_a_proj": (merged, 0),
|
||||
f"{prefix}.kv_a_proj_with_mqa": (merged, 1),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("q_lora_rank", [64, None])
|
||||
def test_update_forward_rewrites_real_attention(q_lora_rank):
|
||||
"""The source rewrite must survive the input-stability check.
|
||||
|
||||
The q-LoRA path hoists the merged down-projection to a top-level statement
|
||||
while its calls sit inside the `q_lora_rank` branches, so the region it
|
||||
spans is wide; a check that over-approximates aliasing would silently refuse
|
||||
here and cost the fusion rather than fail loudly."""
|
||||
fuser = _match(q_lora_rank)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
fuser.update_forward(_attention(q_lora_rank))
|
||||
names = set(fuser.fused_forward.__code__.co_names)
|
||||
if fuser.q_a_proj_name is not None:
|
||||
assert _FUSED_QKV_A_PROJ in names
|
||||
assert not {"q_a_proj", "kv_a_proj_with_mqa"} & names
|
||||
|
||||
|
||||
class _Norm(nn.Module):
|
||||
"""A real RMSNorm computation: `match` verifies chain norms via `RMSNormFuser`."""
|
||||
|
||||
def __init__(self, n: int):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(n))
|
||||
|
||||
def forward(self, x):
|
||||
variance = x.pow(2).mean(-1, keepdim=True)
|
||||
return self.weight * (x * torch.rsqrt(variance + 1e-6))
|
||||
|
||||
|
||||
class RenamedMLA(nn.Module):
|
||||
"""An MLA-shaped attention whose children have non-standard names, proving
|
||||
discovery is by structure and not attribute name."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
heads, kv_lora, rope, nope, v, hidden = 4, 32, 8, 8, 16, 64
|
||||
self.alpha = nn.Linear(hidden, heads * (nope + rope)) # q_proj
|
||||
self.beta = nn.Linear(hidden, kv_lora + rope) # kv_a_proj_with_mqa
|
||||
self.gamma = _Norm(kv_lora) # kv_a_layernorm
|
||||
self.delta = nn.Linear(kv_lora, heads * (nope + v)) # kv_b_proj
|
||||
self.omega = nn.Linear(heads * v, hidden) # o_proj (uncalled)
|
||||
self.kv_lora, self.rope = kv_lora, rope
|
||||
|
||||
def forward(self, hidden_states):
|
||||
q = self.alpha(hidden_states)
|
||||
kv_lora, k_pe = torch.split(
|
||||
self.beta(hidden_states), [self.kv_lora, self.rope], dim=-1
|
||||
)
|
||||
expanded = self.delta(self.gamma(kv_lora))
|
||||
# Stand-in for the attention interface; `match` finds `o_proj` (omega) from
|
||||
# the source as the Linear producing the returned value.
|
||||
attn_output = expanded.sum() + q.sum() + k_pe.sum()
|
||||
attn_output = self.omega(attn_output)
|
||||
return attn_output
|
||||
|
||||
|
||||
def test_discovers_modules_under_arbitrary_names():
|
||||
"""Discovery is purely structural: `RenamedMLA` gives its children non-standard
|
||||
names, and `match` still locates each projection by dataflow."""
|
||||
with torch.device("meta"):
|
||||
module = RenamedMLA()
|
||||
fuser = MLAFuser.match(trace(module), module)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
assert fuser.q_a_proj_name is None
|
||||
assert fuser.q_proj_name == "alpha"
|
||||
assert fuser.kv_a_proj_name == "beta"
|
||||
assert fuser.kv_a_layernorm_name == "gamma"
|
||||
assert fuser.kv_b_proj_name == "delta"
|
||||
assert fuser.o_proj_name == "omega"
|
||||
|
||||
|
||||
class GLU(nn.Module):
|
||||
"""A gated MLP: matches the GLU fuser, never MLA."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.gate = nn.Linear(16, 16)
|
||||
self.up = nn.Linear(16, 16)
|
||||
|
||||
def forward(self, x):
|
||||
return self.up(F.silu(self.gate(x)))
|
||||
|
||||
|
||||
def test_non_mla_is_not_matched():
|
||||
with torch.device("meta"):
|
||||
assert get_fuser(GLU(), MLAFuser) is None
|
||||
@@ -1,611 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Transformers modeling backend's MoE fuser."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.transformers.fusers import MoEBlockFuser
|
||||
|
||||
from .test_linear import GLUMLP
|
||||
|
||||
|
||||
class TopKRouter(nn.Module):
|
||||
"""HF v5 top-k router: `linear -> softmax -> topk (-> renorm)`."""
|
||||
|
||||
def __init__(self, num_experts=8, hidden=16, top_k=2, sigmoid=False):
|
||||
super().__init__()
|
||||
self.top_k = top_k
|
||||
self.sigmoid = sigmoid
|
||||
self.weight = nn.Parameter(torch.zeros(num_experts, hidden))
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = torch.sigmoid(logits) if self.sigmoid else F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
value = value / value.sum(dim=-1, keepdim=True)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class SelectionScoredRouter(TopKRouter):
|
||||
"""Scores the selected logits only (GraniteMoE/Aria style), in fp32."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight).float()
|
||||
value, index = torch.topk(logits, self.top_k, dim=-1)
|
||||
value = (
|
||||
torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1)
|
||||
).type_as(hidden_states)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class EpsilonRenormRouter(TopKRouter):
|
||||
"""Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = torch.sigmoid(logits)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
denominator = value.sum(dim=-1, keepdim=True) + 1e-20
|
||||
value /= denominator
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class UntraceableRouter(TopKRouter):
|
||||
"""Scores and selects, then loses the trace before the renormalization."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
if value.sum() > 0: # data-dependent branch: tracing stops here
|
||||
value = value / value.sum(dim=-1, keepdim=True)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class ScaledRouter(TopKRouter):
|
||||
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
value = value * 16.0
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class Fp32Router(TopKRouter):
|
||||
"""Router that computes its logits in fp32 (DeepSeek/GLM style)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(
|
||||
hidden_states.type(torch.float32), self.weight.type(torch.float32)
|
||||
)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
return logits, value.to(hidden_states.dtype), index
|
||||
|
||||
|
||||
class GroupedRouter(TopKRouter):
|
||||
"""Group-limited router (DeepSeek `group_limited_greedy`), scaled weights."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
group_scores = scores.view(-1, 4, 2).max(dim=-1).values
|
||||
group_idx = torch.topk(group_scores, k=2, dim=-1)[1]
|
||||
group_mask = torch.zeros_like(group_scores)
|
||||
group_mask.scatter_(1, group_idx, 1)
|
||||
score_mask = group_mask.unsqueeze(-1).expand(-1, 4, 2).reshape(-1, 8)
|
||||
scores = scores.masked_fill(~score_mask.bool(), 0.0)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
value = value * 16.0
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class CorrectionRouter(nn.Module):
|
||||
"""Router with a score-correction bias buffer (DeepSeek-V3 noaux) -> matched."""
|
||||
|
||||
def __init__(self, num_experts=8, hidden=16):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.zeros(num_experts, hidden))
|
||||
self.register_buffer("e_score_correction_bias", torch.zeros(num_experts))
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = torch.sigmoid(logits) + self.e_score_correction_bias
|
||||
_, index = torch.topk(scores, 2, dim=-1)
|
||||
return logits, scores, index
|
||||
|
||||
|
||||
class BiasedRouter(TopKRouter):
|
||||
"""A valid top-k router but not `weight`-only (extra `bias` param) -> declined."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.bias = nn.Parameter(torch.zeros(8))
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight) + self.bias
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class DisconnectedRouter(TopKRouter):
|
||||
"""linear+softmax+top-k present but top-k ignores the logits -> not a router."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
_ = F.softmax(logits, dim=-1) # scored, but not consumed by top-k
|
||||
value, index = torch.topk(hidden_states, self.top_k, dim=-1)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class ExtraInputRouter(TopKRouter):
|
||||
"""Router taking a block-owned correction bias as a second input (HYV3)."""
|
||||
|
||||
def forward(self, hidden_states, bias=0.0):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
_, index = torch.topk(logits.sigmoid() + bias, self.top_k, dim=-1)
|
||||
value = logits.sigmoid().gather(1, index)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class MoEExperts(nn.Module):
|
||||
"""Packed experts (3D weights); only its name (`experts`) matters here."""
|
||||
|
||||
def __init__(self, num_experts=8, hidden=16, inter=32):
|
||||
super().__init__()
|
||||
self.gate_up_proj = nn.Parameter(torch.zeros(num_experts, 2 * inter, hidden))
|
||||
self.down_proj = nn.Parameter(torch.zeros(num_experts, hidden, inter))
|
||||
|
||||
def forward(self, hidden_states, index, weights):
|
||||
return hidden_states
|
||||
|
||||
|
||||
class MoEBlock(nn.Module):
|
||||
"""Single-tensor MoE block (Qwen3-style); subclasses override `_shared`."""
|
||||
|
||||
def __init__(self, router_cls=TopKRouter):
|
||||
super().__init__()
|
||||
self.experts = MoEExperts()
|
||||
self.gate = router_cls()
|
||||
|
||||
def _shared(self, x, logits):
|
||||
"""The term added to the experts' output (none for a plain block)."""
|
||||
return None
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
logits, weights, index = self.gate(x)
|
||||
out = self.experts(x, index, weights)
|
||||
if (shared := self._shared(x, logits)) is not None:
|
||||
out = out + shared
|
||||
return out.reshape(hidden_states.shape)
|
||||
|
||||
|
||||
class MoEBlockNoShared(MoEBlock):
|
||||
"""No shared-expert child but a gate-derived add -> declined, not misread."""
|
||||
|
||||
def _shared(self, x, logits):
|
||||
return logits.sum()
|
||||
|
||||
|
||||
class MoEBlockShared(MoEBlock):
|
||||
"""A block with a shared expert and its sigmoid gate (Qwen2-style)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.shared_expert = GLUMLP()
|
||||
self.shared_expert_gate = nn.Linear(16, 1, bias=False)
|
||||
|
||||
def _shared(self, x, logits):
|
||||
return torch.sigmoid(self.shared_expert_gate(x)) * self.shared_expert(x)
|
||||
|
||||
|
||||
class MoEBlockSharedNoGate(MoEBlock):
|
||||
"""A block with an ungated shared expert -> native, shared passed through."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.shared_expert = GLUMLP()
|
||||
|
||||
def _shared(self, x, logits):
|
||||
return self.shared_expert(x)
|
||||
|
||||
|
||||
class MoEBlockTuple(MoEBlock):
|
||||
"""A tuple-returning block (gpt-oss-style) -> must decline."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights), index
|
||||
|
||||
|
||||
class MoEBlockTupleVar(MoEBlock):
|
||||
"""Returns a name bound to a tuple, not a literal tuple -> must still decline."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
result = self.experts(x, index, weights), index
|
||||
return result
|
||||
|
||||
|
||||
class MoEBlockNestedTupleReturn(MoEBlock):
|
||||
"""Tuple `return` in a nested helper; block returns one tensor -> still fuses."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
def keep(a, b):
|
||||
return a, b
|
||||
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
out, _ = keep(self.experts(x, index, weights), index)
|
||||
return out.reshape(hidden_states.shape)
|
||||
|
||||
|
||||
class PlainMLP(nn.Module):
|
||||
"""A non-GLU FFN: `down(act(up(x)))`, no gating multiply."""
|
||||
|
||||
def __init__(self, hidden: int = 16, inter: int = 32):
|
||||
super().__init__()
|
||||
self.up_proj = nn.Linear(hidden, inter, bias=False)
|
||||
self.down_proj = nn.Linear(inter, hidden, bias=False)
|
||||
self.act_fn = nn.SiLU()
|
||||
|
||||
def forward(self, x):
|
||||
return self.down_proj(self.act_fn(self.up_proj(x)))
|
||||
|
||||
|
||||
class MoEBlockSharedNonGLU(MoEBlock):
|
||||
"""A non-GLU shared expert -> detected by dataflow (no gate/up merge)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.shared_expert = PlainMLP()
|
||||
|
||||
def _shared(self, x, logits):
|
||||
return self.shared_expert(x)
|
||||
|
||||
|
||||
class MoEBlockUnaccounted(MoEBlock):
|
||||
"""A weight-bearing child outside the fused dataflow (pre-router) -> declined."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.extra = nn.Linear(16, 16, bias=False)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = self.extra(hidden_states.reshape(-1, hidden_states.shape[-1]))
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights).reshape(hidden_states.shape)
|
||||
|
||||
|
||||
class BufferScale(nn.Module):
|
||||
"""A stateful child carrying only a buffer (no parameters)."""
|
||||
|
||||
def __init__(self, hidden: int = 16):
|
||||
super().__init__()
|
||||
self.register_buffer("scale", torch.ones(hidden))
|
||||
|
||||
def forward(self, x):
|
||||
return x * self.scale
|
||||
|
||||
|
||||
class MoEBlockUnaccountedBuffer(MoEBlockUnaccounted):
|
||||
"""Like `MoEBlockUnaccounted`, but the extra child holds only a buffer."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.extra = BufferScale()
|
||||
|
||||
|
||||
class MoEBlockExpertsKeywords(MoEBlock):
|
||||
"""Passes the experts' routing by keyword -> still fuses."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, weights=weights, index=index)
|
||||
|
||||
|
||||
class ScaledExperts(MoEExperts):
|
||||
def forward(self, hidden_states, index, weights, scale=1.0):
|
||||
return hidden_states * scale
|
||||
|
||||
|
||||
class MoEBlockExpertsExtraArg(MoEBlock):
|
||||
"""Passes the experts an extra argument the fused forward would drop."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.experts = ScaledExperts()
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights, scale=2.0)
|
||||
|
||||
|
||||
class MoEBlockSharedUnflattened(MoEBlockSharedNoGate):
|
||||
"""Gate and shared expert read the unflattened input (Kolibri-1)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
_, weights, index = self.gate(hidden_states)
|
||||
x = hidden_states.view(-1, hidden_states.shape[-1])
|
||||
out = self.experts(x, index, weights).view(*hidden_states.shape)
|
||||
return out + self.shared_expert(hidden_states)
|
||||
|
||||
|
||||
class MoEBlockSharedAveraged(MoEBlockSharedUnflattened):
|
||||
"""Averages the experts' and shared expert's outputs (Cohere2-MoE)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
return super().forward(hidden_states) / 2
|
||||
|
||||
|
||||
class MoEBlockScaled(MoEBlockSharedNoGate):
|
||||
"""Scales the experts' output by `routed_scaling_factor` (Laguna)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights) * 2.5 + self.shared_expert(x)
|
||||
|
||||
|
||||
class MoEBlockScaledBiasedRouter(MoEBlockScaled):
|
||||
"""An unrecognized router with a scaled experts' output."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.gate = BiasedRouter()
|
||||
|
||||
|
||||
class MoEBlockSharedBiasedRouter(MoEBlockSharedUnflattened):
|
||||
"""An unrecognized router with a shared expert."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.gate = BiasedRouter()
|
||||
|
||||
|
||||
class MoEBlockGateExtraInput(MoEBlock):
|
||||
"""Passes the gate a block-owned bias the fused forward would drop (HYV3)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(ExtraInputRouter)
|
||||
self.register_buffer("e_score_correction_bias", torch.zeros(8))
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x, self.e_score_correction_bias)
|
||||
return self.experts(x, index, weights)
|
||||
|
||||
|
||||
class MoEBlockCastCombine(MoEBlockSharedNoGate):
|
||||
"""Combines the outputs in fp32 (HYV3)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
out = self.experts(x, index, weights).float() + self.shared_expert(x).float()
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
class MoEBlockIdentityLatent(MoEBlockSharedNoGate):
|
||||
"""Wraps the experts in unused (identity) latent projections (NemotronH)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.fc1_latent_proj = nn.Identity()
|
||||
self.fc2_latent_proj = nn.Identity()
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
out = self.fc2_latent_proj(
|
||||
self.experts(self.fc1_latent_proj(x), index, weights)
|
||||
)
|
||||
return out + self.shared_expert(x)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sigmoid", [False, True])
|
||||
def test_moe_fuser_detects_router(sigmoid):
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(lambda: TopKRouter(sigmoid=sigmoid))
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.gate_name == "gate"
|
||||
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
|
||||
assert fuser.shared_name is None and fuser.shared_gate_name is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sigmoid", [False, True])
|
||||
def test_moe_fuser_matches_router_scoring_the_selection(sigmoid):
|
||||
"""Scoring the selection rather than all logits still allows the fuser to match."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid))
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.gate_name == "gate"
|
||||
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
|
||||
assert fuser.renormalize is (not sigmoid)
|
||||
assert fuser.router_dtype == torch.float32
|
||||
|
||||
|
||||
def test_moe_fuser_matches_scaled_router():
|
||||
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(ScaledRouter)
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"router_cls,expected",
|
||||
[
|
||||
(TopKRouter, True), # divides by the sum of the selection
|
||||
(EpsilonRenormRouter, True), # ... with an epsilon in the denominator
|
||||
(ScaledRouter, False), # scales the weights instead of renormalizing
|
||||
(UntraceableRouter, None), # trace stops before it could say
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected):
|
||||
"""The router's graph, not the config, decides whether vLLM renormalizes: it
|
||||
is the code being replaced, and it already bakes in `norm_topk_prob` because
|
||||
the flag is a Python bool that fx resolves at trace time. Absence of the
|
||||
division only means "no" on a complete trace."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(router_cls)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.renormalize is expected
|
||||
|
||||
|
||||
def test_moe_fuser_matches_grouped_router():
|
||||
"""Group masking between the score and the routing top-k still matches: the
|
||||
scorer is anchored on the routing (last) top-k, not the group one."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(GroupedRouter)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.scoring_func == "softmax"
|
||||
|
||||
|
||||
def test_moe_fuser_matches_correction_router():
|
||||
"""A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the
|
||||
rebuilt gate carries it in fp32 for FusedMoE's biased routers."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(CorrectionRouter)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.scoring_func == "sigmoid"
|
||||
|
||||
|
||||
def test_moe_fuser_reads_router_dtype_from_the_gate():
|
||||
"""A router that computes its logits in fp32 must keep routing in fp32 when
|
||||
rebuilt, even though no config field names the dtype. The cast back to the
|
||||
activation dtype after the top-k must not be mistaken for the routing dtype."""
|
||||
with torch.device("meta"):
|
||||
fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts")
|
||||
default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts")
|
||||
assert fp32.router_dtype == torch.float32
|
||||
assert default.router_dtype is None
|
||||
|
||||
|
||||
def test_moe_fuser_detects_shared_experts():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockShared()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
assert fuser.shared_gate_name == "shared_expert_gate"
|
||||
|
||||
|
||||
def test_moe_fuser_shared_without_gate():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockSharedNoGate()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
assert fuser.shared_gate_name is None
|
||||
|
||||
|
||||
def test_moe_fuser_detects_non_glu_shared_expert():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockSharedNonGLU()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
# Recognised by dataflow (added to the experts' output), though not a GLU.
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
assert fuser.shared_gate_name is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"block_cls",
|
||||
[
|
||||
lambda: MoEBlock(BiasedRouter), # router with an unrecognized extra param
|
||||
MoEBlockTuple, # tuple-returning block (e.g. gpt-oss)
|
||||
MoEBlockTupleVar, # tuple returned via a name binding, not a literal
|
||||
MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow
|
||||
MoEBlockUnaccountedBuffer, # buffer-only child outside the fused dataflow
|
||||
MoEBlockExpertsExtraArg, # extra experts argument the fused forward drops
|
||||
MoEBlockNoShared, # gate-derived term added to the experts' output
|
||||
MoEBlockSharedAveraged, # experts' and shared expert's outputs averaged
|
||||
MoEBlockGateExtraInput, # gate input the fused forward drops
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_declines_unsupported(block_cls):
|
||||
with torch.device("meta"):
|
||||
block = block_cls()
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
|
||||
def test_moe_fuser_ignores_nested_returns():
|
||||
"""A tuple `return` inside a nested helper must not decline a block whose own
|
||||
forward returns a single tensor."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockNestedTupleReturn()
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
def test_moe_fuser_router_requires_connected_dataflow():
|
||||
"""A gate with linear + softmax + top-k present but not wired as a router
|
||||
(top-k selects over the input, not the scored logits) is not detected."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(DisconnectedRouter)
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
|
||||
def test_moe_fuser_unrecognized_router_moves_shared_expert():
|
||||
"""Without vLLM routing, the shared expert still moves to vLLM."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockSharedBiasedRouter()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert fuser.scoring_func is None
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("block_cls", "scoring_func"),
|
||||
[
|
||||
(MoEBlockSharedUnflattened, None), # routing falls back to Transformers
|
||||
(MoEBlock, "softmax"), # no shared output to divide
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_divided_shared_output(block_cls, scoring_func):
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(block_cls(), "experts", divides_shared_output=True)
|
||||
assert fuser.scoring_func == scoring_func
|
||||
|
||||
|
||||
def test_moe_fuser_scaled_output_needs_vllm_routing():
|
||||
"""Only vLLM routing applies the routed scale, so nothing else moves without it."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockScaledBiasedRouter()
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
|
||||
def test_moe_fuser_matches_experts_called_by_keyword():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockExpertsKeywords()
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
def test_moe_fuser_scaled_output_with_vllm_routing():
|
||||
"""A scaled block fuses because vLLM routing applies `routed_scaling_factor`."""
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(MoEBlockScaled(), "experts")
|
||||
assert fuser.scoring_func == "softmax"
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_cls", [MoEBlockCastCombine, MoEBlockIdentityLatent])
|
||||
def test_moe_fuser_looks_through_casts_and_identity(block_cls):
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(block_cls(), "experts")
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
@@ -1,409 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Transformers modeling backend's RMSNorm fuser."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers import RMSNormFuser
|
||||
|
||||
|
||||
class RMSNorm(nn.Module):
|
||||
"""The canonical HF RMSNorm: `weight * x * rsqrt(mean(x**2) + eps)`."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-5, weight: bool = True):
|
||||
super().__init__()
|
||||
if weight:
|
||||
self.weight = nn.Parameter(torch.ones(hidden))
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def _rms(self, x):
|
||||
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon)
|
||||
|
||||
def forward(self, x):
|
||||
return self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
|
||||
|
||||
|
||||
class GemmaRMSNorm(RMSNorm):
|
||||
"""Zero-centered weight: `(1 + weight) * normalized`."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__(hidden, eps)
|
||||
self.weight = nn.Parameter(torch.zeros(hidden))
|
||||
|
||||
def forward(self, x):
|
||||
return (1.0 + self.weight) * self._rms(x.to(torch.float32)).to(x.dtype)
|
||||
|
||||
|
||||
class WeightlessRMSNorm(RMSNorm):
|
||||
"""No scale parameter (e.g. Gemma3n `with_scale=False`)."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__(hidden, eps, weight=False)
|
||||
|
||||
def forward(self, x):
|
||||
return self._rms(x.to(torch.float32)).to(x.dtype)
|
||||
|
||||
|
||||
class Gemma4RMSNorm(RMSNorm):
|
||||
"""`pow(v, -0.5)` where the others write `rsqrt(v)` (HF `Gemma4RMSNorm._norm`).
|
||||
|
||||
Gemma 4 spells the reciprocal square root as `torch.pow(..., -0.5)` to keep
|
||||
Torch and JAX in agreement, and -- unlike Gemma 1-3 -- its weight is *not*
|
||||
zero-centered, so this fuses to a plain `RMSNorm`.
|
||||
"""
|
||||
|
||||
def _rms(self, x):
|
||||
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
|
||||
return x * torch.pow(mean_squared, -0.5)
|
||||
|
||||
def forward(self, x):
|
||||
return (self._rms(x.float()) * self.weight.float()).type_as(x)
|
||||
|
||||
|
||||
class PowOperatorRMSNorm(Gemma4RMSNorm):
|
||||
"""The `v ** -0.5` operator spelling of the same reciprocal square root."""
|
||||
|
||||
def _rms(self, x):
|
||||
return x * (x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon) ** -0.5
|
||||
|
||||
|
||||
class KwargPowRMSNorm(Gemma4RMSNorm):
|
||||
"""The keyword spelling `torch.pow(v, exponent=-0.5)` of the same rsqrt."""
|
||||
|
||||
def _rms(self, x):
|
||||
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
|
||||
return x * torch.pow(mean_squared, exponent=-0.5)
|
||||
|
||||
|
||||
class KwargBasePowRMSNorm(Gemma4RMSNorm):
|
||||
"""The fully functional spelling `torch.pow(input=v, exponent=-0.5)`."""
|
||||
|
||||
def _rms(self, x):
|
||||
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
|
||||
return x * torch.pow(input=mean_squared, exponent=-0.5)
|
||||
|
||||
|
||||
class WeightlessGemma4RMSNorm(Gemma4RMSNorm):
|
||||
"""Gemma 4 with `with_scale=False`: the `pow` spelling and no scale parameter."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__(hidden, eps, weight=False)
|
||||
|
||||
def forward(self, x):
|
||||
return self._rms(x.float()).type_as(x)
|
||||
|
||||
|
||||
class LayerNorm(RMSNorm):
|
||||
"""An RMSNorm not named `*RMSNorm`, keeping the input dtype (no upcast)."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__(hidden, eps)
|
||||
|
||||
def forward(self, x):
|
||||
return self.weight * self._rms(x)
|
||||
|
||||
|
||||
class NamedEpsRMSNorm(nn.Module):
|
||||
"""Holds eps under `attr`, with no `variance_epsilon` to find instead."""
|
||||
|
||||
attr = "eps"
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden))
|
||||
setattr(self, self.attr, eps)
|
||||
|
||||
def forward(self, x):
|
||||
eps = getattr(self, self.attr)
|
||||
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + eps))
|
||||
|
||||
|
||||
class ToleranceRMSNorm(NamedEpsRMSNorm):
|
||||
"""Names eps something other than `eps`, so only its value identifies it."""
|
||||
|
||||
attr = "tolerance"
|
||||
|
||||
|
||||
class LiteralEpsRMSNorm(RMSNorm):
|
||||
"""eps is a literal in the source, and an unrelated attribute happens to
|
||||
hold the same value: matching on the value alone would bind to it."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-5):
|
||||
super().__init__(hidden, eps)
|
||||
self.epsilon_lookalike = 1e-4
|
||||
|
||||
def forward(self, x):
|
||||
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + 1e-4))
|
||||
|
||||
|
||||
class AmbiguousEpsRMSNorm(nn.Module):
|
||||
"""Two attributes hold the eps value, and the forward reads the second, so
|
||||
their order cannot pick it."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden))
|
||||
self.decoy_eps = eps
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def forward(self, x):
|
||||
var = x.pow(2).mean(-1, True)
|
||||
return self.weight * (x * torch.rsqrt(var + self.variance_epsilon))
|
||||
|
||||
|
||||
class NotAnRMSNorm(RMSNorm):
|
||||
"""Mean-subtracting LayerNorm-like math -> not an RMSNorm."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__(hidden, eps)
|
||||
|
||||
def forward(self, x):
|
||||
x = x - x.mean(-1, keepdim=True)
|
||||
variance = x.var(-1, keepdim=True)
|
||||
return self.weight * x / torch.sqrt(variance + self.variance_epsilon)
|
||||
|
||||
|
||||
class GatedRMSNorm(RMSNorm):
|
||||
"""Second input and tail compute -> not an RMSNorm."""
|
||||
|
||||
def forward(self, x, gate=None):
|
||||
normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
|
||||
return normed * F.silu(gate)
|
||||
|
||||
|
||||
class GatedFusedRMSNorm(nn.Module):
|
||||
"""Same as GatedRMSNorm, but built on the fused `rms_norm` op -> not an RMSNorm."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-5):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden))
|
||||
self.eps = eps
|
||||
|
||||
def forward(self, x, gate=None):
|
||||
return F.rms_norm(x, (x.shape[-1],), self.weight, self.eps) * F.silu(gate)
|
||||
|
||||
|
||||
class UntraceableGatedRMSNorm(RMSNorm):
|
||||
"""Tracer can't see tail compute in forward, but still has a second input (gate)."""
|
||||
|
||||
def forward(self, x, gate=None):
|
||||
normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
|
||||
if gate.sum() > 0: # untraceable -> partial graph, no visible tail
|
||||
normed = normed * F.silu(gate)
|
||||
return normed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls,eps,zero_centered",
|
||||
[
|
||||
(RMSNorm, 1e-5, False),
|
||||
(GemmaRMSNorm, 1e-6, True),
|
||||
(WeightlessRMSNorm, 1e-6, False),
|
||||
(Gemma4RMSNorm, 1e-6, False),
|
||||
(PowOperatorRMSNorm, 1e-6, False),
|
||||
(KwargPowRMSNorm, 1e-6, False),
|
||||
(KwargBasePowRMSNorm, 1e-6, False),
|
||||
(WeightlessGemma4RMSNorm, 1e-6, False),
|
||||
(LayerNorm, 1e-6, False),
|
||||
(torch.nn.RMSNorm, 1e-5, False), # fused `F.rms_norm` op
|
||||
],
|
||||
)
|
||||
def test_detects_rms_norm_variants(cls, eps, zero_centered):
|
||||
with torch.device("meta"):
|
||||
fuser = get_fuser(cls(16, eps=eps), RMSNormFuser)
|
||||
assert isinstance(fuser, RMSNormFuser)
|
||||
assert fuser.zero_centered == zero_centered
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [NotAnRMSNorm, nn.LayerNorm, nn.SiLU])
|
||||
def test_non_rms_norms_are_not_matched(cls):
|
||||
with torch.device("meta"):
|
||||
module = cls(16) if cls is nn.LayerNorm else cls()
|
||||
assert get_fuser(module, RMSNormFuser) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls", [GatedRMSNorm, GatedFusedRMSNorm, UntraceableGatedRMSNorm]
|
||||
)
|
||||
def test_gated_rms_norm_is_not_fused(cls):
|
||||
with torch.device("meta"):
|
||||
assert get_fuser(cls(), RMSNormFuser) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls,expected,zero_centered",
|
||||
[
|
||||
(RMSNorm, "RMSNorm", False),
|
||||
(GemmaRMSNorm, "GemmaRMSNorm", True),
|
||||
(WeightlessRMSNorm, "RMSNorm", False),
|
||||
(Gemma4RMSNorm, "RMSNorm", False),
|
||||
(KwargPowRMSNorm, "RMSNorm", False),
|
||||
(KwargBasePowRMSNorm, "RMSNorm", False),
|
||||
(WeightlessGemma4RMSNorm, "RMSNorm", False),
|
||||
],
|
||||
)
|
||||
def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_config):
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as VLLMGemmaRMSNorm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
|
||||
|
||||
with torch.device("meta"):
|
||||
module = cls()
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
built = fuser.fuse(module, "norm", default_vllm_config)
|
||||
from vllm.model_executor.models.transformers.fusers.rms_norm import (
|
||||
TPAwareNormMixin,
|
||||
)
|
||||
|
||||
types_by_name = {"RMSNorm": VLLMRMSNorm, "GemmaRMSNorm": VLLMGemmaRMSNorm}
|
||||
assert isinstance(built, types_by_name[expected])
|
||||
assert isinstance(built, TPAwareNormMixin)
|
||||
assert built.variance_epsilon == module.variance_epsilon
|
||||
# The weight states the hidden size (and is what the TP check reads); a norm
|
||||
# without one has none to state.
|
||||
weight = getattr(module, "weight", None)
|
||||
assert isinstance(built.weight, nn.Parameter) == (weight is not None)
|
||||
assert built.weight.shape[0] == (weight.size(0) if weight is not None else 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls", [Gemma4RMSNorm, PowOperatorRMSNorm, KwargPowRMSNorm, KwargBasePowRMSNorm]
|
||||
)
|
||||
def test_pow_spelled_norm_fuses_to_equivalent_math(cls, default_vllm_config):
|
||||
"""Matching `pow(v, -0.5)` is only sound if it really is the reciprocal square
|
||||
root: the fused norm must reproduce the unfused forward, not just replace it.
|
||||
`pow` and `rsqrt` take different paths to the same value, so compare at float32
|
||||
tolerance rather than bit-for-bit."""
|
||||
torch.manual_seed(0)
|
||||
module = cls(16)
|
||||
with torch.no_grad():
|
||||
module.weight.copy_(torch.randn(16))
|
||||
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
|
||||
with torch.no_grad():
|
||||
built.weight.copy_(module.weight)
|
||||
x = torch.randn(4, 16)
|
||||
torch.testing.assert_close(built(x), module(x), rtol=1e-5, atol=1e-6)
|
||||
|
||||
|
||||
def test_weightless_norm_has_no_hidden_size(default_vllm_config):
|
||||
"""A weightless norm states no hidden size, and the model's (LM) hidden size
|
||||
would be wrong for one on a sub-dimension: Llama 4's `qk_norm` normalizes
|
||||
head_dim, so sizing it at hidden_size made the TP gather reject its input.
|
||||
It fuses with none, and so normalizes the width it is given at any TP size,
|
||||
matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`.
|
||||
"""
|
||||
module = WeightlessRMSNorm(72)
|
||||
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
|
||||
assert built.hidden_size == 0
|
||||
|
||||
built.tp_size = 2 # emulate TP=2 without a real process group
|
||||
x = torch.randn(4, 72)
|
||||
torch.testing.assert_close(built(x), module(x))
|
||||
|
||||
|
||||
def test_fused_rms_norm_op_default_eps(default_vllm_config):
|
||||
"""`torch.nn.RMSNorm` (a single `F.rms_norm` call) matches via the fast path;
|
||||
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`."""
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
|
||||
|
||||
with torch.device("meta"):
|
||||
module = torch.nn.RMSNorm(16) # forward is a single `F.rms_norm` call
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
assert isinstance(fuser, RMSNormFuser)
|
||||
assert not fuser.zero_centered
|
||||
vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32))
|
||||
built = fuser.fuse(module, "norm", vllm_config)
|
||||
assert isinstance(built, VLLMRMSNorm)
|
||||
assert built.variance_epsilon == torch.finfo(torch.float32).eps
|
||||
|
||||
|
||||
def test_eps_is_derived_per_instance(default_vllm_config):
|
||||
"""Two instances of the same norm class with different eps must fuse to their
|
||||
own eps: the type-cached fuser holds only structure, not this value."""
|
||||
with torch.device("meta"):
|
||||
for eps in (1e-5, 1e-6):
|
||||
module = RMSNorm(16, eps=eps)
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
built = fuser.fuse(module, "norm", default_vllm_config)
|
||||
assert built.variance_epsilon == eps
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm])
|
||||
def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
|
||||
"""The eps attribute is identified by holding the traced value, so a norm
|
||||
stays per-instance correct whatever it names it."""
|
||||
with torch.device("meta"):
|
||||
for eps in (1e-5, 1e-6):
|
||||
module = cls(16, eps=eps)
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
assert fuser.eps_attr == cls.attr
|
||||
built = fuser.fuse(module, "norm", default_vllm_config)
|
||||
assert built.variance_epsilon == eps
|
||||
|
||||
|
||||
def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog):
|
||||
"""A literal eps is recognised as coming from no attribute, even when one
|
||||
holds the same value, and is taken from the traced source instead."""
|
||||
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
|
||||
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
|
||||
module = LiteralEpsRMSNorm()
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
built = fuser.fuse(module, "norm", default_vllm_config)
|
||||
assert fuser.eps_attr is None
|
||||
assert built.variance_epsilon == 1e-4
|
||||
assert "does not hold its eps" in caplog.text
|
||||
|
||||
|
||||
def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
|
||||
"""When several attributes hold the eps value, the one the forward actually
|
||||
reads is identified, and they are all left as they were found."""
|
||||
with torch.device("meta"):
|
||||
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
|
||||
before = dict(vars(module))
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
assert fuser.eps_attr == "variance_epsilon"
|
||||
assert vars(module) == before
|
||||
|
||||
other = AmbiguousEpsRMSNorm(16, eps=1e-5)
|
||||
other.decoy_eps = 1e-3
|
||||
built = fuser.fuse(other, "norm", default_vllm_config)
|
||||
assert built.variance_epsilon == 1e-5
|
||||
|
||||
|
||||
def test_fused_norm_is_gather_capable(default_vllm_config):
|
||||
"""Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded
|
||||
projection (OLMoE-style) self-corrects at runtime with no QKV-specific
|
||||
plumbing. A full-width input skips the gather and equals a plain norm."""
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from vllm.model_executor.models.transformers.fusers import rms_norm
|
||||
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(4, 16)
|
||||
for gathered_cls, plain_cls in [
|
||||
(rms_norm.TPAwareRMSNorm, RMSNorm),
|
||||
(rms_norm.TPAwareGemmaRMSNorm, GemmaRMSNorm),
|
||||
]:
|
||||
gathered = gathered_cls(hidden_size=16, eps=1e-6)
|
||||
assert isinstance(gathered, rms_norm.TPAwareNormMixin)
|
||||
plain = plain_cls(hidden_size=16, eps=1e-6)
|
||||
with torch.no_grad():
|
||||
weight = torch.randn(16)
|
||||
gathered.weight.copy_(weight)
|
||||
plain.weight.copy_(weight)
|
||||
torch.testing.assert_close(gathered(x), plain(x))
|
||||
|
||||
|
||||
def test_gathered_norm_rejects_uneven_sharding(default_vllm_config):
|
||||
"""A sharded input (narrower than the full-width weight) that does not tile
|
||||
the weight evenly across ranks is rejected before any collective."""
|
||||
from vllm.model_executor.models.transformers.fusers import rms_norm
|
||||
|
||||
norm = rms_norm.TPAwareRMSNorm(hidden_size=8, eps=1e-6)
|
||||
norm.tp_size = 2 # emulate TP=2 without a real process group
|
||||
with pytest.raises(ValueError, match="does not tile it evenly"):
|
||||
norm(torch.randn(2, 3)) # 3 * 2 != 8
|
||||
@@ -1,86 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for skipping dynamic rope's device sync where it can never rescale."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import HunYuanDenseV1Config, LlamaConfig
|
||||
from transformers.models.hunyuan_v1_dense.modeling_hunyuan_v1_dense import (
|
||||
HunYuanDenseV1RotaryEmbedding,
|
||||
)
|
||||
from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding
|
||||
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
can_enable_torch_compile,
|
||||
disable_noop_dynamic_rope,
|
||||
)
|
||||
|
||||
MAX_POSITION_EMBEDDINGS = 128
|
||||
CONFIG_KWARGS = dict(
|
||||
hidden_size=64,
|
||||
num_attention_heads=4,
|
||||
num_hidden_layers=1,
|
||||
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
|
||||
)
|
||||
DYNAMIC = {"rope_type": "dynamic", "factor": 4.0, "rope_theta": 1e4}
|
||||
|
||||
|
||||
def make_vllm_config(hf_config, max_model_len: int) -> SimpleNamespace:
|
||||
model_config = SimpleNamespace(hf_config=hf_config, max_model_len=max_model_len)
|
||||
return SimpleNamespace(model_config=model_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rope_parameters,max_model_len,compilable",
|
||||
[
|
||||
(DYNAMIC, MAX_POSITION_EMBEDDINGS, True),
|
||||
(DYNAMIC, MAX_POSITION_EMBEDDINGS + 1, False),
|
||||
# The longrope layer type still syncs
|
||||
(
|
||||
{"full_attention": DYNAMIC, "sliding_attention": {"rope_type": "longrope"}},
|
||||
MAX_POSITION_EMBEDDINGS,
|
||||
False,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_can_enable_torch_compile(rope_parameters, max_model_len, compilable):
|
||||
"""Dynamic rope only blocks compilation when it can still rescale."""
|
||||
config = SimpleNamespace(
|
||||
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
|
||||
rope_parameters=rope_parameters,
|
||||
)
|
||||
config.get_text_config = lambda: config
|
||||
vllm_config = make_vllm_config(config, max_model_len)
|
||||
assert can_enable_torch_compile(vllm_config) is compilable
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config,rotary_cls",
|
||||
[
|
||||
(LlamaConfig(**CONFIG_KWARGS, rope_parameters=DYNAMIC), LlamaRotaryEmbedding),
|
||||
# Hunyuan's dynamic init also applies `alpha`, which must be kept
|
||||
(
|
||||
HunYuanDenseV1Config(
|
||||
**CONFIG_KWARGS, head_dim=16, rope_parameters=DYNAMIC | {"alpha": 1e3}
|
||||
),
|
||||
HunYuanDenseV1RotaryEmbedding,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_disable_noop_dynamic_rope(monkeypatch, config, rotary_cls):
|
||||
"""The update no longer syncs and the frequencies are unchanged."""
|
||||
inputs = torch.zeros(1, 8, 64), torch.arange(8)[None, :]
|
||||
rotary_emb = rotary_cls(config)
|
||||
expected = rotary_emb(*inputs)
|
||||
|
||||
vllm_config = make_vllm_config(config, MAX_POSITION_EMBEDDINGS)
|
||||
disable_noop_dynamic_rope(rotary_emb, vllm_config)
|
||||
|
||||
def no_sync(*args, **kwargs):
|
||||
raise AssertionError("dynamic rope update still runs")
|
||||
|
||||
monkeypatch.setattr(torch, "max", no_sync)
|
||||
actual = rotary_emb(*inputs)
|
||||
assert all(torch.equal(e, a) for e, a in zip(expected, actual))
|
||||
@@ -1,80 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Transformers backend's hw-agnostic layer resolution.
|
||||
|
||||
`layers._resolve` imports a layer symbol from
|
||||
`vllm.model_executor.hw_agnostic.layers.<module>` when `VLLM_USE_HW_AGNOSTIC`
|
||||
is set and the symbol exists, and otherwise falls back to
|
||||
`vllm.model_executor.layers.<module>`. These tests pin that contract and the
|
||||
logging that reports which source was used.
|
||||
|
||||
CPU-only: no engine is built. The end-to-end counterpart that serves a model
|
||||
through `vllm_runner` lives in `test_layer_registry_e2e.py` (GPU job).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.transformers import layers
|
||||
|
||||
HW_MODULE = "vllm.model_executor.hw_agnostic.layers.layernorm"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_hw_layernorm(monkeypatch):
|
||||
"""Inject a hw-agnostic `layernorm` module exposing a sentinel `RMSNorm`.
|
||||
|
||||
A `SimpleNamespace` stands in for the module: `importlib.import_module`
|
||||
returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its
|
||||
attributes are set at construction (no `ModuleType` attribute-set that mypy
|
||||
rejects, no constant `setattr` that ruff rejects)."""
|
||||
module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {}))
|
||||
monkeypatch.setitem(sys.modules, HW_MODULE, module)
|
||||
return module
|
||||
|
||||
|
||||
def test_falls_back_to_vllm_when_disabled(monkeypatch, fake_hw_layernorm):
|
||||
"""Disabled: the vLLM class is used even if a hw-agnostic one exists."""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0")
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
|
||||
|
||||
assert layers._resolve("layernorm", "RMSNorm") is VllmRMSNorm
|
||||
|
||||
|
||||
def test_uses_hw_agnostic_when_enabled(monkeypatch, fake_hw_layernorm, caplog_vllm):
|
||||
"""Enabled and available: the hw-agnostic class is used and logged."""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
with caplog_vllm.at_level(logging.INFO):
|
||||
resolved = layers._resolve("layernorm", "RMSNorm")
|
||||
assert resolved is fake_hw_layernorm.RMSNorm
|
||||
assert "Using hw-agnostic layer: RMSNorm" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_falls_back_when_symbol_missing(monkeypatch, caplog_vllm):
|
||||
"""Enabled but the symbol is not ported: fall back to vLLM and warn."""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
# A hw-agnostic module without the requested attribute triggers fallback.
|
||||
empty = types.ModuleType(HW_MODULE)
|
||||
monkeypatch.setitem(sys.modules, HW_MODULE, empty)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
|
||||
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
resolved = layers._resolve("layernorm", "RMSNorm")
|
||||
assert resolved is VllmRMSNorm
|
||||
assert "falling back to default" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_act_and_mul_falls_back_for_unknown_activation(
|
||||
monkeypatch, default_vllm_config
|
||||
):
|
||||
"""An activation with no hw-agnostic equivalent falls back to vLLM's.
|
||||
|
||||
`default_vllm_config` supplies the config context the CustomOp needs.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
from vllm.model_executor.layers.activation import GeluAndMul
|
||||
|
||||
assert isinstance(layers.get_act_and_mul_fn("gelu"), GeluAndMul)
|
||||
Reference in New Issue
Block a user