[Qwen4Exp] Support UVA PLE-offload and Engram tensor parallelism (#54371)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Huang
2026-09-09 07:32:34 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 62f3bf58a5
commit 3116c5d06b
16 changed files with 1712 additions and 495 deletions
+33
View File
@@ -46,6 +46,39 @@ def test_optional_type():
assert optional_type_func("42") == 42
@pytest.mark.parametrize(
"options",
[
["--engram-config", '{"cpu_offload": false, "embedding_across_dp": true}'],
[
"--engram-config.cpu_offload",
"false",
"--engram-config.embedding_across_dp",
"true",
],
],
)
def test_engram_config_cli(options, monkeypatch):
"""CLI settings take precedence over the legacy offload environment."""
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
args = EngineArgs.from_cli_args(parser.parse_args(options))
assert args.engram_config is not None
assert args.engram_config.cpu_offload is False
assert args.engram_config.embedding_across_dp is True
@pytest.mark.parametrize(
"options, provided",
[([], False), (["--engram-config", "{}"], True)],
)
def test_engram_config_cli_optional(options, provided):
"""An explicit empty config must remain distinct from an omitted config."""
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
args = EngineArgs.from_cli_args(parser.parse_args(options))
assert (args.engram_config is not None) == provided
@pytest.mark.parametrize(
("type_hint", "type", "expected"),
[
+327 -21
View File
@@ -13,27 +13,50 @@ from torch.nn import functional as F
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
import vllm.models.qwen4_exp.nvidia.ngram_embedding as ngram_embedding_module
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptMixedPrecisionConfig,
ModelOptNvFp4Config,
)
from vllm.models.qwen4_exp.common.ple import (
PLEShardOverlap,
compute_ple_shard_overlap,
copy_ple_embedding_shard_,
)
from vllm.models.qwen4_exp.nvidia.ple_layer import (
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpNGramEmbedding,
Qwen4ExpPLEDeviceEmbedding,
Qwen4ExpPLEEmbeddingMethod,
Qwen4ExpPLEFp8EmbeddingMethod,
Qwen4ExpPLELayer,
_get_ple_embedding_quant_method,
Qwen4ExpPLEPinnedHostEmbedding,
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionMetadata,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
def _mock_etp_group(
monkeypatch: pytest.MonkeyPatch,
world_size: int = 1,
all_reduce=lambda tensor: tensor,
) -> None:
group = SimpleNamespace(
rank_in_group=0,
world_size=world_size,
all_reduce=all_reduce,
)
monkeypatch.setattr(ngram_embedding_module, "get_etp_group", lambda: group)
monkeypatch.setattr(
ngram_embedding_module,
"get_tp_group",
lambda: SimpleNamespace(world_size=world_size),
)
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
nn.Module.__init__(module)
@@ -171,11 +194,57 @@ def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
)
@pytest.mark.parametrize(("dp_rank", "local_tokens"), [(0, 2), (1, 3)])
def test_etp_lookup_gathers_and_returns_dp_local_rows(
monkeypatch: pytest.MonkeyPatch,
dp_rank: int,
local_tokens: int,
) -> None:
embedding = Qwen4ExpPLEDeviceEmbedding.__new__(Qwen4ExpPLEDeviceEmbedding)
nn.Module.__init__(embedding)
embedding.tp_size = 2
embedding.etp_data_parallel_size = 2
embedding.data_parallel_rank = dp_rank
gathered_ids = torch.tensor([[10], [11], [0], [20], [21], [22]])
group = SimpleNamespace(
rank_in_group=dp_rank,
all_gather=lambda input_ids, dim: gathered_ids,
all_reduce=lambda embeddings: embeddings,
)
embedding.parallel_group = group
forward_context = SimpleNamespace(
dp_metadata=SimpleNamespace(
num_tokens_across_dp_cpu=torch.tensor([2, 3]),
)
)
monkeypatch.setattr(ngram_embedding_module, "get_dp_group", lambda: group)
monkeypatch.setattr(
ngram_embedding_module, "get_forward_context", lambda: forward_context
)
local_ids = gathered_ids[dp_rank * 3 : dp_rank * 3 + local_tokens]
embeddings = torch.arange(12).reshape(6, 2)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
assert torch.equal(input_ids, gathered_ids)
return embeddings
monkeypatch.setattr(
ngram_embedding_module.PLEVocabParallelEmbedding,
"forward",
forward,
)
output = embedding(local_ids)
expected = embeddings[dp_rank * 3 : dp_rank * 3 + local_tokens]
assert torch.equal(output, expected)
def _make_fp8_embedding_layer(
monkeypatch: pytest.MonkeyPatch,
*,
load_scale: bool = True,
) -> embedding_module.VocabParallelEmbedding:
) -> Qwen4ExpPLEDeviceEmbedding:
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
@@ -185,12 +254,13 @@ def _make_fp8_embedding_layer(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
method = Qwen4ExpPLEFp8EmbeddingMethod()
layer = embedding_module.VocabParallelEmbedding(
layer = Qwen4ExpPLEDeviceEmbedding(
3,
2,
params_dtype=torch.bfloat16,
padding_size=1,
quant_method=method,
prefix="test.ple_embedding",
embedding_method=method,
)
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
@@ -228,7 +298,58 @@ def test_ple_fp8_embedding_rejects_missing_global_scale(monkeypatch) -> None:
layer.quant_method.process_weights_after_loading(layer)
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
@pytest.mark.parametrize("requires_device_loading", [False, True])
@pytest.mark.parametrize("load_scale", [False, True])
def test_pinned_ple_post_load_validates_scale_without_staging(
monkeypatch: pytest.MonkeyPatch, requires_device_loading: bool, load_scale: bool
) -> None:
"""Validate scales through the loader while honoring the staging policy."""
from contextlib import nullcontext
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader import utils as loader_utils
source = _make_fp8_embedding_layer(monkeypatch, load_scale=load_scale)
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding.weight = source.weight
embedding.weight_scale = source.weight_scale
embedding.embedding_method = source.embedding_method
embedding.quant_method = source.embedding_method
embedding.tp_rank = source.tp_rank
embedding.tp_size = source.tp_size
assert QuantizeMethodBase.requires_device_loading
assert not embedding.quant_method.requires_device_loading
if requires_device_loading:
embedding.quant_method.requires_device_loading = True
model = nn.Module()
model.embedding = embedding
weight_ptr = embedding.weight.data_ptr()
staged_modules = []
def track_staging(module, target_device):
staged_modules.append(module)
return nullcontext()
monkeypatch.setattr(loader_utils, "device_loading_context", track_staging)
monkeypatch.setattr(loader_utils, "maybe_retie_word_embeddings", lambda *args: None)
monkeypatch.setattr(
loader_utils, "release_device_memory_under_pressure", lambda *args: None
)
config = SimpleNamespace(quantization="fp8")
if load_scale:
loader_utils.process_weights_after_loading(model, config, torch.device("cuda"))
else:
with pytest.raises(ValueError, match="missing its global scale"):
loader_utils.process_weights_after_loading(
model, config, torch.device("cuda")
)
assert embedding.weight.device.type == "cpu"
assert embedding.weight.data_ptr() == weight_ptr
assert staged_modules == ([embedding] if requires_device_loading else [])
def test_ple_fp8_embedding_uses_int8_for_parallel_reduce(monkeypatch) -> None:
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
@@ -251,17 +372,14 @@ def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
reduced_dtypes.append(tensor.dtype)
return tensor.clone()
monkeypatch.setattr(
embedding_module,
"tensor_model_parallel_all_reduce",
all_reduce,
)
layer = embedding_module.VocabParallelEmbedding(
_mock_etp_group(monkeypatch, world_size=2, all_reduce=all_reduce)
layer = Qwen4ExpPLEDeviceEmbedding(
4,
2,
params_dtype=torch.bfloat16,
padding_size=1,
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
prefix="test.ple_embedding",
embedding_method=Qwen4ExpPLEFp8EmbeddingMethod(),
)
layer.weight.data.copy_(
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
@@ -283,12 +401,200 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
weight_block_size=[128, 128],
)
assert isinstance(
_get_ple_embedding_quant_method(quant_config, prefix),
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
quant_config.ignored_layers = [f"{prefix}.shard_0"]
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(None, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
nvfp4_config = ModelOptNvFp4Config(exclude_modules=[])
with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
dynamic_fp8_config = Fp8Config(
is_checkpoint_fp8_serialized=False,
ignored_layers=[],
)
with pytest.raises(NotImplementedError, match="serialized FP8"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(dynamic_fp8_config, prefix)
def test_ple_embedding_respects_modelopt_exclusion() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
prefix,
"float8_e4m3fn",
),
Qwen4ExpPLEFp8EmbeddingMethod,
)
def test_pinned_embedding_forward_finalizes_prefetched_output(
monkeypatch: pytest.MonkeyPatch,
) -> None:
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding._prefetch_buffer = torch.empty(4, 2, 3, dtype=torch.float8_e4m3fn)
embedding._output_dim = 6
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
expected = torch.arange(12).reshape(2, 6).to(torch.float8_e4m3fn)
def finalize_prefetched(
prefetch_output: torch.Tensor,
output: torch.Tensor,
) -> None:
assert prefetch_output is embedding._prefetch_buffer
output.copy_(expected)
monkeypatch.setattr(
embedding,
"_finalize_prefetch",
finalize_prefetched,
)
output = embedding(hidden_states)
assert output.dtype == embedding._prefetch_buffer.dtype
assert torch.equal(output, expected)
def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None:
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding.tp_size = 2
reduced_dtypes = []
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
reduced_dtypes.append(tensor.dtype)
return tensor.clone()
embedding.parallel_group = SimpleNamespace(all_reduce=all_reduce)
embeddings = torch.arange(8).reshape(2, 4).to(torch.float8_e4m3fn)
output = embedding._reduce_etp_embeddings(embeddings)
assert reduced_dtypes == [torch.int8]
assert output.dtype == embeddings.dtype
torch.testing.assert_close(output.float(), embeddings.float())
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_ple_device_embedding_allocates_on_active_device(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
embedding_method = Qwen4ExpPLEUnquantizedEmbeddingMethod()
with torch.device("cuda:0"):
embedding = Qwen4ExpPLEDeviceEmbedding(
4,
3,
params_dtype=torch.bfloat16,
padding_size=1,
prefix="test.ple_embedding",
embedding_method=embedding_method,
)
assert embedding.weight.device == torch.device("cuda:0")
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("fp8_checkpoint", [False, True])
def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
monkeypatch: pytest.MonkeyPatch,
fp8_checkpoint: bool,
) -> None:
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
with torch.device("cuda:0"):
embedding_method = (
Qwen4ExpPLEFp8EmbeddingMethod()
if fp8_checkpoint
else Qwen4ExpPLEUnquantizedEmbeddingMethod()
)
embedding = Qwen4ExpPLEPinnedHostEmbedding(
4,
3,
params_dtype=torch.bfloat16,
padding_size=1,
prefix="test.ple_embedding",
embedding_method=embedding_method,
)
storage_dtype = torch.float8_e4m3fn if fp8_checkpoint else torch.bfloat16
loaded_weight = (
torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype)
)
copied = copy_ple_embedding_shard_(
embedding.weight,
loaded_weight,
checkpoint_start=0,
tp_start=0,
tp_end=4,
)
input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
output = embedding._lookup(input_ids)
if fp8_checkpoint:
embedding.weight_scale.data.fill_(0.25)
weight_ptr = embedding.weight.data_ptr()
embedding.quant_method.process_weights_after_loading(embedding)
assert embedding.weight.data_ptr() == weight_ptr
dequantized = embedding.dequantize(output, torch.bfloat16)
assert copied == 4
assert embedding.weight.device.type == "cpu"
assert embedding.weight.is_pinned()
assert output.dtype == storage_dtype
expected = loaded_weight[input_ids.cpu()].to(device="cuda:0")
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
expected_scale = 0.25 if fp8_checkpoint else 1.0
torch.testing.assert_close(
dequantized,
expected.to(torch.bfloat16) * expected_scale,
rtol=0,
atol=0,
)
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
@@ -310,15 +616,15 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
)
assert isinstance(
_get_ple_embedding_quant_method(quant_config, prefix),
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
assert (
_get_ple_embedding_quant_method(
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
"model.language_model.layers.2.moe.gate_proj",
)
is None
),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
+154
View File
@@ -22,6 +22,7 @@ from vllm.config import (
CacheConfig,
CompilationConfig,
DeviceConfig,
EngramConfig,
KernelConfig,
KVTransferConfig,
ModelConfig,
@@ -913,6 +914,159 @@ def test_all2all_backend_has_portable_default():
assert ParallelConfig().all2all_backend == "allgather_reducescatter"
@pytest.mark.parametrize(
"dp_size, across_dp, expected",
[(1, False, 4), (1, True, 4), (2, False, 4), (2, True, 8)],
)
def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: int):
parallel = ParallelConfig(
tensor_parallel_size=4,
data_parallel_size=dp_size,
distributed_executor_backend="mp",
)
config = EngramConfig(embedding_across_dp=across_dp)
assert config.get_parallel_size(parallel) == expected
def test_engram_rejects_elastic_cross_dp():
parallel = ParallelConfig(
tensor_parallel_size=4,
data_parallel_size=2,
distributed_executor_backend="mp",
)
parallel.enable_elastic_ep = True
with pytest.raises(ValueError, match="embedding_across_dp.*elastic EP"):
EngramConfig(embedding_across_dp=True).verify_parallel_config(parallel)
@pytest.mark.parametrize("legacy", [None, "0", "1"])
def test_engram_cpu_offload_environment_fallback(monkeypatch, legacy):
"""Explicit settings must override the legacy environment fallback."""
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
if legacy is not None:
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
assert EngramConfig().cpu_offload == (legacy == "1")
assert EngramConfig(cpu_offload=False).cpu_offload is False
assert EngramConfig(cpu_offload=True).cpu_offload is True
@pytest.mark.parametrize(
"architecture, ple_layers, cuda, supported",
[
("Qwen4ExpForCausalLM", [1], True, True),
("Qwen4ExpForConditionalGeneration", [1], True, True),
("Qwen4ExpForCausalLM", [], True, False),
("Qwen4ExpForCausalLM", None, True, False),
("Qwen4ExpForCausalLM", [1], False, False),
("LlamaForCausalLM", [1], True, False),
("Qwen4ExpMTP", [], True, False),
(None, None, True, False),
],
)
def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, supported):
"""A similarly named HF field must not enable unsupported implementations."""
monkeypatch.setattr(current_platform, "is_cuda", lambda: cuda)
model = (
cast(
ModelConfig,
SimpleNamespace(
architecture=architecture,
hf_text_config=SimpleNamespace(ple_layer_ids=ple_layers),
),
)
if architecture is not None
else None
)
config = EngramConfig(cpu_offload=False, embedding_across_dp=False)
if supported:
config.verify_model_config(model)
else:
with pytest.raises(ValueError, match="requires a model with supported Engram"):
config.verify_model_config(model)
def test_engram_config_defaults_to_none(monkeypatch):
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
config = VllmConfig()
assert config.engram_config is None
assert config.compute_hash()
@pytest.mark.parametrize("legacy", ["0", "1"])
def test_engram_none_resolves_legacy_offload(monkeypatch, legacy):
"""Legacy enablement materializes a config before model validation."""
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
config = cast(
VllmConfig,
SimpleNamespace(
model_config=SimpleNamespace(
architecture="Qwen4ExpForCausalLM",
hf_text_config=SimpleNamespace(ple_layer_ids=[1]),
),
speculative_config=None,
engram_config=None,
parallel_config=ParallelConfig(),
),
)
VllmConfig._resolve_and_verify_engram_config(config)
if legacy == "1":
assert config.engram_config is not None
assert config.engram_config.cpu_offload is True
else:
assert config.engram_config is None
@pytest.mark.parametrize("legacy", ["0", "1"])
def test_engram_explicit_config_requires_supported_model(monkeypatch, legacy):
"""Explicit all-false settings still opt into model validation."""
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
with pytest.raises(ValueError, match="requires a model with supported Engram"):
VllmConfig(engram_config=EngramConfig(cpu_offload=False))
def test_engram_legacy_offload_requires_supported_model(monkeypatch):
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
with pytest.raises(ValueError, match="requires a model with supported Engram"):
VllmConfig()
@pytest.mark.parametrize("target_has_ple", [False, True])
def test_engram_draft_config_validates_target(monkeypatch, target_has_ple):
"""MTP may inherit cross-DP sharding without having its own PLE layers."""
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
target = SimpleNamespace(
architecture="Qwen4ExpForCausalLM",
hf_text_config=SimpleNamespace(ple_layer_ids=[1] if target_has_ple else []),
)
draft = SimpleNamespace(architecture="Qwen4ExpMTP")
config = cast(
VllmConfig,
SimpleNamespace(
model_config=draft,
speculative_config=SimpleNamespace(
draft_model_config=draft, target_model_config=target
),
engram_config=EngramConfig(embedding_across_dp=True),
parallel_config=ParallelConfig(),
),
)
if target_has_ple:
VllmConfig._resolve_and_verify_engram_config(config)
else:
with pytest.raises(ValueError, match="requires a model with supported Engram"):
VllmConfig._resolve_and_verify_engram_config(config)
def test_engram_hash_tracks_storage_and_sharding():
configs = [
EngramConfig(cpu_offload=offload, embedding_across_dp=across_dp)
for offload in (False, True)
for across_dp in (False, True)
]
assert len({config.compute_hash() for config in configs}) == 4
@pytest.mark.parametrize("port", [1, 29550, 65535])
def test_data_parallel_rpc_port_accepts_valid_ports(port: int):
assert ParallelConfig(data_parallel_rpc_port=port).data_parallel_rpc_port == port
+3
View File
@@ -13,6 +13,7 @@ from vllm.config.device import DeviceConfig
from vllm.config.diffusion import DiffusionConfig
from vllm.config.ec_manager_config import EncoderCacheManagerConfig
from vllm.config.ec_transfer import ECTransferConfig
from vllm.config.engram import EngramConfig
from vllm.config.fault_tolerance import FaultToleranceConfig
from vllm.config.kernel import KernelConfig
from vllm.config.kv_events import KVEventsConfig
@@ -81,6 +82,8 @@ __all__ = [
"EncoderCacheManagerConfig",
# From vllm.config.ec_transfer
"ECTransferConfig",
# From vllm.config.engram
"EngramConfig",
# From vllm.config.kernel
"KernelConfig",
# From vllm.config.kv_events
+82
View File
@@ -0,0 +1,82 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from pydantic import Field
import vllm.envs as envs
from vllm.config.utils import config, get_hash_factors, hash_factors
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.config.parallel import ParallelConfig
logger = init_logger(__name__)
def _default_cpu_offload() -> bool:
"""Honor the legacy environment variable only when the field is omitted."""
if envs.is_set("VLLM_PLE_CPU_OFFLOAD"):
logger.warning_once(
"VLLM_PLE_CPU_OFFLOAD is a legacy setting and may be removed in a "
"future release. Use --engram-config.cpu_offload instead."
)
return envs.VLLM_PLE_CPU_OFFLOAD
@config
class EngramConfig:
"""Configuration for Engram embedding storage and sharding."""
cpu_offload: bool = Field(default_factory=_default_cpu_offload)
"""Store embedding weights in pinned CPU memory for UVA lookup.
Defaults to False, or VLLM_PLE_CPU_OFFLOAD when set for compatibility.
An explicit value takes precedence over the legacy environment variable."""
embedding_across_dp: bool = False
"""Shard embeddings across TP and all DP ranks when enabled.
Otherwise, each DP rank has a separate TP-sharded embedding replica."""
def verify_model_config(self, model_config: "ModelConfig | None") -> None:
"""Reject Engram configuration for models without supported embeddings."""
from vllm.platforms import current_platform
supported_architectures = {
"Qwen4ExpForCausalLM",
"Qwen4ExpForConditionalGeneration",
}
if (
model_config is None
or model_config.architecture not in supported_architectures
or not current_platform.is_cuda()
or not getattr(model_config.hf_text_config, "ple_layer_ids", None)
):
raise ValueError(
"EngramConfig requires a model with supported Engram "
"embeddings. Currently only the CUDA Qwen4Exp implementation "
"with non-empty ple_layer_ids is supported."
)
def verify_parallel_config(self, parallel_config: "ParallelConfig") -> None:
"""Reject unsupported embedding parallel topologies."""
if (
self.embedding_across_dp
and parallel_config.data_parallel_size > 1
and parallel_config.enable_elastic_ep
):
raise ValueError(
"Engram embedding_across_dp is not supported with elastic EP yet."
)
def get_parallel_size(self, parallel_config: "ParallelConfig") -> int:
"""Derive the embedding group size from the parallel configuration."""
size = parallel_config.tensor_parallel_size
if self.embedding_across_dp and parallel_config.data_parallel_size > 1:
size *= parallel_config.data_parallel_size
return size
def compute_hash(self) -> str:
"""Hash settings that affect embedding execution and graph structure."""
return hash_factors(get_hash_factors(self, set()))
+28
View File
@@ -34,6 +34,7 @@ from .device import DeviceConfig
from .diffusion import DiffusionConfig
from .ec_manager_config import EncoderCacheManagerConfig
from .ec_transfer import ECTransferConfig
from .engram import EngramConfig
from .kernel import KernelConfig
from .kv_events import KVEventsConfig
from .kv_transfer import KVTransferConfig
@@ -370,6 +371,8 @@ class VllmConfig:
"""Model weight offloading configuration."""
attention_config: AttentionConfig = Field(default_factory=AttentionConfig)
"""Attention configuration."""
engram_config: EngramConfig | None = None
"""Optional Engram configuration, only valid for supported PLE models."""
mamba_config: MambaConfig = Field(default_factory=MambaConfig)
"""Mamba configuration."""
kernel_config: KernelConfig = Field(default_factory=KernelConfig)
@@ -506,6 +509,11 @@ class VllmConfig:
vllm_factors.append(self.attention_config.compute_hash())
else:
vllm_factors.append("None")
vllm_factors.append(
self.engram_config.compute_hash()
if self.engram_config is not None
else "None"
)
if self.lora_config:
vllm_factors.append(self.lora_config.compute_hash())
else:
@@ -1083,6 +1091,25 @@ class VllmConfig:
if not self.use_v2_model_runner:
raise ValueError("trace replay requires Model Runner V2")
def _resolve_and_verify_engram_config(self) -> None:
"""Resolve legacy offload settings and validate model and parallel configs."""
if self.engram_config is None:
if not envs.VLLM_PLE_CPU_OFFLOAD:
return
self.engram_config = EngramConfig()
model_config = self.model_config
speculative_config = self.speculative_config
# Draft configs inherit the target's communication groups and settings.
# Qwen4Exp MTP itself disables PLE, so validate its target instead.
if (
speculative_config is not None
and model_config is speculative_config.draft_model_config
):
model_config = speculative_config.target_model_config
self.engram_config.verify_model_config(model_config)
self.engram_config.verify_parallel_config(self.parallel_config)
logger.info_once("Resolved Engram configuration: %s", str(self.engram_config))
def __post_init__(self):
"""Verify configs are valid & consistent with each other."""
@@ -1095,6 +1122,7 @@ class VllmConfig:
logger.info_once("Performance mode set to '%s'.", self.performance_mode)
self.try_verify_and_update_config()
self._resolve_and_verify_engram_config()
# Models may have supplied their own DCP defaults above; anything still
# unset falls back to the stock ones.
@@ -47,7 +47,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
global_world_size,
use_all2all=use_all2all,
)
if "tp" not in unique_name:
# Match the group name exactly so ETP does not enable TP-only backends.
if unique_name.split(":")[0] != "tp":
# custom allreduce or torch symm mem can be used only by tp
use_custom_allreduce = False
use_torch_symm_mem = False
+38 -2
View File
@@ -1551,6 +1551,15 @@ def get_tp_group() -> GroupCoordinator:
return _TP
_ETP: GroupCoordinator | None = None
def get_etp_group() -> GroupCoordinator:
"""Return the Engram Tensor Parallel (ETP) group that shards one embedding table."""
assert _ETP is not None, "Engram tensor-parallel group is not initialized"
return _ETP
_DCP: GroupCoordinator | None = None
@@ -2008,6 +2017,28 @@ def initialize_model_parallel(
group_name="tp",
)
global _ETP
assert _ETP is None, "Engram tensor-parallel group is already initialized"
engram_tensor_parallel_size = (
config.engram_config.get_parallel_size(parallel_config)
if config.engram_config is not None
else tensor_model_parallel_size
)
if engram_tensor_parallel_size == tensor_model_parallel_size:
_ETP = _TP
else:
group_ranks = (
all_ranks.permute(0, 2, 3, 1, 4)
.reshape(-1, engram_tensor_parallel_size)
.unbind(0)
)
_ETP = init_model_parallel_group(
[ranks.tolist() for ranks in group_ranks],
get_world_group().local_rank,
backend,
group_name="etp",
)
# Build the DCP model-parallel groups.
global _DCP
assert _DCP is None, "decode context model parallel group is already initialized"
@@ -2142,13 +2173,14 @@ def initialize_model_parallel(
logger.info_once(
"rank %s in world size %s is assigned as "
"DP rank %s, PP rank %s, PCP rank %s, "
"TP rank %s, EP rank %s, EPLB rank %s",
"TP rank %s, ETP rank %s, EP rank %s, EPLB rank %s",
rank,
world_size,
_DP.rank_in_group,
_PP.rank_in_group,
_PCP.rank_in_group,
_TP.rank_in_group,
_ETP.rank_in_group,
_EP.rank_in_group if _EP is not None else "N/A",
_EPLB.rank_in_group if _EPLB is not None else "N/A",
)
@@ -2243,7 +2275,11 @@ def get_node_count() -> int:
def destroy_model_parallel():
"""Set the groups to none and destroy them."""
global _TP
global _TP, _ETP
if _ETP and _ETP is not _TP:
_ETP.destroy()
_ETP = None
if _TP:
_TP.destroy()
+6
View File
@@ -42,6 +42,7 @@ from vllm.config import (
DiffusionConfig,
ECTransferConfig,
EncoderCacheManagerConfig,
EngramConfig,
EPLBConfig,
FaultToleranceConfig,
KernelConfig,
@@ -688,6 +689,7 @@ class EngineArgs:
pooler_config: PoolerConfig | None = ModelConfig.pooler_config
compilation_config: CompilationConfig = get_field(VllmConfig, "compilation_config")
attention_config: AttentionConfig = get_field(VllmConfig, "attention_config")
engram_config: EngramConfig | None = VllmConfig.engram_config
mamba_config: MambaConfig = get_field(VllmConfig, "mamba_config")
kernel_config: KernelConfig = get_field(VllmConfig, "kernel_config")
enable_flashinfer_autotune: bool = get_field(
@@ -788,6 +790,8 @@ class EngineArgs:
self.compilation_config = CompilationConfig(**self.compilation_config)
if isinstance(self.attention_config, dict):
self.attention_config = AttentionConfig(**self.attention_config)
if isinstance(self.engram_config, dict):
self.engram_config = EngramConfig(**self.engram_config)
if isinstance(self.mamba_config, dict):
self.mamba_config = MambaConfig(**self.mamba_config)
if isinstance(self.kernel_config, dict):
@@ -1701,6 +1705,7 @@ class EngineArgs:
vllm_group.add_argument(
"--attention-config", "-ac", **vllm_kwargs["attention_config"]
)
vllm_group.add_argument("--engram-config", **vllm_kwargs["engram_config"])
vllm_group.add_argument("--reasoning-config", **vllm_kwargs["reasoning_config"])
vllm_group.add_argument("--kernel-config", **vllm_kwargs["kernel_config"])
vllm_group.add_argument(
@@ -2599,6 +2604,7 @@ class EngineArgs:
load_config=load_config,
offload_config=offload_config,
attention_config=attention_config,
engram_config=self.engram_config,
mamba_config=mamba_config,
kernel_config=kernel_config,
lora_config=lora_config,
+6
View File
@@ -155,6 +155,7 @@ if TYPE_CHECKING:
VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT: bool = False
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
VLLM_PLE_CPU_OFFLOAD: bool = False
VLLM_DISABLE_COMPILE_CACHE: bool = False
VLLM_REPLICATE_EMBED: bool = False
VLLM_USE_LAYERNAME: bool = True
@@ -2077,6 +2078,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_LOG_MODEL_INSPECTION": lambda: bool(
int(os.getenv("VLLM_LOG_MODEL_INSPECTION", "0"))
),
# Keep Qwen4Exp PLE embedding tables in pinned CPU memory and gather their
# rows through UVA on a dedicated CUDA stream.
# Legacy fallback for EngramConfig.cpu_offload, which takes precedence.
# This environment variable may be removed in a future release.
"VLLM_PLE_CPU_OFFLOAD": lambda: bool(int(os.getenv("VLLM_PLE_CPU_OFFLOAD", "0"))),
# Debug logging for --enable-mfu-metrics
"VLLM_DEBUG_MFU_METRICS": lambda: bool(
int(os.getenv("VLLM_DEBUG_MFU_METRICS", "0"))
@@ -23,6 +23,9 @@ else:
class QuantizeMethodBase(ABC):
"""Base class for different quantized methods."""
requires_device_loading: bool = True
"""Whether post-load processing requires parameters on the target device."""
uses_meta_device: bool = False
"""
Whether this method creates weights on meta device for online quantization.
@@ -11,6 +11,7 @@ from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.distributed import (
GroupCoordinator,
divide,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -240,6 +241,7 @@ class VocabParallelEmbedding(PluggableLayer):
prefix: full name of the layer in the state dict
disable_tp: If true, tensor parallelism will be disabled for this layer.
quant_method: Preselected quantization method for model-specific layers.
parallel_group: Process group used to shard and reduce the embedding.
""" # noqa: E501
# --8<-- [end:vocab_parallel_embedding]
@@ -256,13 +258,18 @@ class VocabParallelEmbedding(PluggableLayer):
*,
disable_tp: bool = False,
quant_method: QuantizeMethodBase | None = None,
parallel_group: GroupCoordinator | None = None,
):
super().__init__()
# Keep the input dimensions.
self.disable_tp = disable_tp
self.parallel_group = parallel_group
if disable_tp:
tp_rank, self.tp_size = 0, 1
elif parallel_group is not None:
tp_rank = parallel_group.rank_in_group
self.tp_size = parallel_group.world_size
else:
tp_rank = get_tensor_model_parallel_rank()
self.tp_size = get_tensor_model_parallel_world_size()
@@ -535,10 +542,16 @@ class VocabParallelEmbedding(PluggableLayer):
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
comm_output = output_parallel.view(torch.int8)
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
output = tensor_model_parallel_all_reduce(comm_output)
output = (
self.parallel_group.all_reduce(comm_output)
if self.parallel_group is not None
else tensor_model_parallel_all_reduce(comm_output)
)
return output.view(output_parallel.dtype)
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
# Reduce across all the model parallel GPUs.
if self.parallel_group is not None:
return self.parallel_group.all_reduce(output_parallel)
return tensor_model_parallel_all_reduce(output_parallel)
def extra_repr(self) -> str:
+10 -3
View File
@@ -4,7 +4,7 @@
import inspect
import warnings
from contextlib import contextmanager
from contextlib import contextmanager, nullcontext
from typing import Any
import torch
@@ -120,11 +120,18 @@ def process_weights_after_loading(
"does not support pre-processed weights"
)
# When quant methods need to process weights after loading
# (for repacking, quantizing, etc), they expect parameters
# (for repacking, quantizing, etc), they typically expect parameters
# to be on the global target device. This scope is for the
# case where cpu offloading is used, where we will move the
# parameters onto device for processing and back off after.
with device_loading_context(module, target_device):
# Methods that can process weights in place (e.g. PLE scale
# validation) set requires_device_loading=False to skip this move.
loading_context = (
device_loading_context(module, target_device)
if quant_method.requires_device_loading
else nullcontext()
)
with loading_context:
quant_method.process_weights_after_loading(module)
# process_weights_after_loading may swap in freshly-created
# Parameters (e.g. FP8 requantization), which are stamped with the
+37
View File
@@ -461,6 +461,27 @@ class Qwen4ExpModel(nn.Module):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
@staticmethod
def _start_layer_ple_prefetch(
layer: nn.Module,
hidden_states: torch.Tensor,
input_ids: torch.Tensor | None,
query_start_loc: torch.Tensor | None,
ngram_context: torch.Tensor | None,
) -> None:
"""Start a layer's PLE prefetch when the required inputs exist."""
ple: Qwen4ExpPLELayer | None = getattr(layer, "ple", None)
if ple is None:
return
if input_ids is None or query_start_loc is None or ngram_context is None:
raise RuntimeError("PLE inputs were not prepared")
ple.start_prefetch(
hidden_states,
input_ids,
query_start_loc,
ngram_context,
)
def forward(
self,
input_ids: torch.Tensor | None,
@@ -487,10 +508,26 @@ class Qwen4ExpModel(nn.Module):
block_output = None
injection = None
last_layer = None
if self.start_layer < self.end_layer:
self._start_layer_ple_prefetch(
self.layers[self.start_layer],
hidden_states,
input_ids,
query_start_loc,
ngram_context,
)
for layer_idx, layer in islice(
enumerate(self.layers), self.start_layer, self.end_layer
):
last_layer = layer
if layer_idx + 1 < self.end_layer:
self._start_layer_ple_prefetch(
self.layers[layer_idx + 1],
hidden_states,
input_ids,
query_start_loc,
ngram_context,
)
hidden_states, block_output, injection = layer(
hidden_states=hidden_states,
prev_block_output=block_output,
@@ -0,0 +1,939 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp n-gram embeddings with device and pinned-host storage."""
from abc import ABC, abstractmethod
from collections.abc import Iterable
from typing import ClassVar
import torch
import torch.nn.functional as F
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.distributed import get_dp_group, get_etp_group, get_tp_group
from vllm.forward_context import DPMetadata, get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptMixedPrecisionConfig,
ModelOptQuantConfigBase,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
create_fp8_scale_parameter,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.model_executor.parameter import (
ModelWeightParameter,
PerTensorScaleParameter,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.triton_utils import tl, triton
from vllm.utils.platform_utils import is_uva_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
from ..common.ple import PLEVocabParallelEmbedding
from .ops.ple import ple_ngram_ids
logger = init_logger(__name__)
class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
"""ETP-sharded PLE table shared by device and pinned-host backends."""
supports_prefetch: ClassVar[bool] = False
def __init__(
self,
num_embeddings: int,
embedding_dim: int,
*,
params_dtype: torch.dtype,
padding_size: int,
prefix: str,
embedding_method: "Qwen4ExpPLEEmbeddingMethod",
num_ngram_heads: int = 1,
max_total_tokens: int = 0,
data_parallel_rank: int = 0,
) -> None:
del num_ngram_heads, max_total_tokens
super().__init__(
num_embeddings,
embedding_dim,
params_dtype=params_dtype,
padding_size=padding_size,
prefix=prefix,
quant_method=embedding_method,
parallel_group=get_etp_group(),
)
self.embedding_method = embedding_method
self.data_parallel_rank = data_parallel_rank
tp_size = get_tp_group().world_size
if self.tp_size % tp_size:
raise ValueError(
"ETP size must be divisible by TP size, but got "
f"ETP={self.tp_size} and TP={tp_size}"
)
self.etp_data_parallel_size = self.tp_size // tp_size
@abstractmethod
def allocate_embedding_weight(
self,
num_embeddings: int,
embedding_dim: int,
dtype: torch.dtype,
) -> torch.Tensor:
"""Allocate storage for the complete embedding weight."""
raise NotImplementedError
def dequantize(
self,
embeddings: torch.Tensor,
output_dtype: torch.dtype,
) -> torch.Tensor:
"""Delegate storage-format conversion to the embedding method."""
return self.embedding_method.dequantize(self, embeddings, output_dtype)
def _get_dp_gather_slot(self, local_num_tokens: int) -> tuple[int, int]:
"""Return the per-DP slot size and this rank's slot offset."""
if self.etp_data_parallel_size == 1:
return local_num_tokens, 0
dp_metadata: DPMetadata | None = get_forward_context().dp_metadata
if dp_metadata is None:
raise RuntimeError("ETP spanning DP requires DP token metadata")
group_start = (self.data_parallel_rank // self.etp_data_parallel_size) * (
self.etp_data_parallel_size
)
group_end = group_start + self.etp_data_parallel_size
token_counts = dp_metadata.num_tokens_across_dp_cpu.tolist()
group_counts = token_counts[group_start:group_end]
slot_size = max(group_counts)
dp_rank = get_dp_group().rank_in_group
return slot_size, dp_rank * slot_size
def _gather_dp_ids(
self,
ngram_ids: torch.Tensor,
slot_size: int,
) -> torch.Tensor:
"""Gather DP-local IDs that share one ETP-sharded PLE table."""
if self.etp_data_parallel_size == 1:
return ngram_ids
if ngram_ids.shape[0] < slot_size:
padding = ngram_ids.new_zeros(
slot_size - ngram_ids.shape[0], ngram_ids.shape[1]
)
ngram_ids = torch.cat((ngram_ids, padding), dim=0)
return get_dp_group().all_gather(ngram_ids, dim=0)
def _select_embeddings(
self,
embeddings: torch.Tensor,
local_num_tokens: int,
slot_offset: int,
) -> torch.Tensor:
"""Select this DP rank's rows from the ETP-reduced embeddings."""
if self.etp_data_parallel_size == 1:
return embeddings
return embeddings.narrow(0, slot_offset, local_num_tokens)
@abstractmethod
def start_prefetch(
self,
hidden_states: torch.Tensor,
ngram_ids: torch.Tensor,
) -> None:
"""Start an asynchronous lookup when supported."""
raise NotImplementedError
class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
"""Quantization interface shared by resident and pinned PLE tables."""
# PLE post-load processing only validates scales in their current storage.
requires_device_loading: bool = False
@staticmethod
def from_quant_config(
quant_config: QuantizationConfig | None,
prefix: str,
embedding_dtype: str | None = None,
) -> "Qwen4ExpPLEEmbeddingMethod":
"""Select the concrete PLE embedding format for a layer."""
if embedding_dtype == "float8_e4m3fn":
return Qwen4ExpPLEFp8EmbeddingMethod()
if quant_config is None:
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
if quant_config._resolve_quant_algo(prefix) == "FP8":
return Qwen4ExpPLEFp8EmbeddingMethod()
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if isinstance(
quant_config, ModelOptQuantConfigBase
) and quant_config.is_layer_excluded(prefix):
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if not isinstance(quant_config, Fp8Config):
raise NotImplementedError(
"Qwen4Exp PLE embedding does not support quantization config "
f"{type(quant_config).__name__}"
)
ignored_layers = quant_config.ignored_layers
if is_layer_skipped(
prefix,
ignored_layers,
quant_config.packed_modules_mapping,
match_mode=quant_config.ignored_layers_match_mode,
):
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
# PLE checkpoint shards form one runtime embedding parameter.
shard_prefix = f"{prefix}.shard_"
if any(name.startswith(shard_prefix) for name in ignored_layers):
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
if not quant_config.is_checkpoint_fp8_serialized:
raise NotImplementedError(
"Qwen4Exp PLE embedding only supports serialized FP8 checkpoints"
)
return Qwen4ExpPLEFp8EmbeddingMethod()
def apply(
self,
layer: nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
raise NotImplementedError("PLE weights only support embedding lookup")
def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor:
return F.embedding(input_, layer.weight)
@abstractmethod
def dequantize(
self,
layer: nn.Module,
embeddings: torch.Tensor,
output_dtype: torch.dtype,
) -> torch.Tensor:
"""Convert looked-up PLE rows to the activation dtype."""
raise NotImplementedError
class Qwen4ExpPLEUnquantizedEmbeddingMethod(Qwen4ExpPLEEmbeddingMethod):
"""Unquantized PLE embedding storage and lookup semantics."""
def create_weights(
self,
layer: Qwen4ExpPLEEmbedding,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size
weight = nn.Parameter(
layer.allocate_embedding_weight(
sum(output_partition_sizes),
input_size_per_partition,
params_dtype,
),
requires_grad=False,
)
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
set_weight_attrs(weight, extra_weight_attrs)
layer.register_parameter("weight", weight)
def dequantize(
self,
layer: nn.Module,
embeddings: torch.Tensor,
output_dtype: torch.dtype,
) -> torch.Tensor:
del layer, output_dtype
return embeddings
class Qwen4ExpPLEFp8EmbeddingMethod(Qwen4ExpPLEEmbeddingMethod):
"""FP8 PLE embedding with one global checkpoint scale."""
def create_weights(
self,
layer: Qwen4ExpPLEEmbedding,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size, params_dtype
weight_loader = extra_weight_attrs.get("weight_loader")
weight = ModelWeightParameter(
data=layer.allocate_embedding_weight(
sum(output_partition_sizes),
input_size_per_partition,
torch.float8_e4m3fn,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
weight_scale = create_fp8_scale_parameter(
PerTensorScaleParameter,
output_partition_sizes,
input_size_per_partition,
None,
weight_loader,
scale_dtype=torch.float32,
)
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: nn.Module) -> None:
"""Reject FP8 PLE checkpoints without a global scale."""
sentinel = torch.finfo(torch.float32).min
if torch.any(layer.weight_scale == sentinel):
raise ValueError("FP8 PLE checkpoint is missing its global scale")
def dequantize(
self,
layer: nn.Module,
embeddings: torch.Tensor,
output_dtype: torch.dtype,
) -> torch.Tensor:
weight_scale = getattr(layer, "weight_scale", None)
if weight_scale is None:
raise RuntimeError("FP8 PLE embedding is missing its global scale")
if weight_scale.device != embeddings.device:
raise RuntimeError("FP8 PLE embedding scale must be on the output device")
return embeddings.to(output_dtype) * weight_scale.to(output_dtype)
class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding):
"""PLE table allocated on the active model device."""
def allocate_embedding_weight(
self,
num_embeddings: int,
embedding_dim: int,
dtype: torch.dtype,
) -> torch.Tensor:
"""Allocate the complete PLE weight on the active device."""
return torch.empty(num_embeddings, embedding_dim, dtype=dtype)
def start_prefetch(
self,
hidden_states: torch.Tensor,
ngram_ids: torch.Tensor,
) -> None:
"""Resident embedding prefetch is a no-op."""
return None
def forward(self, ngram_ids: torch.Tensor) -> torch.Tensor:
"""Gather ETP inputs, look up embeddings, and select local rows."""
slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0])
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
embeddings = super().forward(gathered_ids)
return self._select_embeddings(
embeddings,
ngram_ids.shape[0],
slot_offset,
)
@triton.jit
def _lookup_ple_embedding_from_pinned_kernel(
weight_ptr,
ids_ptr,
output_ptr,
embedding_dim,
tp_vocab_start,
tp_vocab_end,
BLOCK_D: tl.constexpr,
):
"""Look up TP-owned PLE rows through a CUDA view of pinned host memory."""
row_id = tl.program_id(0)
global_idx = tl.load(ids_ptr + row_id)
in_range = (global_idx >= tp_vocab_start) & (global_idx < tp_vocab_end)
local_idx = tl.where(in_range, global_idx - tp_vocab_start, 0)
offsets = tl.arange(0, BLOCK_D)
store_mask = offsets < embedding_dim
load_mask = store_mask & in_range
values = tl.load(
weight_ptr + local_idx * embedding_dim + offsets,
mask=load_mask,
other=0.0,
)
tl.store(
output_ptr + row_id * embedding_dim + offsets,
values,
mask=store_mask,
)
class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
"""PLE table loaded into pinned CPU memory and looked up through UVA."""
supports_prefetch: ClassVar[bool] = True
def __init__(
self,
num_embeddings: int,
embedding_dim: int,
*,
params_dtype: torch.dtype,
padding_size: int,
prefix: str,
embedding_method: Qwen4ExpPLEEmbeddingMethod,
num_ngram_heads: int = 1,
max_total_tokens: int = 0,
data_parallel_rank: int = 0,
) -> None:
if not is_uva_available():
raise RuntimeError("Engram CPU offload requires UVA support")
super().__init__(
num_embeddings,
embedding_dim,
params_dtype=params_dtype,
padding_size=padding_size,
prefix=prefix,
embedding_method=embedding_method,
num_ngram_heads=num_ngram_heads,
max_total_tokens=max_total_tokens,
data_parallel_rank=data_parallel_rank,
)
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
self._block_d = triton.next_power_of_2(self.embedding_dim)
self._prefetch_stream = torch.cuda.Stream(device=self._uva_weight.device)
self._prefetch_buffer = torch.empty(
max_total_tokens * self.etp_data_parallel_size,
num_ngram_heads,
self.embedding_dim,
dtype=self.weight.dtype,
device=self._uva_weight.device,
)
self._output_dim = num_ngram_heads * self.embedding_dim
def allocate_embedding_weight(
self,
num_embeddings: int,
embedding_dim: int,
dtype: torch.dtype,
) -> torch.Tensor:
"""Allocate the complete PLE weight directly in pinned CPU memory."""
return torch.empty(
num_embeddings,
embedding_dim,
dtype=dtype,
device="cpu",
pin_memory=True,
)
def _lookup(
self,
input_ids: torch.Tensor,
output: torch.Tensor | None = None,
) -> torch.Tensor:
"""Look up local ETP rows while preserving the weight storage dtype."""
expected_shape = (*input_ids.shape, self.embedding_dim)
if output is None:
output = torch.empty(
expected_shape,
dtype=self.weight.dtype,
device=input_ids.device,
)
elif (
tuple(output.shape) != expected_shape
or output.dtype != self.weight.dtype
or output.device != input_ids.device
):
raise ValueError(
"PLE prefetch output must match the input shape, weight dtype, "
"and input device"
)
flat_ids = input_ids.reshape(-1).long()
if flat_ids.numel():
_lookup_ple_embedding_from_pinned_kernel[(flat_ids.numel(),)](
self._uva_weight,
flat_ids,
output,
self.embedding_dim,
self.shard_indices.org_vocab_start_index,
self.shard_indices.org_vocab_end_index,
BLOCK_D=self._block_d,
)
return output
def _reduce_etp_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor:
"""Combine pinned lookup results owned by different ETP ranks."""
if self.tp_size == 1:
return embeddings
assert self.parallel_group is not None
if embeddings.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
# Each vocabulary row has one owner, so reduce the raw FP8 bytes.
reduced = self.parallel_group.all_reduce(embeddings.view(torch.int8))
return reduced.view(embeddings.dtype)
return self.parallel_group.all_reduce(embeddings)
@eager_break_during_capture
def start_prefetch(
self,
hidden_states: torch.Tensor,
ngram_ids: torch.Tensor,
) -> None:
"""Gather ETP IDs and launch their UVA lookup on the side stream."""
slot_size, _ = self._get_dp_gather_slot(ngram_ids.shape[0])
gathered_ids = self._gather_dp_ids(ngram_ids, slot_size)
active_output = self._prefetch_buffer[: gathered_ids.shape[0]]
prefetch_stream = self._prefetch_stream
prefetch_stream.wait_stream(torch.cuda.current_stream())
gathered_ids.record_stream(prefetch_stream)
with torch.cuda.stream(prefetch_stream):
self._lookup(gathered_ids, output=active_output)
@eager_break_during_capture
def _finalize_prefetch(
self,
prefetch_output: torch.Tensor,
output: torch.Tensor,
) -> None:
"""Join the side stream, reduce ETP shards, and select local rows."""
torch.cuda.current_stream().wait_stream(self._prefetch_stream)
slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0])
active_output = prefetch_output[: slot_size * self.etp_data_parallel_size]
embeddings = self._reduce_etp_embeddings(active_output)
embeddings = self._select_embeddings(
embeddings,
output.shape[0],
slot_offset,
)
output.copy_(embeddings.flatten(-2))
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Finish the pinned lookup into graph-owned output storage."""
output = self._prefetch_buffer.new_empty(
(hidden_states.shape[0], self._output_dim)
)
self._finalize_prefetch(self._prefetch_buffer, output)
return output
class Qwen4ExpNGramEmbedding(nn.Module):
_MASK64 = (1 << 64) - 1
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
_SPLITMIX_M1 = 0xBF58476D1CE4E5B9
_SPLITMIX_M2 = 0x94D049BB133111EB
_PLE_LAYER_PRIME = 10007
@classmethod
def _splitmix64(cls, value: int) -> int:
"""Mix an integer into a deterministic unsigned 64-bit value."""
value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64
value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64
value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64
return (value ^ (value >> 31)) & cls._MASK64
@staticmethod
def _is_prime_64(value: int) -> bool:
"""Return whether a 64-bit integer is prime."""
if value < 2:
return False
for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37):
if value % prime == 0:
return value == prime
exponent = value - 1
shifts = 0
while exponent % 2 == 0:
exponent //= 2
shifts += 1
for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022):
if base % value == 0:
continue
witness = pow(base, exponent, value)
if witness in (1, value - 1):
continue
for _ in range(shifts - 1):
witness = pow(witness, 2, value)
if witness == value - 1:
break
else:
return False
return True
@classmethod
def _nth_prime_after(cls, start: int, count: int) -> int:
"""Return the ``count``-th prime strictly greater than ``start``."""
prime = int(start)
for _ in range(count):
candidate = prime + 1
if candidate <= 2:
prime = 2
continue
if candidate % 2 == 0:
candidate += 1
while not cls._is_prime_64(candidate):
candidate += 2
prime = candidate
return prime
@classmethod
def _make_layer_multipliers(
cls,
*,
ngram_size: int,
unigram_vocab_size: int,
seed: int,
ple_dense_layer_id: int,
) -> list[int]:
"""Build deterministic hash multipliers for one PLE layer."""
max_multiplier = ((1 << 63) - 1) // unigram_vocab_size
half_bound = max(1, max_multiplier // 2)
base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id
multipliers = []
for index in range(ngram_size):
value = base_seed + cls._SPLITMIX_GAMMA * (index + 1)
multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1)
return multipliers
@classmethod
def _make_vocab_layout(
cls,
*,
ngram_vocab_size_base: int,
ngram_heads: int,
ple_dense_layer_id: int,
) -> tuple[list[int], list[int], int]:
"""Build per-head vocabulary sizes, offsets, and total row count."""
sizes: list[int] = []
offsets: list[int] = []
offset = 0
for local_head in range(ngram_heads):
global_head = ple_dense_layer_id * ngram_heads + local_head
size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1)
sizes.append(size)
offsets.append(offset)
offset += size
return sizes, offsets, offset
def __init__(
self,
config: Qwen4ExpTextConfig,
embedding_dim: int,
ple_dense_layer_id: int,
max_total_tokens: int,
*,
data_parallel_rank: int,
prefix: str,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.embedding_dim = embedding_dim
self.ngram_size = int(config.ngram_size)
self.heads_per_ngram = int(config.heads_per_ngram)
self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
if self.ngram_size < 2:
raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
if self.heads_per_ngram <= 0:
raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}")
if embedding_dim % self.ngram_heads:
raise ValueError(
"ple_embed_dim must be divisible by total ngram heads: "
f"{embedding_dim} % {self.ngram_heads} != 0"
)
self.head_dim = embedding_dim // self.ngram_heads
self.eos_token_id = int(config.eos_token_id)
self.unigram_vocab_size = int(config.vocab_size)
self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512))
if self.split_ngram_parts <= 0:
raise ValueError("split_ngram_parts must be positive")
multipliers = self._make_layer_multipliers(
ngram_size=self.ngram_size,
unigram_vocab_size=self.unigram_vocab_size,
seed=int(getattr(config, "seed", 1234)),
ple_dense_layer_id=ple_dense_layer_id,
)
self.register_buffer(
"layer_multipliers",
torch.tensor(multipliers, dtype=torch.long),
persistent=True,
)
sizes, offsets, total_vocab_size = self._make_vocab_layout(
ngram_vocab_size_base=int(config.ngram_vocab_size_base),
ngram_heads=self.ngram_heads,
ple_dense_layer_id=ple_dense_layer_id,
)
self.register_buffer(
"ngram_heads_vocab_sizes",
torch.tensor(sizes, dtype=torch.long),
persistent=True,
)
self.register_buffer(
"ngram_heads_offsets",
torch.tensor(offsets, dtype=torch.long),
persistent=True,
)
divisor = int(config.make_ngram_vocab_size_divisible_by)
padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor
embedding_prefix = f"{prefix}.ngram_embedding"
embedding_quant_method = Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
embedding_prefix,
getattr(config, "ple_embedding_dtype", None),
)
if params_dtype is None:
params_dtype = torch.get_default_dtype()
engram_config = get_current_vllm_config().engram_config
embedding_cls = (
Qwen4ExpPLEPinnedHostEmbedding
if engram_config is not None and engram_config.cpu_offload
else Qwen4ExpPLEDeviceEmbedding
)
self.ngram_embedding = embedding_cls(
padded_vocab_size,
self.head_dim,
params_dtype=params_dtype,
padding_size=divisor,
prefix=embedding_prefix,
embedding_method=embedding_quant_method,
num_ngram_heads=self.ngram_heads,
max_total_tokens=max_total_tokens,
data_parallel_rank=data_parallel_rank,
)
weight = self.ngram_embedding.weight
logger.info(
"Initialized PLE embedding %s: quantization_method=%s, "
"weight_dtype=%s, weight_device=%s, pinned=%s",
embedding_prefix,
type(embedding_quant_method).__name__,
weight.dtype,
weight.device,
weight.is_pinned(),
)
@staticmethod
def _shift_precompute(
tokens: torch.Tensor, eos_token_id: int
) -> tuple[torch.Tensor, torch.Tensor]:
if tokens.dim() != 2:
raise ValueError("tokens must be a 2D tensor")
batch_size, seq_len = tokens.shape
positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64)
eos_positions = torch.where(tokens == eos_token_id, positions, -1)
previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values
previous_eos = torch.cat(
[
eos_positions.new_full((batch_size, 1), -1),
previous_eos_inclusive[:, :-1],
],
dim=1,
)
return positions, positions.unsqueeze(0) - previous_eos - 1
@staticmethod
def _shift_apply(
tokens: torch.Tensor,
positions: torch.Tensor,
position_in_segment: torch.Tensor,
shift: int,
eos_token_id: int,
) -> torch.Tensor:
if shift == 0:
return tokens
source = positions - shift
gather_indices = source.clamp_min(0).unsqueeze(0).expand(tokens.shape[0], -1)
shifted = tokens.gather(1, gather_indices)
valid = (source.unsqueeze(0) >= 0) & (position_in_segment >= shift)
return torch.where(valid, shifted, tokens.new_full((), eos_token_id))
def compute_ngram_ids(
self,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
output: torch.Tensor | None = None,
) -> torch.Tensor:
"""Compute n-gram embedding indices for the current request layout."""
input_ids = input_ids.reshape(-1)
num_reqs = query_start_loc.numel() - 1
num_tokens = input_ids.shape[0]
if input_ids.is_cuda:
return ple_ngram_ids(
input_ids=input_ids,
query_start_loc=query_start_loc,
ngram_context=ngram_context,
layer_multipliers=self.layer_multipliers,
ngram_heads_vocab_sizes=self.ngram_heads_vocab_sizes,
ngram_heads_offsets=self.ngram_heads_offsets,
eos_token_id=self.eos_token_id,
heads_per_ngram=self.heads_per_ngram,
output=output,
)
input_ids = input_ids.long()
query_start_loc = query_start_loc.long()
positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64)
packed = torch.full(
(num_reqs, num_tokens),
self.eos_token_id,
device=input_ids.device,
dtype=torch.int64,
)
request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1
request_indices.clamp_(max=num_reqs - 1)
columns = (positions - query_start_loc[request_indices]).clamp(
0, packed.shape[1] - 1
)
packed[request_indices, columns] = input_ids
ngram_context = ngram_context[:num_reqs].to(
device=input_ids.device, dtype=torch.long
)
context = torch.cat([ngram_context, packed], dim=-1)
positions_2d, position_in_segment = self._shift_precompute(
context, self.eos_token_id
)
shifted = [context]
for shift in range(1, self.ngram_size):
shifted.append(
self._shift_apply(
context,
positions_2d,
position_in_segment,
shift,
self.eos_token_id,
)
)
adjusted_columns = columns + self.ngram_size - 1
id_blocks = []
for ngram in range(2, self.ngram_size + 1):
start = (ngram - 2) * self.heads_per_ngram
end = start + self.heads_per_ngram
mixed = shifted[0] * self.layer_multipliers[0]
for index in range(1, ngram):
mixed = torch.bitwise_xor(
mixed, shifted[index] * self.layer_multipliers[index]
)
sizes = self.ngram_heads_vocab_sizes[start:end]
offsets = self.ngram_heads_offsets[start:end]
ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets
id_blocks.append(ids[request_indices, adjusted_columns])
return torch.cat(id_blocks, dim=-1)
def forward(
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
embedding = self.ngram_embedding
if embedding.supports_prefetch:
return embedding(hidden_states)
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
return self.ngram_embedding(ngram_ids).flatten(-2)
def start_prefetch(
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> None:
"""Start the pinned lookup while the preceding decoder layer runs."""
embedding = self.ngram_embedding
if not embedding.supports_prefetch:
return
ngram_ids = self.compute_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
)
embedding.start_prefetch(hidden_states, ngram_ids)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load hash buffers and checkpoint-split embedding rows."""
persistent_buffers = {
"layer_multipliers": self.layer_multipliers,
"ngram_heads_offsets": self.ngram_heads_offsets,
"ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes,
}
loaded: set[str] = set()
regular_weights: list[tuple[str, torch.Tensor]] = []
shard_prefix = "ngram_embedding.shard_"
for name, loaded_weight in weights:
leaf_name = name.rsplit(".", 1)[-1]
if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup":
continue
if name in persistent_buffers:
buffer = persistent_buffers[name]
if buffer.shape != loaded_weight.shape:
raise ValueError(
f"Shape mismatch for {name}: expected "
f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}"
)
buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype))
loaded.add(name)
continue
if name.startswith(shard_prefix) and name.endswith(".weight"):
shard_text = name[len(shard_prefix) : -len(".weight")]
if not shard_text.isdigit():
regular_weights.append((name, loaded_weight))
continue
shard_index = int(shard_text)
if shard_index >= self.split_ngram_parts:
raise ValueError(
f"PLE embedding shard index {shard_index} exceeds "
f"split_ngram_parts={self.split_ngram_parts}"
)
embedding = self.ngram_embedding
shard_size = (
embedding.org_vocab_size + self.split_ngram_parts - 1
) // self.split_ngram_parts
checkpoint_start = shard_index * shard_size
expected_rows = max(
0,
min(shard_size, embedding.org_vocab_size - checkpoint_start),
)
expected_shape = (expected_rows, embedding.embedding_dim)
if tuple(loaded_weight.shape) != expected_shape:
raise ValueError(
f"Shape mismatch for PLE embedding shard {shard_index}: "
f"expected {expected_shape}, got "
f"{tuple(loaded_weight.shape)}"
)
embedding.weight.weight_loader(
embedding.weight,
loaded_weight,
checkpoint_start=checkpoint_start,
)
loaded.add("ngram_embedding.weight")
continue
regular_weights.append((name, loaded_weight))
if regular_weights:
loaded.update(AutoWeightsLoader(self).load_weights(regular_weights))
return loaded
__all__ = [
"Qwen4ExpNGramEmbedding",
]
+30 -467
View File
@@ -2,10 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GPU-resident Qwen4Exp position-learning enhancement layers."""
from collections.abc import Iterable, Sequence
from collections.abc import Sequence
import torch
import torch.nn.functional as F
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
@@ -18,24 +17,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptMixedPrecisionConfig,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
create_fp8_scale_parameter,
create_fp8_weight_parameter,
is_fp8,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.model_executor.parameter import PerTensorScaleParameter
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
@@ -45,8 +26,8 @@ from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionMetadata,
)
from ..common.ple import PLEVocabParallelEmbedding
from .ops.ple import ple_conv, ple_gate, ple_ngram_ids
from .ngram_embedding import Qwen4ExpNGramEmbedding
from .ops.ple import ple_conv, ple_gate
class Qwen4ExpPLEGroupedNorm(nn.Module):
@@ -82,437 +63,6 @@ class Qwen4ExpPLEGroupedNorm(nn.Module):
return (normalized * (1.0 + self.weight.float())).to(input_dtype)
class Qwen4ExpPLEFp8EmbeddingMethod(QuantizeMethodBase):
"""FP8 PLE embedding with one global checkpoint scale."""
def create_weights(
self,
layer: nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size, params_dtype
weight_loader = extra_weight_attrs.get("weight_loader")
weight = create_fp8_weight_parameter(
sum(output_partition_sizes), input_size_per_partition, weight_loader
)
layer.register_parameter("weight", weight)
weight_scale = create_fp8_scale_parameter(
PerTensorScaleParameter,
output_partition_sizes,
input_size_per_partition,
None,
weight_loader,
scale_dtype=torch.float32,
)
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: nn.Module) -> None:
"""Reject FP8 PLE checkpoints without a global scale."""
sentinel = torch.finfo(torch.float32).min
if torch.any(layer.weight_scale == sentinel):
raise ValueError("FP8 PLE checkpoint is missing its global scale")
def apply(
self,
layer: nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
raise NotImplementedError("PLE FP8 weights only support embedding lookup")
def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor:
return F.embedding(input_, layer.weight)
def _get_ple_embedding_quant_method(
quant_config: QuantizationConfig | None,
prefix: str,
) -> QuantizeMethodBase | None:
"""Select global-scale FP8 only for quantized PLE checkpoint shards."""
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
if quant_config._resolve_quant_algo(prefix) == "FP8":
return Qwen4ExpPLEFp8EmbeddingMethod()
return None
if not isinstance(quant_config, Fp8Config):
return None
if not quant_config.is_checkpoint_fp8_serialized:
return None
ignored_layers = quant_config.ignored_layers
if is_layer_skipped(
prefix,
ignored_layers,
quant_config.packed_modules_mapping,
match_mode=quant_config.ignored_layers_match_mode,
):
return None
# PLE checkpoint shards form one runtime embedding parameter.
shard_prefix = f"{prefix}.shard_"
if any(name.startswith(shard_prefix) for name in ignored_layers):
return None
return Qwen4ExpPLEFp8EmbeddingMethod()
class Qwen4ExpNGramEmbedding(nn.Module):
_MASK64 = (1 << 64) - 1
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
_SPLITMIX_M1 = 0xBF58476D1CE4E5B9
_SPLITMIX_M2 = 0x94D049BB133111EB
_PLE_LAYER_PRIME = 10007
@classmethod
def _splitmix64(cls, value: int) -> int:
"""Mix an integer into a deterministic unsigned 64-bit value."""
value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64
value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64
value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64
return (value ^ (value >> 31)) & cls._MASK64
@staticmethod
def _is_prime_64(value: int) -> bool:
"""Return whether a 64-bit integer is prime."""
if value < 2:
return False
for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37):
if value % prime == 0:
return value == prime
exponent = value - 1
shifts = 0
while exponent % 2 == 0:
exponent //= 2
shifts += 1
for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022):
if base % value == 0:
continue
witness = pow(base, exponent, value)
if witness in (1, value - 1):
continue
for _ in range(shifts - 1):
witness = pow(witness, 2, value)
if witness == value - 1:
break
else:
return False
return True
@classmethod
def _nth_prime_after(cls, start: int, count: int) -> int:
"""Return the ``count``-th prime strictly greater than ``start``."""
prime = int(start)
for _ in range(count):
candidate = prime + 1
if candidate <= 2:
prime = 2
continue
if candidate % 2 == 0:
candidate += 1
while not cls._is_prime_64(candidate):
candidate += 2
prime = candidate
return prime
@classmethod
def _make_layer_multipliers(
cls,
*,
ngram_size: int,
unigram_vocab_size: int,
seed: int,
ple_dense_layer_id: int,
) -> list[int]:
"""Build deterministic hash multipliers for one PLE layer."""
max_multiplier = ((1 << 63) - 1) // unigram_vocab_size
half_bound = max(1, max_multiplier // 2)
base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id
multipliers = []
for index in range(ngram_size):
value = base_seed + cls._SPLITMIX_GAMMA * (index + 1)
multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1)
return multipliers
@classmethod
def _make_vocab_layout(
cls,
*,
ngram_vocab_size_base: int,
ngram_heads: int,
ple_dense_layer_id: int,
) -> tuple[list[int], list[int], int]:
"""Build per-head vocabulary sizes, offsets, and total row count."""
sizes: list[int] = []
offsets: list[int] = []
offset = 0
for local_head in range(ngram_heads):
global_head = ple_dense_layer_id * ngram_heads + local_head
size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1)
sizes.append(size)
offsets.append(offset)
offset += size
return sizes, offsets, offset
def __init__(
self,
config: Qwen4ExpTextConfig,
embedding_dim: int,
ple_dense_layer_id: int,
prefix: str,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.embedding_dim = embedding_dim
self.ngram_size = int(config.ngram_size)
self.heads_per_ngram = int(config.heads_per_ngram)
self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
if self.ngram_size < 2:
raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
if self.heads_per_ngram <= 0:
raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}")
if embedding_dim % self.ngram_heads:
raise ValueError(
"ple_embed_dim must be divisible by total ngram heads: "
f"{embedding_dim} % {self.ngram_heads} != 0"
)
self.head_dim = embedding_dim // self.ngram_heads
self.eos_token_id = int(config.eos_token_id)
self.unigram_vocab_size = int(config.vocab_size)
self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512))
if self.split_ngram_parts <= 0:
raise ValueError("split_ngram_parts must be positive")
multipliers = self._make_layer_multipliers(
ngram_size=self.ngram_size,
unigram_vocab_size=self.unigram_vocab_size,
seed=int(getattr(config, "seed", 1234)),
ple_dense_layer_id=ple_dense_layer_id,
)
self.register_buffer(
"layer_multipliers",
torch.tensor(multipliers, dtype=torch.long),
persistent=True,
)
sizes, offsets, total_vocab_size = self._make_vocab_layout(
ngram_vocab_size_base=int(config.ngram_vocab_size_base),
ngram_heads=self.ngram_heads,
ple_dense_layer_id=ple_dense_layer_id,
)
self.register_buffer(
"ngram_heads_vocab_sizes",
torch.tensor(sizes, dtype=torch.long),
persistent=True,
)
self.register_buffer(
"ngram_heads_offsets",
torch.tensor(offsets, dtype=torch.long),
persistent=True,
)
divisor = int(config.make_ngram_vocab_size_divisible_by)
padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor
self.ngram_embedding = PLEVocabParallelEmbedding(
padded_vocab_size,
self.head_dim,
params_dtype=params_dtype,
padding_size=divisor,
prefix=f"{prefix}.ngram_embedding",
quant_method=_get_ple_embedding_quant_method(
quant_config, f"{prefix}.ngram_embedding"
),
)
@staticmethod
def _shift_precompute(
tokens: torch.Tensor, eos_token_id: int
) -> tuple[torch.Tensor, torch.Tensor]:
if tokens.dim() != 2:
raise ValueError("tokens must be a 2D tensor")
batch_size, seq_len = tokens.shape
positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64)
eos_positions = torch.where(tokens == eos_token_id, positions, -1)
previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values
previous_eos = torch.cat(
[
eos_positions.new_full((batch_size, 1), -1),
previous_eos_inclusive[:, :-1],
],
dim=1,
)
return positions, positions.unsqueeze(0) - previous_eos - 1
@staticmethod
def _shift_apply(
tokens: torch.Tensor,
positions: torch.Tensor,
position_in_segment: torch.Tensor,
shift: int,
eos_token_id: int,
) -> torch.Tensor:
if shift == 0:
return tokens
source = positions - shift
gather_indices = source.clamp_min(0).unsqueeze(0).expand(tokens.shape[0], -1)
shifted = tokens.gather(1, gather_indices)
valid = (source.unsqueeze(0) >= 0) & (position_in_segment >= shift)
return torch.where(valid, shifted, tokens.new_full((), eos_token_id))
def compute_ngram_ids(
self,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
output: torch.Tensor | None = None,
) -> torch.Tensor:
"""Compute n-gram embedding indices for the current request layout."""
input_ids = input_ids.reshape(-1)
num_reqs = query_start_loc.numel() - 1
num_tokens = input_ids.shape[0]
if input_ids.is_cuda:
return ple_ngram_ids(
input_ids=input_ids,
query_start_loc=query_start_loc,
ngram_context=ngram_context,
layer_multipliers=self.layer_multipliers,
ngram_heads_vocab_sizes=self.ngram_heads_vocab_sizes,
ngram_heads_offsets=self.ngram_heads_offsets,
eos_token_id=self.eos_token_id,
heads_per_ngram=self.heads_per_ngram,
output=output,
)
input_ids = input_ids.long()
query_start_loc = query_start_loc.long()
positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64)
packed = torch.full(
(num_reqs, num_tokens),
self.eos_token_id,
device=input_ids.device,
dtype=torch.int64,
)
request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1
request_indices.clamp_(max=num_reqs - 1)
columns = (positions - query_start_loc[request_indices]).clamp(
0, packed.shape[1] - 1
)
packed[request_indices, columns] = input_ids
ngram_context = ngram_context[:num_reqs].to(
device=input_ids.device, dtype=torch.long
)
context = torch.cat([ngram_context, packed], dim=-1)
positions_2d, position_in_segment = self._shift_precompute(
context, self.eos_token_id
)
shifted = [context]
for shift in range(1, self.ngram_size):
shifted.append(
self._shift_apply(
context,
positions_2d,
position_in_segment,
shift,
self.eos_token_id,
)
)
adjusted_columns = columns + self.ngram_size - 1
id_blocks = []
for ngram in range(2, self.ngram_size + 1):
start = (ngram - 2) * self.heads_per_ngram
end = start + self.heads_per_ngram
mixed = shifted[0] * self.layer_multipliers[0]
for index in range(1, ngram):
mixed = torch.bitwise_xor(
mixed, shifted[index] * self.layer_multipliers[index]
)
sizes = self.ngram_heads_vocab_sizes[start:end]
offsets = self.ngram_heads_offsets[start:end]
ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets
id_blocks.append(ids[request_indices, adjusted_columns])
return torch.cat(id_blocks, dim=-1)
def forward(
self,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
return self.ngram_embedding(ngram_ids).flatten(-2)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load hash buffers and checkpoint-split embedding rows."""
persistent_buffers = {
"layer_multipliers": self.layer_multipliers,
"ngram_heads_offsets": self.ngram_heads_offsets,
"ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes,
}
loaded: set[str] = set()
regular_weights: list[tuple[str, torch.Tensor]] = []
shard_prefix = "ngram_embedding.shard_"
for name, loaded_weight in weights:
leaf_name = name.rsplit(".", 1)[-1]
if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup":
continue
if name in persistent_buffers:
buffer = persistent_buffers[name]
if buffer.shape != loaded_weight.shape:
raise ValueError(
f"Shape mismatch for {name}: expected "
f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}"
)
buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype))
loaded.add(name)
continue
if name.startswith(shard_prefix) and name.endswith(".weight"):
shard_text = name[len(shard_prefix) : -len(".weight")]
if not shard_text.isdigit():
regular_weights.append((name, loaded_weight))
continue
shard_index = int(shard_text)
if shard_index >= self.split_ngram_parts:
raise ValueError(
f"PLE embedding shard index {shard_index} exceeds "
f"split_ngram_parts={self.split_ngram_parts}"
)
embedding = self.ngram_embedding
shard_size = (
embedding.org_vocab_size + self.split_ngram_parts - 1
) // self.split_ngram_parts
checkpoint_start = shard_index * shard_size
expected_rows = max(
0,
min(shard_size, embedding.org_vocab_size - checkpoint_start),
)
expected_shape = (expected_rows, embedding.embedding_dim)
if tuple(loaded_weight.shape) != expected_shape:
raise ValueError(
f"Shape mismatch for PLE embedding shard {shard_index}: "
f"expected {expected_shape}, got "
f"{tuple(loaded_weight.shape)}"
)
embedding.weight.weight_loader(
embedding.weight,
loaded_weight,
checkpoint_start=checkpoint_start,
)
loaded.add("ngram_embedding.weight")
continue
regular_weights.append((name, loaded_weight))
if regular_weights:
loaded.update(AutoWeightsLoader(self).load_weights(regular_weights))
return loaded
class Qwen4ExpPLELayer(nn.Module, MambaBase):
def __init__(
self,
@@ -547,6 +97,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
config,
int(config.ple_embed_dim),
self.ple_dense_layer_id,
vllm_config.scheduler_config.max_num_batched_tokens,
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
prefix=f"{prefix}.ple_embedding",
quant_config=quant_config,
params_dtype=model_config.dtype,
@@ -588,10 +140,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def _get_embedding_weight_scale(self) -> torch.Tensor | None:
embedding = getattr(self.ple_embedding, "ngram_embedding", None)
return getattr(embedding, "weight_scale", None)
def _dequantize_embeddings(
self,
embeddings: torch.Tensor,
@@ -599,14 +147,25 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
) -> torch.Tensor:
"""Dequantize PLE lookup output."""
if not is_fp8(embeddings):
return embeddings
weight_scale = self._get_embedding_weight_scale()
if weight_scale is None:
raise RuntimeError("FP8 PLE embedding is missing its global scale")
if weight_scale.device != embeddings.device:
raise RuntimeError("FP8 PLE embedding scale must be on the output device")
return embeddings.to(output_dtype) * weight_scale.to(output_dtype)
return self.ple_embedding.ngram_embedding.dequantize(
embeddings,
output_dtype,
)
def start_prefetch(
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> None:
"""Start the pinned PLE lookup while the preceding decoder layer runs."""
self.ple_embedding.start_prefetch(
hidden_states,
input_ids,
query_start_loc,
ngram_context,
)
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
@@ -827,7 +386,12 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
f"token length, got {input_ids.shape[0]} and "
f"{hidden_states.shape[0]}"
)
embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context)
embeddings = self.ple_embedding(
hidden_states,
input_ids,
query_start_loc,
ngram_context,
)
embeddings = self._dequantize_embeddings(embeddings, hidden_states.dtype)
kv, _ = self.kv_proj(embeddings)
key, value = kv.split(self.kv_proj.output_sizes, dim=-1)
@@ -845,7 +409,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
__all__ = [
"Qwen4ExpNGramEmbedding",
"Qwen4ExpPLEGroupedNorm",
"Qwen4ExpPLELayer",
]