[Qwen4Exp] Support UVA PLE-offload and Engram tensor parallelism (#54371)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Huang
2026-09-09 07:32:34 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 62f3bf58a5
commit 3116c5d06b
16 changed files with 1712 additions and 495 deletions
+327 -21
View File
@@ -13,27 +13,50 @@ from torch.nn import functional as F
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
import vllm.models.qwen4_exp.nvidia.ngram_embedding as ngram_embedding_module
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptMixedPrecisionConfig,
ModelOptNvFp4Config,
)
from vllm.models.qwen4_exp.common.ple import (
PLEShardOverlap,
compute_ple_shard_overlap,
copy_ple_embedding_shard_,
)
from vllm.models.qwen4_exp.nvidia.ple_layer import (
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpNGramEmbedding,
Qwen4ExpPLEDeviceEmbedding,
Qwen4ExpPLEEmbeddingMethod,
Qwen4ExpPLEFp8EmbeddingMethod,
Qwen4ExpPLELayer,
_get_ple_embedding_quant_method,
Qwen4ExpPLEPinnedHostEmbedding,
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionMetadata,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
def _mock_etp_group(
monkeypatch: pytest.MonkeyPatch,
world_size: int = 1,
all_reduce=lambda tensor: tensor,
) -> None:
group = SimpleNamespace(
rank_in_group=0,
world_size=world_size,
all_reduce=all_reduce,
)
monkeypatch.setattr(ngram_embedding_module, "get_etp_group", lambda: group)
monkeypatch.setattr(
ngram_embedding_module,
"get_tp_group",
lambda: SimpleNamespace(world_size=world_size),
)
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
nn.Module.__init__(module)
@@ -171,11 +194,57 @@ def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
)
@pytest.mark.parametrize(("dp_rank", "local_tokens"), [(0, 2), (1, 3)])
def test_etp_lookup_gathers_and_returns_dp_local_rows(
monkeypatch: pytest.MonkeyPatch,
dp_rank: int,
local_tokens: int,
) -> None:
embedding = Qwen4ExpPLEDeviceEmbedding.__new__(Qwen4ExpPLEDeviceEmbedding)
nn.Module.__init__(embedding)
embedding.tp_size = 2
embedding.etp_data_parallel_size = 2
embedding.data_parallel_rank = dp_rank
gathered_ids = torch.tensor([[10], [11], [0], [20], [21], [22]])
group = SimpleNamespace(
rank_in_group=dp_rank,
all_gather=lambda input_ids, dim: gathered_ids,
all_reduce=lambda embeddings: embeddings,
)
embedding.parallel_group = group
forward_context = SimpleNamespace(
dp_metadata=SimpleNamespace(
num_tokens_across_dp_cpu=torch.tensor([2, 3]),
)
)
monkeypatch.setattr(ngram_embedding_module, "get_dp_group", lambda: group)
monkeypatch.setattr(
ngram_embedding_module, "get_forward_context", lambda: forward_context
)
local_ids = gathered_ids[dp_rank * 3 : dp_rank * 3 + local_tokens]
embeddings = torch.arange(12).reshape(6, 2)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
assert torch.equal(input_ids, gathered_ids)
return embeddings
monkeypatch.setattr(
ngram_embedding_module.PLEVocabParallelEmbedding,
"forward",
forward,
)
output = embedding(local_ids)
expected = embeddings[dp_rank * 3 : dp_rank * 3 + local_tokens]
assert torch.equal(output, expected)
def _make_fp8_embedding_layer(
monkeypatch: pytest.MonkeyPatch,
*,
load_scale: bool = True,
) -> embedding_module.VocabParallelEmbedding:
) -> Qwen4ExpPLEDeviceEmbedding:
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
@@ -185,12 +254,13 @@ def _make_fp8_embedding_layer(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
method = Qwen4ExpPLEFp8EmbeddingMethod()
layer = embedding_module.VocabParallelEmbedding(
layer = Qwen4ExpPLEDeviceEmbedding(
3,
2,
params_dtype=torch.bfloat16,
padding_size=1,
quant_method=method,
prefix="test.ple_embedding",
embedding_method=method,
)
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
@@ -228,7 +298,58 @@ def test_ple_fp8_embedding_rejects_missing_global_scale(monkeypatch) -> None:
layer.quant_method.process_weights_after_loading(layer)
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
@pytest.mark.parametrize("requires_device_loading", [False, True])
@pytest.mark.parametrize("load_scale", [False, True])
def test_pinned_ple_post_load_validates_scale_without_staging(
monkeypatch: pytest.MonkeyPatch, requires_device_loading: bool, load_scale: bool
) -> None:
"""Validate scales through the loader while honoring the staging policy."""
from contextlib import nullcontext
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader import utils as loader_utils
source = _make_fp8_embedding_layer(monkeypatch, load_scale=load_scale)
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding.weight = source.weight
embedding.weight_scale = source.weight_scale
embedding.embedding_method = source.embedding_method
embedding.quant_method = source.embedding_method
embedding.tp_rank = source.tp_rank
embedding.tp_size = source.tp_size
assert QuantizeMethodBase.requires_device_loading
assert not embedding.quant_method.requires_device_loading
if requires_device_loading:
embedding.quant_method.requires_device_loading = True
model = nn.Module()
model.embedding = embedding
weight_ptr = embedding.weight.data_ptr()
staged_modules = []
def track_staging(module, target_device):
staged_modules.append(module)
return nullcontext()
monkeypatch.setattr(loader_utils, "device_loading_context", track_staging)
monkeypatch.setattr(loader_utils, "maybe_retie_word_embeddings", lambda *args: None)
monkeypatch.setattr(
loader_utils, "release_device_memory_under_pressure", lambda *args: None
)
config = SimpleNamespace(quantization="fp8")
if load_scale:
loader_utils.process_weights_after_loading(model, config, torch.device("cuda"))
else:
with pytest.raises(ValueError, match="missing its global scale"):
loader_utils.process_weights_after_loading(
model, config, torch.device("cuda")
)
assert embedding.weight.device.type == "cpu"
assert embedding.weight.data_ptr() == weight_ptr
assert staged_modules == ([embedding] if requires_device_loading else [])
def test_ple_fp8_embedding_uses_int8_for_parallel_reduce(monkeypatch) -> None:
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
@@ -251,17 +372,14 @@ def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
reduced_dtypes.append(tensor.dtype)
return tensor.clone()
monkeypatch.setattr(
embedding_module,
"tensor_model_parallel_all_reduce",
all_reduce,
)
layer = embedding_module.VocabParallelEmbedding(
_mock_etp_group(monkeypatch, world_size=2, all_reduce=all_reduce)
layer = Qwen4ExpPLEDeviceEmbedding(
4,
2,
params_dtype=torch.bfloat16,
padding_size=1,
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
prefix="test.ple_embedding",
embedding_method=Qwen4ExpPLEFp8EmbeddingMethod(),
)
layer.weight.data.copy_(
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
@@ -283,12 +401,200 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
weight_block_size=[128, 128],
)
assert isinstance(
_get_ple_embedding_quant_method(quant_config, prefix),
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
quant_config.ignored_layers = [f"{prefix}.shard_0"]
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(None, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
nvfp4_config = ModelOptNvFp4Config(exclude_modules=[])
with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
dynamic_fp8_config = Fp8Config(
is_checkpoint_fp8_serialized=False,
ignored_layers=[],
)
with pytest.raises(NotImplementedError, match="serialized FP8"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(dynamic_fp8_config, prefix)
def test_ple_embedding_respects_modelopt_exclusion() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
prefix,
"float8_e4m3fn",
),
Qwen4ExpPLEFp8EmbeddingMethod,
)
def test_pinned_embedding_forward_finalizes_prefetched_output(
monkeypatch: pytest.MonkeyPatch,
) -> None:
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding._prefetch_buffer = torch.empty(4, 2, 3, dtype=torch.float8_e4m3fn)
embedding._output_dim = 6
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
expected = torch.arange(12).reshape(2, 6).to(torch.float8_e4m3fn)
def finalize_prefetched(
prefetch_output: torch.Tensor,
output: torch.Tensor,
) -> None:
assert prefetch_output is embedding._prefetch_buffer
output.copy_(expected)
monkeypatch.setattr(
embedding,
"_finalize_prefetch",
finalize_prefetched,
)
output = embedding(hidden_states)
assert output.dtype == embedding._prefetch_buffer.dtype
assert torch.equal(output, expected)
def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None:
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding.tp_size = 2
reduced_dtypes = []
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
reduced_dtypes.append(tensor.dtype)
return tensor.clone()
embedding.parallel_group = SimpleNamespace(all_reduce=all_reduce)
embeddings = torch.arange(8).reshape(2, 4).to(torch.float8_e4m3fn)
output = embedding._reduce_etp_embeddings(embeddings)
assert reduced_dtypes == [torch.int8]
assert output.dtype == embeddings.dtype
torch.testing.assert_close(output.float(), embeddings.float())
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_ple_device_embedding_allocates_on_active_device(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
embedding_method = Qwen4ExpPLEUnquantizedEmbeddingMethod()
with torch.device("cuda:0"):
embedding = Qwen4ExpPLEDeviceEmbedding(
4,
3,
params_dtype=torch.bfloat16,
padding_size=1,
prefix="test.ple_embedding",
embedding_method=embedding_method,
)
assert embedding.weight.device == torch.device("cuda:0")
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("fp8_checkpoint", [False, True])
def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
monkeypatch: pytest.MonkeyPatch,
fp8_checkpoint: bool,
) -> None:
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
with torch.device("cuda:0"):
embedding_method = (
Qwen4ExpPLEFp8EmbeddingMethod()
if fp8_checkpoint
else Qwen4ExpPLEUnquantizedEmbeddingMethod()
)
embedding = Qwen4ExpPLEPinnedHostEmbedding(
4,
3,
params_dtype=torch.bfloat16,
padding_size=1,
prefix="test.ple_embedding",
embedding_method=embedding_method,
)
storage_dtype = torch.float8_e4m3fn if fp8_checkpoint else torch.bfloat16
loaded_weight = (
torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype)
)
copied = copy_ple_embedding_shard_(
embedding.weight,
loaded_weight,
checkpoint_start=0,
tp_start=0,
tp_end=4,
)
input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
output = embedding._lookup(input_ids)
if fp8_checkpoint:
embedding.weight_scale.data.fill_(0.25)
weight_ptr = embedding.weight.data_ptr()
embedding.quant_method.process_weights_after_loading(embedding)
assert embedding.weight.data_ptr() == weight_ptr
dequantized = embedding.dequantize(output, torch.bfloat16)
assert copied == 4
assert embedding.weight.device.type == "cpu"
assert embedding.weight.is_pinned()
assert output.dtype == storage_dtype
expected = loaded_weight[input_ids.cpu()].to(device="cuda:0")
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
expected_scale = 0.25 if fp8_checkpoint else 1.0
torch.testing.assert_close(
dequantized,
expected.to(torch.bfloat16) * expected_scale,
rtol=0,
atol=0,
)
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
@@ -310,15 +616,15 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
)
assert isinstance(
_get_ple_embedding_quant_method(quant_config, prefix),
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
assert (
_get_ple_embedding_quant_method(
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
"model.language_model.layers.2.moe.gate_proj",
)
is None
),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)