[Qwen4Exp] Support UVA PLE-offload and Engram tensor parallelism (#54371)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
62f3bf58a5
commit
3116c5d06b
@@ -13,27 +13,50 @@ from torch.nn import functional as F
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
import vllm.models.qwen4_exp.nvidia.ngram_embedding as ngram_embedding_module
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptMixedPrecisionConfig,
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
from vllm.models.qwen4_exp.common.ple import (
|
||||
PLEShardOverlap,
|
||||
compute_ple_shard_overlap,
|
||||
copy_ple_embedding_shard_,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import (
|
||||
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
Qwen4ExpPLEEmbeddingMethod,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLELayer,
|
||||
_get_ple_embedding_quant_method,
|
||||
Qwen4ExpPLEPinnedHostEmbedding,
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def _mock_etp_group(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
world_size: int = 1,
|
||||
all_reduce=lambda tensor: tensor,
|
||||
) -> None:
|
||||
group = SimpleNamespace(
|
||||
rank_in_group=0,
|
||||
world_size=world_size,
|
||||
all_reduce=all_reduce,
|
||||
)
|
||||
monkeypatch.setattr(ngram_embedding_module, "get_etp_group", lambda: group)
|
||||
monkeypatch.setattr(
|
||||
ngram_embedding_module,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(world_size=world_size),
|
||||
)
|
||||
|
||||
|
||||
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
|
||||
nn.Module.__init__(module)
|
||||
@@ -171,11 +194,57 @@ def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("dp_rank", "local_tokens"), [(0, 2), (1, 3)])
|
||||
def test_etp_lookup_gathers_and_returns_dp_local_rows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dp_rank: int,
|
||||
local_tokens: int,
|
||||
) -> None:
|
||||
embedding = Qwen4ExpPLEDeviceEmbedding.__new__(Qwen4ExpPLEDeviceEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.tp_size = 2
|
||||
embedding.etp_data_parallel_size = 2
|
||||
embedding.data_parallel_rank = dp_rank
|
||||
gathered_ids = torch.tensor([[10], [11], [0], [20], [21], [22]])
|
||||
group = SimpleNamespace(
|
||||
rank_in_group=dp_rank,
|
||||
all_gather=lambda input_ids, dim: gathered_ids,
|
||||
all_reduce=lambda embeddings: embeddings,
|
||||
)
|
||||
embedding.parallel_group = group
|
||||
forward_context = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(
|
||||
num_tokens_across_dp_cpu=torch.tensor([2, 3]),
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(ngram_embedding_module, "get_dp_group", lambda: group)
|
||||
monkeypatch.setattr(
|
||||
ngram_embedding_module, "get_forward_context", lambda: forward_context
|
||||
)
|
||||
local_ids = gathered_ids[dp_rank * 3 : dp_rank * 3 + local_tokens]
|
||||
embeddings = torch.arange(12).reshape(6, 2)
|
||||
|
||||
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
assert torch.equal(input_ids, gathered_ids)
|
||||
return embeddings
|
||||
|
||||
monkeypatch.setattr(
|
||||
ngram_embedding_module.PLEVocabParallelEmbedding,
|
||||
"forward",
|
||||
forward,
|
||||
)
|
||||
output = embedding(local_ids)
|
||||
|
||||
expected = embeddings[dp_rank * 3 : dp_rank * 3 + local_tokens]
|
||||
assert torch.equal(output, expected)
|
||||
|
||||
|
||||
def _make_fp8_embedding_layer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
load_scale: bool = True,
|
||||
) -> embedding_module.VocabParallelEmbedding:
|
||||
) -> Qwen4ExpPLEDeviceEmbedding:
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
@@ -185,12 +254,13 @@ def _make_fp8_embedding_layer(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
method = Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
layer = Qwen4ExpPLEDeviceEmbedding(
|
||||
3,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=method,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=method,
|
||||
)
|
||||
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
|
||||
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
|
||||
@@ -228,7 +298,58 @@ def test_ple_fp8_embedding_rejects_missing_global_scale(monkeypatch) -> None:
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
|
||||
@pytest.mark.parametrize("requires_device_loading", [False, True])
|
||||
@pytest.mark.parametrize("load_scale", [False, True])
|
||||
def test_pinned_ple_post_load_validates_scale_without_staging(
|
||||
monkeypatch: pytest.MonkeyPatch, requires_device_loading: bool, load_scale: bool
|
||||
) -> None:
|
||||
"""Validate scales through the loader while honoring the staging policy."""
|
||||
from contextlib import nullcontext
|
||||
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader import utils as loader_utils
|
||||
|
||||
source = _make_fp8_embedding_layer(monkeypatch, load_scale=load_scale)
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.weight = source.weight
|
||||
embedding.weight_scale = source.weight_scale
|
||||
embedding.embedding_method = source.embedding_method
|
||||
embedding.quant_method = source.embedding_method
|
||||
embedding.tp_rank = source.tp_rank
|
||||
embedding.tp_size = source.tp_size
|
||||
assert QuantizeMethodBase.requires_device_loading
|
||||
assert not embedding.quant_method.requires_device_loading
|
||||
if requires_device_loading:
|
||||
embedding.quant_method.requires_device_loading = True
|
||||
model = nn.Module()
|
||||
model.embedding = embedding
|
||||
weight_ptr = embedding.weight.data_ptr()
|
||||
staged_modules = []
|
||||
|
||||
def track_staging(module, target_device):
|
||||
staged_modules.append(module)
|
||||
return nullcontext()
|
||||
|
||||
monkeypatch.setattr(loader_utils, "device_loading_context", track_staging)
|
||||
monkeypatch.setattr(loader_utils, "maybe_retie_word_embeddings", lambda *args: None)
|
||||
monkeypatch.setattr(
|
||||
loader_utils, "release_device_memory_under_pressure", lambda *args: None
|
||||
)
|
||||
config = SimpleNamespace(quantization="fp8")
|
||||
if load_scale:
|
||||
loader_utils.process_weights_after_loading(model, config, torch.device("cuda"))
|
||||
else:
|
||||
with pytest.raises(ValueError, match="missing its global scale"):
|
||||
loader_utils.process_weights_after_loading(
|
||||
model, config, torch.device("cuda")
|
||||
)
|
||||
assert embedding.weight.device.type == "cpu"
|
||||
assert embedding.weight.data_ptr() == weight_ptr
|
||||
assert staged_modules == ([embedding] if requires_device_loading else [])
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_uses_int8_for_parallel_reduce(monkeypatch) -> None:
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
@@ -251,17 +372,14 @@ def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
return tensor.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding_module,
|
||||
"tensor_model_parallel_all_reduce",
|
||||
all_reduce,
|
||||
)
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
_mock_etp_group(monkeypatch, world_size=2, all_reduce=all_reduce)
|
||||
layer = Qwen4ExpPLEDeviceEmbedding(
|
||||
4,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=Qwen4ExpPLEFp8EmbeddingMethod(),
|
||||
)
|
||||
layer.weight.data.copy_(
|
||||
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
|
||||
@@ -283,12 +401,200 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
|
||||
weight_block_size=[128, 128],
|
||||
)
|
||||
assert isinstance(
|
||||
_get_ple_embedding_quant_method(quant_config, prefix),
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
|
||||
quant_config.ignored_layers = [f"{prefix}.shard_0"]
|
||||
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(None, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
nvfp4_config = ModelOptNvFp4Config(exclude_modules=[])
|
||||
with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
|
||||
|
||||
dynamic_fp8_config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=False,
|
||||
ignored_layers=[],
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="serialized FP8"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(dynamic_fp8_config, prefix)
|
||||
|
||||
|
||||
def test_ple_embedding_respects_modelopt_exclusion() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
prefix,
|
||||
"float8_e4m3fn",
|
||||
),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_pinned_embedding_forward_finalizes_prefetched_output(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = torch.empty(4, 2, 3, dtype=torch.float8_e4m3fn)
|
||||
embedding._output_dim = 6
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
|
||||
expected = torch.arange(12).reshape(2, 6).to(torch.float8_e4m3fn)
|
||||
|
||||
def finalize_prefetched(
|
||||
prefetch_output: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
assert prefetch_output is embedding._prefetch_buffer
|
||||
output.copy_(expected)
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding,
|
||||
"_finalize_prefetch",
|
||||
finalize_prefetched,
|
||||
)
|
||||
|
||||
output = embedding(hidden_states)
|
||||
|
||||
assert output.dtype == embedding._prefetch_buffer.dtype
|
||||
assert torch.equal(output, expected)
|
||||
|
||||
|
||||
def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None:
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.tp_size = 2
|
||||
reduced_dtypes = []
|
||||
|
||||
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
return tensor.clone()
|
||||
|
||||
embedding.parallel_group = SimpleNamespace(all_reduce=all_reduce)
|
||||
embeddings = torch.arange(8).reshape(2, 4).to(torch.float8_e4m3fn)
|
||||
|
||||
output = embedding._reduce_etp_embeddings(embeddings)
|
||||
|
||||
assert reduced_dtypes == [torch.int8]
|
||||
assert output.dtype == embeddings.dtype
|
||||
torch.testing.assert_close(output.float(), embeddings.float())
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_ple_device_embedding_allocates_on_active_device(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
embedding_method = Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
|
||||
with torch.device("cuda:0"):
|
||||
embedding = Qwen4ExpPLEDeviceEmbedding(
|
||||
4,
|
||||
3,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=embedding_method,
|
||||
)
|
||||
|
||||
assert embedding.weight.device == torch.device("cuda:0")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("fp8_checkpoint", [False, True])
|
||||
def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_checkpoint: bool,
|
||||
) -> None:
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
|
||||
with torch.device("cuda:0"):
|
||||
embedding_method = (
|
||||
Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
if fp8_checkpoint
|
||||
else Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
)
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding(
|
||||
4,
|
||||
3,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=embedding_method,
|
||||
)
|
||||
|
||||
storage_dtype = torch.float8_e4m3fn if fp8_checkpoint else torch.bfloat16
|
||||
loaded_weight = (
|
||||
torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype)
|
||||
)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=0,
|
||||
tp_start=0,
|
||||
tp_end=4,
|
||||
)
|
||||
input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
output = embedding._lookup(input_ids)
|
||||
if fp8_checkpoint:
|
||||
embedding.weight_scale.data.fill_(0.25)
|
||||
weight_ptr = embedding.weight.data_ptr()
|
||||
embedding.quant_method.process_weights_after_loading(embedding)
|
||||
assert embedding.weight.data_ptr() == weight_ptr
|
||||
dequantized = embedding.dequantize(output, torch.bfloat16)
|
||||
|
||||
assert copied == 4
|
||||
assert embedding.weight.device.type == "cpu"
|
||||
assert embedding.weight.is_pinned()
|
||||
assert output.dtype == storage_dtype
|
||||
expected = loaded_weight[input_ids.cpu()].to(device="cuda:0")
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
expected_scale = 0.25 if fp8_checkpoint else 1.0
|
||||
torch.testing.assert_close(
|
||||
dequantized,
|
||||
expected.to(torch.bfloat16) * expected_scale,
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
@@ -310,15 +616,15 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
_get_ple_embedding_quant_method(quant_config, prefix),
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
assert (
|
||||
_get_ple_embedding_quant_method(
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
"model.language_model.layers.2.moe.gate_proj",
|
||||
)
|
||||
is None
|
||||
),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user