[Qwen4Exp] Support UVA PLE-offload and Engram tensor parallelism (#54371)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
62f3bf58a5
commit
3116c5d06b
@@ -46,6 +46,39 @@ def test_optional_type():
|
||||
assert optional_type_func("42") == 42
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options",
|
||||
[
|
||||
["--engram-config", '{"cpu_offload": false, "embedding_across_dp": true}'],
|
||||
[
|
||||
"--engram-config.cpu_offload",
|
||||
"false",
|
||||
"--engram-config.embedding_across_dp",
|
||||
"true",
|
||||
],
|
||||
],
|
||||
)
|
||||
def test_engram_config_cli(options, monkeypatch):
|
||||
"""CLI settings take precedence over the legacy offload environment."""
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
args = EngineArgs.from_cli_args(parser.parse_args(options))
|
||||
assert args.engram_config is not None
|
||||
assert args.engram_config.cpu_offload is False
|
||||
assert args.engram_config.embedding_across_dp is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options, provided",
|
||||
[([], False), (["--engram-config", "{}"], True)],
|
||||
)
|
||||
def test_engram_config_cli_optional(options, provided):
|
||||
"""An explicit empty config must remain distinct from an omitted config."""
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
args = EngineArgs.from_cli_args(parser.parse_args(options))
|
||||
assert (args.engram_config is not None) == provided
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("type_hint", "type", "expected"),
|
||||
[
|
||||
|
||||
@@ -13,27 +13,50 @@ from torch.nn import functional as F
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
import vllm.models.qwen4_exp.nvidia.ngram_embedding as ngram_embedding_module
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptMixedPrecisionConfig,
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
from vllm.models.qwen4_exp.common.ple import (
|
||||
PLEShardOverlap,
|
||||
compute_ple_shard_overlap,
|
||||
copy_ple_embedding_shard_,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import (
|
||||
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
Qwen4ExpPLEEmbeddingMethod,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLELayer,
|
||||
_get_ple_embedding_quant_method,
|
||||
Qwen4ExpPLEPinnedHostEmbedding,
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def _mock_etp_group(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
world_size: int = 1,
|
||||
all_reduce=lambda tensor: tensor,
|
||||
) -> None:
|
||||
group = SimpleNamespace(
|
||||
rank_in_group=0,
|
||||
world_size=world_size,
|
||||
all_reduce=all_reduce,
|
||||
)
|
||||
monkeypatch.setattr(ngram_embedding_module, "get_etp_group", lambda: group)
|
||||
monkeypatch.setattr(
|
||||
ngram_embedding_module,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(world_size=world_size),
|
||||
)
|
||||
|
||||
|
||||
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
|
||||
nn.Module.__init__(module)
|
||||
@@ -171,11 +194,57 @@ def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("dp_rank", "local_tokens"), [(0, 2), (1, 3)])
|
||||
def test_etp_lookup_gathers_and_returns_dp_local_rows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dp_rank: int,
|
||||
local_tokens: int,
|
||||
) -> None:
|
||||
embedding = Qwen4ExpPLEDeviceEmbedding.__new__(Qwen4ExpPLEDeviceEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.tp_size = 2
|
||||
embedding.etp_data_parallel_size = 2
|
||||
embedding.data_parallel_rank = dp_rank
|
||||
gathered_ids = torch.tensor([[10], [11], [0], [20], [21], [22]])
|
||||
group = SimpleNamespace(
|
||||
rank_in_group=dp_rank,
|
||||
all_gather=lambda input_ids, dim: gathered_ids,
|
||||
all_reduce=lambda embeddings: embeddings,
|
||||
)
|
||||
embedding.parallel_group = group
|
||||
forward_context = SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(
|
||||
num_tokens_across_dp_cpu=torch.tensor([2, 3]),
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(ngram_embedding_module, "get_dp_group", lambda: group)
|
||||
monkeypatch.setattr(
|
||||
ngram_embedding_module, "get_forward_context", lambda: forward_context
|
||||
)
|
||||
local_ids = gathered_ids[dp_rank * 3 : dp_rank * 3 + local_tokens]
|
||||
embeddings = torch.arange(12).reshape(6, 2)
|
||||
|
||||
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
assert torch.equal(input_ids, gathered_ids)
|
||||
return embeddings
|
||||
|
||||
monkeypatch.setattr(
|
||||
ngram_embedding_module.PLEVocabParallelEmbedding,
|
||||
"forward",
|
||||
forward,
|
||||
)
|
||||
output = embedding(local_ids)
|
||||
|
||||
expected = embeddings[dp_rank * 3 : dp_rank * 3 + local_tokens]
|
||||
assert torch.equal(output, expected)
|
||||
|
||||
|
||||
def _make_fp8_embedding_layer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
load_scale: bool = True,
|
||||
) -> embedding_module.VocabParallelEmbedding:
|
||||
) -> Qwen4ExpPLEDeviceEmbedding:
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
@@ -185,12 +254,13 @@ def _make_fp8_embedding_layer(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
method = Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
layer = Qwen4ExpPLEDeviceEmbedding(
|
||||
3,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=method,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=method,
|
||||
)
|
||||
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
|
||||
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
|
||||
@@ -228,7 +298,58 @@ def test_ple_fp8_embedding_rejects_missing_global_scale(monkeypatch) -> None:
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
|
||||
@pytest.mark.parametrize("requires_device_loading", [False, True])
|
||||
@pytest.mark.parametrize("load_scale", [False, True])
|
||||
def test_pinned_ple_post_load_validates_scale_without_staging(
|
||||
monkeypatch: pytest.MonkeyPatch, requires_device_loading: bool, load_scale: bool
|
||||
) -> None:
|
||||
"""Validate scales through the loader while honoring the staging policy."""
|
||||
from contextlib import nullcontext
|
||||
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader import utils as loader_utils
|
||||
|
||||
source = _make_fp8_embedding_layer(monkeypatch, load_scale=load_scale)
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.weight = source.weight
|
||||
embedding.weight_scale = source.weight_scale
|
||||
embedding.embedding_method = source.embedding_method
|
||||
embedding.quant_method = source.embedding_method
|
||||
embedding.tp_rank = source.tp_rank
|
||||
embedding.tp_size = source.tp_size
|
||||
assert QuantizeMethodBase.requires_device_loading
|
||||
assert not embedding.quant_method.requires_device_loading
|
||||
if requires_device_loading:
|
||||
embedding.quant_method.requires_device_loading = True
|
||||
model = nn.Module()
|
||||
model.embedding = embedding
|
||||
weight_ptr = embedding.weight.data_ptr()
|
||||
staged_modules = []
|
||||
|
||||
def track_staging(module, target_device):
|
||||
staged_modules.append(module)
|
||||
return nullcontext()
|
||||
|
||||
monkeypatch.setattr(loader_utils, "device_loading_context", track_staging)
|
||||
monkeypatch.setattr(loader_utils, "maybe_retie_word_embeddings", lambda *args: None)
|
||||
monkeypatch.setattr(
|
||||
loader_utils, "release_device_memory_under_pressure", lambda *args: None
|
||||
)
|
||||
config = SimpleNamespace(quantization="fp8")
|
||||
if load_scale:
|
||||
loader_utils.process_weights_after_loading(model, config, torch.device("cuda"))
|
||||
else:
|
||||
with pytest.raises(ValueError, match="missing its global scale"):
|
||||
loader_utils.process_weights_after_loading(
|
||||
model, config, torch.device("cuda")
|
||||
)
|
||||
assert embedding.weight.device.type == "cpu"
|
||||
assert embedding.weight.data_ptr() == weight_ptr
|
||||
assert staged_modules == ([embedding] if requires_device_loading else [])
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_uses_int8_for_parallel_reduce(monkeypatch) -> None:
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
@@ -251,17 +372,14 @@ def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
return tensor.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding_module,
|
||||
"tensor_model_parallel_all_reduce",
|
||||
all_reduce,
|
||||
)
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
_mock_etp_group(monkeypatch, world_size=2, all_reduce=all_reduce)
|
||||
layer = Qwen4ExpPLEDeviceEmbedding(
|
||||
4,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=Qwen4ExpPLEFp8EmbeddingMethod(),
|
||||
)
|
||||
layer.weight.data.copy_(
|
||||
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
|
||||
@@ -283,12 +401,200 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
|
||||
weight_block_size=[128, 128],
|
||||
)
|
||||
assert isinstance(
|
||||
_get_ple_embedding_quant_method(quant_config, prefix),
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
|
||||
quant_config.ignored_layers = [f"{prefix}.shard_0"]
|
||||
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(None, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
nvfp4_config = ModelOptNvFp4Config(exclude_modules=[])
|
||||
with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
|
||||
|
||||
dynamic_fp8_config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=False,
|
||||
ignored_layers=[],
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="serialized FP8"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(dynamic_fp8_config, prefix)
|
||||
|
||||
|
||||
def test_ple_embedding_respects_modelopt_exclusion() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
prefix,
|
||||
"float8_e4m3fn",
|
||||
),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
def test_pinned_embedding_forward_finalizes_prefetched_output(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = torch.empty(4, 2, 3, dtype=torch.float8_e4m3fn)
|
||||
embedding._output_dim = 6
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
|
||||
expected = torch.arange(12).reshape(2, 6).to(torch.float8_e4m3fn)
|
||||
|
||||
def finalize_prefetched(
|
||||
prefetch_output: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
assert prefetch_output is embedding._prefetch_buffer
|
||||
output.copy_(expected)
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding,
|
||||
"_finalize_prefetch",
|
||||
finalize_prefetched,
|
||||
)
|
||||
|
||||
output = embedding(hidden_states)
|
||||
|
||||
assert output.dtype == embedding._prefetch_buffer.dtype
|
||||
assert torch.equal(output, expected)
|
||||
|
||||
|
||||
def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None:
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding.tp_size = 2
|
||||
reduced_dtypes = []
|
||||
|
||||
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
return tensor.clone()
|
||||
|
||||
embedding.parallel_group = SimpleNamespace(all_reduce=all_reduce)
|
||||
embeddings = torch.arange(8).reshape(2, 4).to(torch.float8_e4m3fn)
|
||||
|
||||
output = embedding._reduce_etp_embeddings(embeddings)
|
||||
|
||||
assert reduced_dtypes == [torch.int8]
|
||||
assert output.dtype == embeddings.dtype
|
||||
torch.testing.assert_close(output.float(), embeddings.float())
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_ple_device_embedding_allocates_on_active_device(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
embedding_method = Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
|
||||
with torch.device("cuda:0"):
|
||||
embedding = Qwen4ExpPLEDeviceEmbedding(
|
||||
4,
|
||||
3,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=embedding_method,
|
||||
)
|
||||
|
||||
assert embedding.weight.device == torch.device("cuda:0")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("fp8_checkpoint", [False, True])
|
||||
def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_checkpoint: bool,
|
||||
) -> None:
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
|
||||
with torch.device("cuda:0"):
|
||||
embedding_method = (
|
||||
Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
if fp8_checkpoint
|
||||
else Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
)
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding(
|
||||
4,
|
||||
3,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=embedding_method,
|
||||
)
|
||||
|
||||
storage_dtype = torch.float8_e4m3fn if fp8_checkpoint else torch.bfloat16
|
||||
loaded_weight = (
|
||||
torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype)
|
||||
)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=0,
|
||||
tp_start=0,
|
||||
tp_end=4,
|
||||
)
|
||||
input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
output = embedding._lookup(input_ids)
|
||||
if fp8_checkpoint:
|
||||
embedding.weight_scale.data.fill_(0.25)
|
||||
weight_ptr = embedding.weight.data_ptr()
|
||||
embedding.quant_method.process_weights_after_loading(embedding)
|
||||
assert embedding.weight.data_ptr() == weight_ptr
|
||||
dequantized = embedding.dequantize(output, torch.bfloat16)
|
||||
|
||||
assert copied == 4
|
||||
assert embedding.weight.device.type == "cpu"
|
||||
assert embedding.weight.is_pinned()
|
||||
assert output.dtype == storage_dtype
|
||||
expected = loaded_weight[input_ids.cpu()].to(device="cuda:0")
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
expected_scale = 0.25 if fp8_checkpoint else 1.0
|
||||
torch.testing.assert_close(
|
||||
dequantized,
|
||||
expected.to(torch.bfloat16) * expected_scale,
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
@@ -310,15 +616,15 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
_get_ple_embedding_quant_method(quant_config, prefix),
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
assert (
|
||||
_get_ple_embedding_quant_method(
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
"model.language_model.layers.2.moe.gate_proj",
|
||||
)
|
||||
is None
|
||||
),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.config import (
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
DeviceConfig,
|
||||
EngramConfig,
|
||||
KernelConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
@@ -913,6 +914,159 @@ def test_all2all_backend_has_portable_default():
|
||||
assert ParallelConfig().all2all_backend == "allgather_reducescatter"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dp_size, across_dp, expected",
|
||||
[(1, False, 4), (1, True, 4), (2, False, 4), (2, True, 8)],
|
||||
)
|
||||
def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: int):
|
||||
parallel = ParallelConfig(
|
||||
tensor_parallel_size=4,
|
||||
data_parallel_size=dp_size,
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
config = EngramConfig(embedding_across_dp=across_dp)
|
||||
assert config.get_parallel_size(parallel) == expected
|
||||
|
||||
|
||||
def test_engram_rejects_elastic_cross_dp():
|
||||
parallel = ParallelConfig(
|
||||
tensor_parallel_size=4,
|
||||
data_parallel_size=2,
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
parallel.enable_elastic_ep = True
|
||||
with pytest.raises(ValueError, match="embedding_across_dp.*elastic EP"):
|
||||
EngramConfig(embedding_across_dp=True).verify_parallel_config(parallel)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", [None, "0", "1"])
|
||||
def test_engram_cpu_offload_environment_fallback(monkeypatch, legacy):
|
||||
"""Explicit settings must override the legacy environment fallback."""
|
||||
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
|
||||
if legacy is not None:
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
|
||||
assert EngramConfig().cpu_offload == (legacy == "1")
|
||||
assert EngramConfig(cpu_offload=False).cpu_offload is False
|
||||
assert EngramConfig(cpu_offload=True).cpu_offload is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"architecture, ple_layers, cuda, supported",
|
||||
[
|
||||
("Qwen4ExpForCausalLM", [1], True, True),
|
||||
("Qwen4ExpForConditionalGeneration", [1], True, True),
|
||||
("Qwen4ExpForCausalLM", [], True, False),
|
||||
("Qwen4ExpForCausalLM", None, True, False),
|
||||
("Qwen4ExpForCausalLM", [1], False, False),
|
||||
("LlamaForCausalLM", [1], True, False),
|
||||
("Qwen4ExpMTP", [], True, False),
|
||||
(None, None, True, False),
|
||||
],
|
||||
)
|
||||
def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, supported):
|
||||
"""A similarly named HF field must not enable unsupported implementations."""
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: cuda)
|
||||
model = (
|
||||
cast(
|
||||
ModelConfig,
|
||||
SimpleNamespace(
|
||||
architecture=architecture,
|
||||
hf_text_config=SimpleNamespace(ple_layer_ids=ple_layers),
|
||||
),
|
||||
)
|
||||
if architecture is not None
|
||||
else None
|
||||
)
|
||||
config = EngramConfig(cpu_offload=False, embedding_across_dp=False)
|
||||
if supported:
|
||||
config.verify_model_config(model)
|
||||
else:
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
config.verify_model_config(model)
|
||||
|
||||
|
||||
def test_engram_config_defaults_to_none(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
|
||||
config = VllmConfig()
|
||||
assert config.engram_config is None
|
||||
assert config.compute_hash()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", ["0", "1"])
|
||||
def test_engram_none_resolves_legacy_offload(monkeypatch, legacy):
|
||||
"""Legacy enablement materializes a config before model validation."""
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
config = cast(
|
||||
VllmConfig,
|
||||
SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
architecture="Qwen4ExpForCausalLM",
|
||||
hf_text_config=SimpleNamespace(ple_layer_ids=[1]),
|
||||
),
|
||||
speculative_config=None,
|
||||
engram_config=None,
|
||||
parallel_config=ParallelConfig(),
|
||||
),
|
||||
)
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
if legacy == "1":
|
||||
assert config.engram_config is not None
|
||||
assert config.engram_config.cpu_offload is True
|
||||
else:
|
||||
assert config.engram_config is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", ["0", "1"])
|
||||
def test_engram_explicit_config_requires_supported_model(monkeypatch, legacy):
|
||||
"""Explicit all-false settings still opt into model validation."""
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
VllmConfig(engram_config=EngramConfig(cpu_offload=False))
|
||||
|
||||
|
||||
def test_engram_legacy_offload_requires_supported_model(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
VllmConfig()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target_has_ple", [False, True])
|
||||
def test_engram_draft_config_validates_target(monkeypatch, target_has_ple):
|
||||
"""MTP may inherit cross-DP sharding without having its own PLE layers."""
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
target = SimpleNamespace(
|
||||
architecture="Qwen4ExpForCausalLM",
|
||||
hf_text_config=SimpleNamespace(ple_layer_ids=[1] if target_has_ple else []),
|
||||
)
|
||||
draft = SimpleNamespace(architecture="Qwen4ExpMTP")
|
||||
config = cast(
|
||||
VllmConfig,
|
||||
SimpleNamespace(
|
||||
model_config=draft,
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=draft, target_model_config=target
|
||||
),
|
||||
engram_config=EngramConfig(embedding_across_dp=True),
|
||||
parallel_config=ParallelConfig(),
|
||||
),
|
||||
)
|
||||
if target_has_ple:
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
else:
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
|
||||
|
||||
def test_engram_hash_tracks_storage_and_sharding():
|
||||
configs = [
|
||||
EngramConfig(cpu_offload=offload, embedding_across_dp=across_dp)
|
||||
for offload in (False, True)
|
||||
for across_dp in (False, True)
|
||||
]
|
||||
assert len({config.compute_hash() for config in configs}) == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("port", [1, 29550, 65535])
|
||||
def test_data_parallel_rpc_port_accepts_valid_ports(port: int):
|
||||
assert ParallelConfig(data_parallel_rpc_port=port).data_parallel_rpc_port == port
|
||||
|
||||
Reference in New Issue
Block a user