diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 7338eefbbb..eb96af2972 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -46,6 +46,39 @@ def test_optional_type(): assert optional_type_func("42") == 42 +@pytest.mark.parametrize( + "options", + [ + ["--engram-config", '{"cpu_offload": false, "embedding_across_dp": true}'], + [ + "--engram-config.cpu_offload", + "false", + "--engram-config.embedding_across_dp", + "true", + ], + ], +) +def test_engram_config_cli(options, monkeypatch): + """CLI settings take precedence over the legacy offload environment.""" + monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1") + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + args = EngineArgs.from_cli_args(parser.parse_args(options)) + assert args.engram_config is not None + assert args.engram_config.cpu_offload is False + assert args.engram_config.embedding_across_dp is True + + +@pytest.mark.parametrize( + "options, provided", + [([], False), (["--engram-config", "{}"], True)], +) +def test_engram_config_cli_optional(options, provided): + """An explicit empty config must remain distinct from an omitted config.""" + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + args = EngineArgs.from_cli_args(parser.parse_args(options)) + assert (args.engram_config is not None) == provided + + @pytest.mark.parametrize( ("type_hint", "type", "expected"), [ diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index 9d188af81f..020e1cc8fc 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -13,27 +13,50 @@ from torch.nn import functional as F import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module import vllm.model_executor.parameter as parameter_module +import vllm.models.qwen4_exp.nvidia.ngram_embedding as ngram_embedding_module from vllm.model_executor.layers.quantization.fp8 import Fp8Config from vllm.model_executor.layers.quantization.modelopt import ( ModelOptMixedPrecisionConfig, + ModelOptNvFp4Config, ) from vllm.models.qwen4_exp.common.ple import ( PLEShardOverlap, compute_ple_shard_overlap, copy_ple_embedding_shard_, ) -from vllm.models.qwen4_exp.nvidia.ple_layer import ( +from vllm.models.qwen4_exp.nvidia.ngram_embedding import ( Qwen4ExpNGramEmbedding, + Qwen4ExpPLEDeviceEmbedding, + Qwen4ExpPLEEmbeddingMethod, Qwen4ExpPLEFp8EmbeddingMethod, - Qwen4ExpPLELayer, - _get_ple_embedding_quant_method, + Qwen4ExpPLEPinnedHostEmbedding, + Qwen4ExpPLEUnquantizedEmbeddingMethod, ) +from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer from vllm.v1.attention.backends.short_conv_attn import ( PleShortConvAttentionMetadata, ) from vllm.v1.attention.backends.utils import NULL_BLOCK_ID +def _mock_etp_group( + monkeypatch: pytest.MonkeyPatch, + world_size: int = 1, + all_reduce=lambda tensor: tensor, +) -> None: + group = SimpleNamespace( + rank_in_group=0, + world_size=world_size, + all_reduce=all_reduce, + ) + monkeypatch.setattr(ngram_embedding_module, "get_etp_group", lambda: group) + monkeypatch.setattr( + ngram_embedding_module, + "get_tp_group", + lambda: SimpleNamespace(world_size=world_size), + ) + + def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding: module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding) nn.Module.__init__(module) @@ -171,11 +194,57 @@ def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None: ) +@pytest.mark.parametrize(("dp_rank", "local_tokens"), [(0, 2), (1, 3)]) +def test_etp_lookup_gathers_and_returns_dp_local_rows( + monkeypatch: pytest.MonkeyPatch, + dp_rank: int, + local_tokens: int, +) -> None: + embedding = Qwen4ExpPLEDeviceEmbedding.__new__(Qwen4ExpPLEDeviceEmbedding) + nn.Module.__init__(embedding) + embedding.tp_size = 2 + embedding.etp_data_parallel_size = 2 + embedding.data_parallel_rank = dp_rank + gathered_ids = torch.tensor([[10], [11], [0], [20], [21], [22]]) + group = SimpleNamespace( + rank_in_group=dp_rank, + all_gather=lambda input_ids, dim: gathered_ids, + all_reduce=lambda embeddings: embeddings, + ) + embedding.parallel_group = group + forward_context = SimpleNamespace( + dp_metadata=SimpleNamespace( + num_tokens_across_dp_cpu=torch.tensor([2, 3]), + ) + ) + monkeypatch.setattr(ngram_embedding_module, "get_dp_group", lambda: group) + monkeypatch.setattr( + ngram_embedding_module, "get_forward_context", lambda: forward_context + ) + local_ids = gathered_ids[dp_rank * 3 : dp_rank * 3 + local_tokens] + embeddings = torch.arange(12).reshape(6, 2) + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + assert torch.equal(input_ids, gathered_ids) + return embeddings + + monkeypatch.setattr( + ngram_embedding_module.PLEVocabParallelEmbedding, + "forward", + forward, + ) + output = embedding(local_ids) + + expected = embeddings[dp_rank * 3 : dp_rank * 3 + local_tokens] + assert torch.equal(output, expected) + + def _make_fp8_embedding_layer( monkeypatch: pytest.MonkeyPatch, *, load_scale: bool = True, -) -> embedding_module.VocabParallelEmbedding: +) -> Qwen4ExpPLEDeviceEmbedding: + _mock_etp_group(monkeypatch) monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0) monkeypatch.setattr( embedding_module, "get_tensor_model_parallel_world_size", lambda: 1 @@ -185,12 +254,13 @@ def _make_fp8_embedding_layer( parameter_module, "get_tensor_model_parallel_world_size", lambda: 1 ) method = Qwen4ExpPLEFp8EmbeddingMethod() - layer = embedding_module.VocabParallelEmbedding( + layer = Qwen4ExpPLEDeviceEmbedding( 3, 2, params_dtype=torch.bfloat16, padding_size=1, - quant_method=method, + prefix="test.ple_embedding", + embedding_method=method, ) weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]]) layer.weight.data.copy_(weight.to(torch.float8_e4m3fn)) @@ -228,7 +298,58 @@ def test_ple_fp8_embedding_rejects_missing_global_scale(monkeypatch) -> None: layer.quant_method.process_weights_after_loading(layer) -def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None: +@pytest.mark.parametrize("requires_device_loading", [False, True]) +@pytest.mark.parametrize("load_scale", [False, True]) +def test_pinned_ple_post_load_validates_scale_without_staging( + monkeypatch: pytest.MonkeyPatch, requires_device_loading: bool, load_scale: bool +) -> None: + """Validate scales through the loader while honoring the staging policy.""" + from contextlib import nullcontext + + from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase + from vllm.model_executor.model_loader import utils as loader_utils + + source = _make_fp8_embedding_layer(monkeypatch, load_scale=load_scale) + embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding) + nn.Module.__init__(embedding) + embedding.weight = source.weight + embedding.weight_scale = source.weight_scale + embedding.embedding_method = source.embedding_method + embedding.quant_method = source.embedding_method + embedding.tp_rank = source.tp_rank + embedding.tp_size = source.tp_size + assert QuantizeMethodBase.requires_device_loading + assert not embedding.quant_method.requires_device_loading + if requires_device_loading: + embedding.quant_method.requires_device_loading = True + model = nn.Module() + model.embedding = embedding + weight_ptr = embedding.weight.data_ptr() + staged_modules = [] + + def track_staging(module, target_device): + staged_modules.append(module) + return nullcontext() + + monkeypatch.setattr(loader_utils, "device_loading_context", track_staging) + monkeypatch.setattr(loader_utils, "maybe_retie_word_embeddings", lambda *args: None) + monkeypatch.setattr( + loader_utils, "release_device_memory_under_pressure", lambda *args: None + ) + config = SimpleNamespace(quantization="fp8") + if load_scale: + loader_utils.process_weights_after_loading(model, config, torch.device("cuda")) + else: + with pytest.raises(ValueError, match="missing its global scale"): + loader_utils.process_weights_after_loading( + model, config, torch.device("cuda") + ) + assert embedding.weight.device.type == "cpu" + assert embedding.weight.data_ptr() == weight_ptr + assert staged_modules == ([embedding] if requires_device_loading else []) + + +def test_ple_fp8_embedding_uses_int8_for_parallel_reduce(monkeypatch) -> None: monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0) monkeypatch.setattr( embedding_module, "get_tensor_model_parallel_world_size", lambda: 2 @@ -251,17 +372,14 @@ def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None: reduced_dtypes.append(tensor.dtype) return tensor.clone() - monkeypatch.setattr( - embedding_module, - "tensor_model_parallel_all_reduce", - all_reduce, - ) - layer = embedding_module.VocabParallelEmbedding( + _mock_etp_group(monkeypatch, world_size=2, all_reduce=all_reduce) + layer = Qwen4ExpPLEDeviceEmbedding( 4, 2, params_dtype=torch.bfloat16, padding_size=1, - quant_method=Qwen4ExpPLEFp8EmbeddingMethod(), + prefix="test.ple_embedding", + embedding_method=Qwen4ExpPLEFp8EmbeddingMethod(), ) layer.weight.data.copy_( torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn) @@ -283,12 +401,200 @@ def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None: weight_block_size=[128, 128], ) assert isinstance( - _get_ple_embedding_quant_method(quant_config, prefix), + Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix), Qwen4ExpPLEFp8EmbeddingMethod, ) quant_config.ignored_layers = [f"{prefix}.shard_0"] - assert _get_ple_embedding_quant_method(quant_config, prefix) is None + assert isinstance( + Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix), + Qwen4ExpPLEUnquantizedEmbeddingMethod, + ) + assert isinstance( + Qwen4ExpPLEEmbeddingMethod.from_quant_config(None, prefix), + Qwen4ExpPLEUnquantizedEmbeddingMethod, + ) + + +def test_ple_embedding_rejects_unsupported_quantization_configs() -> None: + prefix = "model.layers.1.ple.ple_embedding.ngram_embedding" + nvfp4_config = ModelOptNvFp4Config(exclude_modules=[]) + with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"): + Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix) + + dynamic_fp8_config = Fp8Config( + is_checkpoint_fp8_serialized=False, + ignored_layers=[], + ) + with pytest.raises(NotImplementedError, match="serialized FP8"): + Qwen4ExpPLEEmbeddingMethod.from_quant_config(dynamic_fp8_config, prefix) + + +def test_ple_embedding_respects_modelopt_exclusion() -> None: + prefix = "model.layers.1.ple.ple_embedding.ngram_embedding" + quant_config = ModelOptNvFp4Config(exclude_modules=[prefix]) + + assert isinstance( + Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix), + Qwen4ExpPLEUnquantizedEmbeddingMethod, + ) + + +def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None: + prefix = "model.layers.1.ple.ple_embedding.ngram_embedding" + quant_config = ModelOptNvFp4Config(exclude_modules=[prefix]) + + assert isinstance( + Qwen4ExpPLEEmbeddingMethod.from_quant_config( + quant_config, + prefix, + "float8_e4m3fn", + ), + Qwen4ExpPLEFp8EmbeddingMethod, + ) + + +def test_pinned_embedding_forward_finalizes_prefetched_output( + monkeypatch: pytest.MonkeyPatch, +) -> None: + embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding) + nn.Module.__init__(embedding) + embedding._prefetch_buffer = torch.empty(4, 2, 3, dtype=torch.float8_e4m3fn) + embedding._output_dim = 6 + hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16) + expected = torch.arange(12).reshape(2, 6).to(torch.float8_e4m3fn) + + def finalize_prefetched( + prefetch_output: torch.Tensor, + output: torch.Tensor, + ) -> None: + assert prefetch_output is embedding._prefetch_buffer + output.copy_(expected) + + monkeypatch.setattr( + embedding, + "_finalize_prefetch", + finalize_prefetched, + ) + + output = embedding(hidden_states) + + assert output.dtype == embedding._prefetch_buffer.dtype + assert torch.equal(output, expected) + + +def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None: + embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding) + nn.Module.__init__(embedding) + embedding.tp_size = 2 + reduced_dtypes = [] + + def all_reduce(tensor: torch.Tensor) -> torch.Tensor: + reduced_dtypes.append(tensor.dtype) + return tensor.clone() + + embedding.parallel_group = SimpleNamespace(all_reduce=all_reduce) + embeddings = torch.arange(8).reshape(2, 4).to(torch.float8_e4m3fn) + + output = embedding._reduce_etp_embeddings(embeddings) + + assert reduced_dtypes == [torch.int8] + assert output.dtype == embeddings.dtype + torch.testing.assert_close(output.float(), embeddings.float()) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_ple_device_embedding_allocates_on_active_device( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _mock_etp_group(monkeypatch) + monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + embedding_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + parameter_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + embedding_method = Qwen4ExpPLEUnquantizedEmbeddingMethod() + + with torch.device("cuda:0"): + embedding = Qwen4ExpPLEDeviceEmbedding( + 4, + 3, + params_dtype=torch.bfloat16, + padding_size=1, + prefix="test.ple_embedding", + embedding_method=embedding_method, + ) + + assert embedding.weight.device == torch.device("cuda:0") + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("fp8_checkpoint", [False, True]) +def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva( + monkeypatch: pytest.MonkeyPatch, + fp8_checkpoint: bool, +) -> None: + _mock_etp_group(monkeypatch) + monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + embedding_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + parameter_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + + with torch.device("cuda:0"): + embedding_method = ( + Qwen4ExpPLEFp8EmbeddingMethod() + if fp8_checkpoint + else Qwen4ExpPLEUnquantizedEmbeddingMethod() + ) + embedding = Qwen4ExpPLEPinnedHostEmbedding( + 4, + 3, + params_dtype=torch.bfloat16, + padding_size=1, + prefix="test.ple_embedding", + embedding_method=embedding_method, + ) + + storage_dtype = torch.float8_e4m3fn if fp8_checkpoint else torch.bfloat16 + loaded_weight = ( + torch.arange(12, dtype=torch.float32).reshape(4, 3).to(storage_dtype) + ) + copied = copy_ple_embedding_shard_( + embedding.weight, + loaded_weight, + checkpoint_start=0, + tp_start=0, + tp_end=4, + ) + input_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0") + output = embedding._lookup(input_ids) + if fp8_checkpoint: + embedding.weight_scale.data.fill_(0.25) + weight_ptr = embedding.weight.data_ptr() + embedding.quant_method.process_weights_after_loading(embedding) + assert embedding.weight.data_ptr() == weight_ptr + dequantized = embedding.dequantize(output, torch.bfloat16) + + assert copied == 4 + assert embedding.weight.device.type == "cpu" + assert embedding.weight.is_pinned() + assert output.dtype == storage_dtype + expected = loaded_weight[input_ids.cpu()].to(device="cuda:0") + torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0) + expected_scale = 0.25 if fp8_checkpoint else 1.0 + torch.testing.assert_close( + dequantized, + expected.to(torch.bfloat16) * expected_scale, + rtol=0, + atol=0, + ) def test_ple_fp8_embedding_supports_mixed_precision_config() -> None: @@ -310,15 +616,15 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None: ) assert isinstance( - _get_ple_embedding_quant_method(quant_config, prefix), + Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix), Qwen4ExpPLEFp8EmbeddingMethod, ) - assert ( - _get_ple_embedding_quant_method( + assert isinstance( + Qwen4ExpPLEEmbeddingMethod.from_quant_config( quant_config, "model.language_model.layers.2.moe.gate_proj", - ) - is None + ), + Qwen4ExpPLEUnquantizedEmbeddingMethod, ) diff --git a/tests/test_config.py b/tests/test_config.py index 7df0489c67..eae797adf5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -22,6 +22,7 @@ from vllm.config import ( CacheConfig, CompilationConfig, DeviceConfig, + EngramConfig, KernelConfig, KVTransferConfig, ModelConfig, @@ -913,6 +914,159 @@ def test_all2all_backend_has_portable_default(): assert ParallelConfig().all2all_backend == "allgather_reducescatter" +@pytest.mark.parametrize( + "dp_size, across_dp, expected", + [(1, False, 4), (1, True, 4), (2, False, 4), (2, True, 8)], +) +def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: int): + parallel = ParallelConfig( + tensor_parallel_size=4, + data_parallel_size=dp_size, + distributed_executor_backend="mp", + ) + config = EngramConfig(embedding_across_dp=across_dp) + assert config.get_parallel_size(parallel) == expected + + +def test_engram_rejects_elastic_cross_dp(): + parallel = ParallelConfig( + tensor_parallel_size=4, + data_parallel_size=2, + distributed_executor_backend="mp", + ) + parallel.enable_elastic_ep = True + with pytest.raises(ValueError, match="embedding_across_dp.*elastic EP"): + EngramConfig(embedding_across_dp=True).verify_parallel_config(parallel) + + +@pytest.mark.parametrize("legacy", [None, "0", "1"]) +def test_engram_cpu_offload_environment_fallback(monkeypatch, legacy): + """Explicit settings must override the legacy environment fallback.""" + monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False) + if legacy is not None: + monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy) + assert EngramConfig().cpu_offload == (legacy == "1") + assert EngramConfig(cpu_offload=False).cpu_offload is False + assert EngramConfig(cpu_offload=True).cpu_offload is True + + +@pytest.mark.parametrize( + "architecture, ple_layers, cuda, supported", + [ + ("Qwen4ExpForCausalLM", [1], True, True), + ("Qwen4ExpForConditionalGeneration", [1], True, True), + ("Qwen4ExpForCausalLM", [], True, False), + ("Qwen4ExpForCausalLM", None, True, False), + ("Qwen4ExpForCausalLM", [1], False, False), + ("LlamaForCausalLM", [1], True, False), + ("Qwen4ExpMTP", [], True, False), + (None, None, True, False), + ], +) +def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, supported): + """A similarly named HF field must not enable unsupported implementations.""" + monkeypatch.setattr(current_platform, "is_cuda", lambda: cuda) + model = ( + cast( + ModelConfig, + SimpleNamespace( + architecture=architecture, + hf_text_config=SimpleNamespace(ple_layer_ids=ple_layers), + ), + ) + if architecture is not None + else None + ) + config = EngramConfig(cpu_offload=False, embedding_across_dp=False) + if supported: + config.verify_model_config(model) + else: + with pytest.raises(ValueError, match="requires a model with supported Engram"): + config.verify_model_config(model) + + +def test_engram_config_defaults_to_none(monkeypatch): + monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False) + config = VllmConfig() + assert config.engram_config is None + assert config.compute_hash() + + +@pytest.mark.parametrize("legacy", ["0", "1"]) +def test_engram_none_resolves_legacy_offload(monkeypatch, legacy): + """Legacy enablement materializes a config before model validation.""" + monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy) + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + config = cast( + VllmConfig, + SimpleNamespace( + model_config=SimpleNamespace( + architecture="Qwen4ExpForCausalLM", + hf_text_config=SimpleNamespace(ple_layer_ids=[1]), + ), + speculative_config=None, + engram_config=None, + parallel_config=ParallelConfig(), + ), + ) + VllmConfig._resolve_and_verify_engram_config(config) + if legacy == "1": + assert config.engram_config is not None + assert config.engram_config.cpu_offload is True + else: + assert config.engram_config is None + + +@pytest.mark.parametrize("legacy", ["0", "1"]) +def test_engram_explicit_config_requires_supported_model(monkeypatch, legacy): + """Explicit all-false settings still opt into model validation.""" + monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy) + with pytest.raises(ValueError, match="requires a model with supported Engram"): + VllmConfig(engram_config=EngramConfig(cpu_offload=False)) + + +def test_engram_legacy_offload_requires_supported_model(monkeypatch): + monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1") + with pytest.raises(ValueError, match="requires a model with supported Engram"): + VllmConfig() + + +@pytest.mark.parametrize("target_has_ple", [False, True]) +def test_engram_draft_config_validates_target(monkeypatch, target_has_ple): + """MTP may inherit cross-DP sharding without having its own PLE layers.""" + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + target = SimpleNamespace( + architecture="Qwen4ExpForCausalLM", + hf_text_config=SimpleNamespace(ple_layer_ids=[1] if target_has_ple else []), + ) + draft = SimpleNamespace(architecture="Qwen4ExpMTP") + config = cast( + VllmConfig, + SimpleNamespace( + model_config=draft, + speculative_config=SimpleNamespace( + draft_model_config=draft, target_model_config=target + ), + engram_config=EngramConfig(embedding_across_dp=True), + parallel_config=ParallelConfig(), + ), + ) + if target_has_ple: + VllmConfig._resolve_and_verify_engram_config(config) + else: + with pytest.raises(ValueError, match="requires a model with supported Engram"): + VllmConfig._resolve_and_verify_engram_config(config) + + +def test_engram_hash_tracks_storage_and_sharding(): + configs = [ + EngramConfig(cpu_offload=offload, embedding_across_dp=across_dp) + for offload in (False, True) + for across_dp in (False, True) + ] + assert len({config.compute_hash() for config in configs}) == 4 + + @pytest.mark.parametrize("port", [1, 29550, 65535]) def test_data_parallel_rpc_port_accepts_valid_ports(port: int): assert ParallelConfig(data_parallel_rpc_port=port).data_parallel_rpc_port == port diff --git a/vllm/config/__init__.py b/vllm/config/__init__.py index 24a8b5a317..8036d29093 100644 --- a/vllm/config/__init__.py +++ b/vllm/config/__init__.py @@ -13,6 +13,7 @@ from vllm.config.device import DeviceConfig from vllm.config.diffusion import DiffusionConfig from vllm.config.ec_manager_config import EncoderCacheManagerConfig from vllm.config.ec_transfer import ECTransferConfig +from vllm.config.engram import EngramConfig from vllm.config.fault_tolerance import FaultToleranceConfig from vllm.config.kernel import KernelConfig from vllm.config.kv_events import KVEventsConfig @@ -81,6 +82,8 @@ __all__ = [ "EncoderCacheManagerConfig", # From vllm.config.ec_transfer "ECTransferConfig", + # From vllm.config.engram + "EngramConfig", # From vllm.config.kernel "KernelConfig", # From vllm.config.kv_events diff --git a/vllm/config/engram.py b/vllm/config/engram.py new file mode 100644 index 0000000000..b46311315f --- /dev/null +++ b/vllm/config/engram.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import TYPE_CHECKING + +from pydantic import Field + +import vllm.envs as envs +from vllm.config.utils import config, get_hash_factors, hash_factors +from vllm.logger import init_logger + +if TYPE_CHECKING: + from vllm.config.model import ModelConfig + from vllm.config.parallel import ParallelConfig + +logger = init_logger(__name__) + + +def _default_cpu_offload() -> bool: + """Honor the legacy environment variable only when the field is omitted.""" + if envs.is_set("VLLM_PLE_CPU_OFFLOAD"): + logger.warning_once( + "VLLM_PLE_CPU_OFFLOAD is a legacy setting and may be removed in a " + "future release. Use --engram-config.cpu_offload instead." + ) + return envs.VLLM_PLE_CPU_OFFLOAD + + +@config +class EngramConfig: + """Configuration for Engram embedding storage and sharding.""" + + cpu_offload: bool = Field(default_factory=_default_cpu_offload) + """Store embedding weights in pinned CPU memory for UVA lookup. + Defaults to False, or VLLM_PLE_CPU_OFFLOAD when set for compatibility. + An explicit value takes precedence over the legacy environment variable.""" + + embedding_across_dp: bool = False + """Shard embeddings across TP and all DP ranks when enabled. + Otherwise, each DP rank has a separate TP-sharded embedding replica.""" + + def verify_model_config(self, model_config: "ModelConfig | None") -> None: + """Reject Engram configuration for models without supported embeddings.""" + from vllm.platforms import current_platform + + supported_architectures = { + "Qwen4ExpForCausalLM", + "Qwen4ExpForConditionalGeneration", + } + if ( + model_config is None + or model_config.architecture not in supported_architectures + or not current_platform.is_cuda() + or not getattr(model_config.hf_text_config, "ple_layer_ids", None) + ): + raise ValueError( + "EngramConfig requires a model with supported Engram " + "embeddings. Currently only the CUDA Qwen4Exp implementation " + "with non-empty ple_layer_ids is supported." + ) + + def verify_parallel_config(self, parallel_config: "ParallelConfig") -> None: + """Reject unsupported embedding parallel topologies.""" + if ( + self.embedding_across_dp + and parallel_config.data_parallel_size > 1 + and parallel_config.enable_elastic_ep + ): + raise ValueError( + "Engram embedding_across_dp is not supported with elastic EP yet." + ) + + def get_parallel_size(self, parallel_config: "ParallelConfig") -> int: + """Derive the embedding group size from the parallel configuration.""" + size = parallel_config.tensor_parallel_size + if self.embedding_across_dp and parallel_config.data_parallel_size > 1: + size *= parallel_config.data_parallel_size + return size + + def compute_hash(self) -> str: + """Hash settings that affect embedding execution and graph structure.""" + return hash_factors(get_hash_factors(self, set())) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 2af8400c26..0564cd420e 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -34,6 +34,7 @@ from .device import DeviceConfig from .diffusion import DiffusionConfig from .ec_manager_config import EncoderCacheManagerConfig from .ec_transfer import ECTransferConfig +from .engram import EngramConfig from .kernel import KernelConfig from .kv_events import KVEventsConfig from .kv_transfer import KVTransferConfig @@ -370,6 +371,8 @@ class VllmConfig: """Model weight offloading configuration.""" attention_config: AttentionConfig = Field(default_factory=AttentionConfig) """Attention configuration.""" + engram_config: EngramConfig | None = None + """Optional Engram configuration, only valid for supported PLE models.""" mamba_config: MambaConfig = Field(default_factory=MambaConfig) """Mamba configuration.""" kernel_config: KernelConfig = Field(default_factory=KernelConfig) @@ -506,6 +509,11 @@ class VllmConfig: vllm_factors.append(self.attention_config.compute_hash()) else: vllm_factors.append("None") + vllm_factors.append( + self.engram_config.compute_hash() + if self.engram_config is not None + else "None" + ) if self.lora_config: vllm_factors.append(self.lora_config.compute_hash()) else: @@ -1083,6 +1091,25 @@ class VllmConfig: if not self.use_v2_model_runner: raise ValueError("trace replay requires Model Runner V2") + def _resolve_and_verify_engram_config(self) -> None: + """Resolve legacy offload settings and validate model and parallel configs.""" + if self.engram_config is None: + if not envs.VLLM_PLE_CPU_OFFLOAD: + return + self.engram_config = EngramConfig() + model_config = self.model_config + speculative_config = self.speculative_config + # Draft configs inherit the target's communication groups and settings. + # Qwen4Exp MTP itself disables PLE, so validate its target instead. + if ( + speculative_config is not None + and model_config is speculative_config.draft_model_config + ): + model_config = speculative_config.target_model_config + self.engram_config.verify_model_config(model_config) + self.engram_config.verify_parallel_config(self.parallel_config) + logger.info_once("Resolved Engram configuration: %s", str(self.engram_config)) + def __post_init__(self): """Verify configs are valid & consistent with each other.""" @@ -1095,6 +1122,7 @@ class VllmConfig: logger.info_once("Performance mode set to '%s'.", self.performance_mode) self.try_verify_and_update_config() + self._resolve_and_verify_engram_config() # Models may have supplied their own DCP defaults above; anything still # unset falls back to the stock ones. diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index 25b500b8fa..faa9fd0987 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -47,7 +47,8 @@ class CudaCommunicator(DeviceCommunicatorBase): global_world_size, use_all2all=use_all2all, ) - if "tp" not in unique_name: + # Match the group name exactly so ETP does not enable TP-only backends. + if unique_name.split(":")[0] != "tp": # custom allreduce or torch symm mem can be used only by tp use_custom_allreduce = False use_torch_symm_mem = False diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index 9a0f79fcc2..92deff1183 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -1551,6 +1551,15 @@ def get_tp_group() -> GroupCoordinator: return _TP +_ETP: GroupCoordinator | None = None + + +def get_etp_group() -> GroupCoordinator: + """Return the Engram Tensor Parallel (ETP) group that shards one embedding table.""" + assert _ETP is not None, "Engram tensor-parallel group is not initialized" + return _ETP + + _DCP: GroupCoordinator | None = None @@ -2008,6 +2017,28 @@ def initialize_model_parallel( group_name="tp", ) + global _ETP + assert _ETP is None, "Engram tensor-parallel group is already initialized" + engram_tensor_parallel_size = ( + config.engram_config.get_parallel_size(parallel_config) + if config.engram_config is not None + else tensor_model_parallel_size + ) + if engram_tensor_parallel_size == tensor_model_parallel_size: + _ETP = _TP + else: + group_ranks = ( + all_ranks.permute(0, 2, 3, 1, 4) + .reshape(-1, engram_tensor_parallel_size) + .unbind(0) + ) + _ETP = init_model_parallel_group( + [ranks.tolist() for ranks in group_ranks], + get_world_group().local_rank, + backend, + group_name="etp", + ) + # Build the DCP model-parallel groups. global _DCP assert _DCP is None, "decode context model parallel group is already initialized" @@ -2142,13 +2173,14 @@ def initialize_model_parallel( logger.info_once( "rank %s in world size %s is assigned as " "DP rank %s, PP rank %s, PCP rank %s, " - "TP rank %s, EP rank %s, EPLB rank %s", + "TP rank %s, ETP rank %s, EP rank %s, EPLB rank %s", rank, world_size, _DP.rank_in_group, _PP.rank_in_group, _PCP.rank_in_group, _TP.rank_in_group, + _ETP.rank_in_group, _EP.rank_in_group if _EP is not None else "N/A", _EPLB.rank_in_group if _EPLB is not None else "N/A", ) @@ -2243,7 +2275,11 @@ def get_node_count() -> int: def destroy_model_parallel(): """Set the groups to none and destroy them.""" - global _TP + global _TP, _ETP + + if _ETP and _ETP is not _TP: + _ETP.destroy() + _ETP = None if _TP: _TP.destroy() diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index e7b3c3f48c..acdbf63b7f 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -42,6 +42,7 @@ from vllm.config import ( DiffusionConfig, ECTransferConfig, EncoderCacheManagerConfig, + EngramConfig, EPLBConfig, FaultToleranceConfig, KernelConfig, @@ -688,6 +689,7 @@ class EngineArgs: pooler_config: PoolerConfig | None = ModelConfig.pooler_config compilation_config: CompilationConfig = get_field(VllmConfig, "compilation_config") attention_config: AttentionConfig = get_field(VllmConfig, "attention_config") + engram_config: EngramConfig | None = VllmConfig.engram_config mamba_config: MambaConfig = get_field(VllmConfig, "mamba_config") kernel_config: KernelConfig = get_field(VllmConfig, "kernel_config") enable_flashinfer_autotune: bool = get_field( @@ -788,6 +790,8 @@ class EngineArgs: self.compilation_config = CompilationConfig(**self.compilation_config) if isinstance(self.attention_config, dict): self.attention_config = AttentionConfig(**self.attention_config) + if isinstance(self.engram_config, dict): + self.engram_config = EngramConfig(**self.engram_config) if isinstance(self.mamba_config, dict): self.mamba_config = MambaConfig(**self.mamba_config) if isinstance(self.kernel_config, dict): @@ -1701,6 +1705,7 @@ class EngineArgs: vllm_group.add_argument( "--attention-config", "-ac", **vllm_kwargs["attention_config"] ) + vllm_group.add_argument("--engram-config", **vllm_kwargs["engram_config"]) vllm_group.add_argument("--reasoning-config", **vllm_kwargs["reasoning_config"]) vllm_group.add_argument("--kernel-config", **vllm_kwargs["kernel_config"]) vllm_group.add_argument( @@ -2599,6 +2604,7 @@ class EngineArgs: load_config=load_config, offload_config=offload_config, attention_config=attention_config, + engram_config=self.engram_config, mamba_config=mamba_config, kernel_config=kernel_config, lora_config=lora_config, diff --git a/vllm/envs.py b/vllm/envs.py index ae506acee2..d429686d9c 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -155,6 +155,7 @@ if TYPE_CHECKING: VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT: bool = False VLLM_ENABLE_V1_MULTIPROCESSING: bool = True VLLM_LOG_BATCHSIZE_INTERVAL: float = -1 + VLLM_PLE_CPU_OFFLOAD: bool = False VLLM_DISABLE_COMPILE_CACHE: bool = False VLLM_REPLICATE_EMBED: bool = False VLLM_USE_LAYERNAME: bool = True @@ -2077,6 +2078,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_LOG_MODEL_INSPECTION": lambda: bool( int(os.getenv("VLLM_LOG_MODEL_INSPECTION", "0")) ), + # Keep Qwen4Exp PLE embedding tables in pinned CPU memory and gather their + # rows through UVA on a dedicated CUDA stream. + # Legacy fallback for EngramConfig.cpu_offload, which takes precedence. + # This environment variable may be removed in a future release. + "VLLM_PLE_CPU_OFFLOAD": lambda: bool(int(os.getenv("VLLM_PLE_CPU_OFFLOAD", "0"))), # Debug logging for --enable-mfu-metrics "VLLM_DEBUG_MFU_METRICS": lambda: bool( int(os.getenv("VLLM_DEBUG_MFU_METRICS", "0")) diff --git a/vllm/model_executor/layers/quantization/base_config.py b/vllm/model_executor/layers/quantization/base_config.py index 5f2c7eebd2..d1f8df2df7 100644 --- a/vllm/model_executor/layers/quantization/base_config.py +++ b/vllm/model_executor/layers/quantization/base_config.py @@ -23,6 +23,9 @@ else: class QuantizeMethodBase(ABC): """Base class for different quantized methods.""" + requires_device_loading: bool = True + """Whether post-load processing requires parameters on the target device.""" + uses_meta_device: bool = False """ Whether this method creates weights on meta device for online quantization. diff --git a/vllm/model_executor/layers/vocab_parallel_embedding.py b/vllm/model_executor/layers/vocab_parallel_embedding.py index 66cbc65051..456a22a7a2 100644 --- a/vllm/model_executor/layers/vocab_parallel_embedding.py +++ b/vllm/model_executor/layers/vocab_parallel_embedding.py @@ -11,6 +11,7 @@ from torch.nn.parameter import Parameter import vllm.envs as envs from vllm import _custom_ops as ops from vllm.distributed import ( + GroupCoordinator, divide, get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -240,6 +241,7 @@ class VocabParallelEmbedding(PluggableLayer): prefix: full name of the layer in the state dict disable_tp: If true, tensor parallelism will be disabled for this layer. quant_method: Preselected quantization method for model-specific layers. + parallel_group: Process group used to shard and reduce the embedding. """ # noqa: E501 # --8<-- [end:vocab_parallel_embedding] @@ -256,13 +258,18 @@ class VocabParallelEmbedding(PluggableLayer): *, disable_tp: bool = False, quant_method: QuantizeMethodBase | None = None, + parallel_group: GroupCoordinator | None = None, ): super().__init__() # Keep the input dimensions. self.disable_tp = disable_tp + self.parallel_group = parallel_group if disable_tp: tp_rank, self.tp_size = 0, 1 + elif parallel_group is not None: + tp_rank = parallel_group.rank_in_group + self.tp_size = parallel_group.world_size else: tp_rank = get_tensor_model_parallel_rank() self.tp_size = get_tensor_model_parallel_world_size() @@ -535,10 +542,16 @@ class VocabParallelEmbedding(PluggableLayer): # Each vocab token has one owner, so FP8 bytes can use int8 SUM. comm_output = output_parallel.view(torch.int8) comm_output.masked_fill_(input_mask.unsqueeze(-1), 0) - output = tensor_model_parallel_all_reduce(comm_output) + output = ( + self.parallel_group.all_reduce(comm_output) + if self.parallel_group is not None + else tensor_model_parallel_all_reduce(comm_output) + ) return output.view(output_parallel.dtype) output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0) # Reduce across all the model parallel GPUs. + if self.parallel_group is not None: + return self.parallel_group.all_reduce(output_parallel) return tensor_model_parallel_all_reduce(output_parallel) def extra_repr(self) -> str: diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index 03039cf019..e12910865e 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -4,7 +4,7 @@ import inspect import warnings -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from typing import Any import torch @@ -120,11 +120,18 @@ def process_weights_after_loading( "does not support pre-processed weights" ) # When quant methods need to process weights after loading - # (for repacking, quantizing, etc), they expect parameters + # (for repacking, quantizing, etc), they typically expect parameters # to be on the global target device. This scope is for the # case where cpu offloading is used, where we will move the # parameters onto device for processing and back off after. - with device_loading_context(module, target_device): + # Methods that can process weights in place (e.g. PLE scale + # validation) set requires_device_loading=False to skip this move. + loading_context = ( + device_loading_context(module, target_device) + if quant_method.requires_device_loading + else nullcontext() + ) + with loading_context: quant_method.process_weights_after_loading(module) # process_weights_after_loading may swap in freshly-created # Parameters (e.g. FP8 requantization), which are stamped with the diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index da74356a62..28c3ea69d1 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -461,6 +461,27 @@ class Qwen4ExpModel(nn.Module): def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) + @staticmethod + def _start_layer_ple_prefetch( + layer: nn.Module, + hidden_states: torch.Tensor, + input_ids: torch.Tensor | None, + query_start_loc: torch.Tensor | None, + ngram_context: torch.Tensor | None, + ) -> None: + """Start a layer's PLE prefetch when the required inputs exist.""" + ple: Qwen4ExpPLELayer | None = getattr(layer, "ple", None) + if ple is None: + return + if input_ids is None or query_start_loc is None or ngram_context is None: + raise RuntimeError("PLE inputs were not prepared") + ple.start_prefetch( + hidden_states, + input_ids, + query_start_loc, + ngram_context, + ) + def forward( self, input_ids: torch.Tensor | None, @@ -487,10 +508,26 @@ class Qwen4ExpModel(nn.Module): block_output = None injection = None last_layer = None + if self.start_layer < self.end_layer: + self._start_layer_ple_prefetch( + self.layers[self.start_layer], + hidden_states, + input_ids, + query_start_loc, + ngram_context, + ) for layer_idx, layer in islice( enumerate(self.layers), self.start_layer, self.end_layer ): last_layer = layer + if layer_idx + 1 < self.end_layer: + self._start_layer_ple_prefetch( + self.layers[layer_idx + 1], + hidden_states, + input_ids, + query_start_loc, + ngram_context, + ) hidden_states, block_output, injection = layer( hidden_states=hidden_states, prev_block_output=block_output, diff --git a/vllm/models/qwen4_exp/nvidia/ngram_embedding.py b/vllm/models/qwen4_exp/nvidia/ngram_embedding.py new file mode 100644 index 0000000000..84c4e7c5e1 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ngram_embedding.py @@ -0,0 +1,939 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen4Exp n-gram embeddings with device and pinned-host storage.""" + +from abc import ABC, abstractmethod +from collections.abc import Iterable +from typing import ClassVar + +import torch +import torch.nn.functional as F +from torch import nn + +from vllm.compilation.breakable_cudagraph import eager_break_during_capture +from vllm.config import get_current_vllm_config +from vllm.distributed import get_dp_group, get_etp_group, get_tp_group +from vllm.forward_context import DPMetadata, get_forward_context +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, + QuantizeMethodBase, +) +from vllm.model_executor.layers.quantization.fp8 import Fp8Config +from vllm.model_executor.layers.quantization.modelopt import ( + ModelOptMixedPrecisionConfig, + ModelOptQuantConfigBase, +) +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + create_fp8_scale_parameter, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + is_layer_skipped, +) +from vllm.model_executor.models.utils import AutoWeightsLoader +from vllm.model_executor.parameter import ( + ModelWeightParameter, + PerTensorScaleParameter, +) +from vllm.model_executor.utils import set_weight_attrs +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) +from vllm.triton_utils import tl, triton +from vllm.utils.platform_utils import is_uva_available +from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor + +from ..common.ple import PLEVocabParallelEmbedding +from .ops.ple import ple_ngram_ids + +logger = init_logger(__name__) + + +class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC): + """ETP-sharded PLE table shared by device and pinned-host backends.""" + + supports_prefetch: ClassVar[bool] = False + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + *, + params_dtype: torch.dtype, + padding_size: int, + prefix: str, + embedding_method: "Qwen4ExpPLEEmbeddingMethod", + num_ngram_heads: int = 1, + max_total_tokens: int = 0, + data_parallel_rank: int = 0, + ) -> None: + del num_ngram_heads, max_total_tokens + super().__init__( + num_embeddings, + embedding_dim, + params_dtype=params_dtype, + padding_size=padding_size, + prefix=prefix, + quant_method=embedding_method, + parallel_group=get_etp_group(), + ) + self.embedding_method = embedding_method + self.data_parallel_rank = data_parallel_rank + tp_size = get_tp_group().world_size + if self.tp_size % tp_size: + raise ValueError( + "ETP size must be divisible by TP size, but got " + f"ETP={self.tp_size} and TP={tp_size}" + ) + self.etp_data_parallel_size = self.tp_size // tp_size + + @abstractmethod + def allocate_embedding_weight( + self, + num_embeddings: int, + embedding_dim: int, + dtype: torch.dtype, + ) -> torch.Tensor: + """Allocate storage for the complete embedding weight.""" + raise NotImplementedError + + def dequantize( + self, + embeddings: torch.Tensor, + output_dtype: torch.dtype, + ) -> torch.Tensor: + """Delegate storage-format conversion to the embedding method.""" + return self.embedding_method.dequantize(self, embeddings, output_dtype) + + def _get_dp_gather_slot(self, local_num_tokens: int) -> tuple[int, int]: + """Return the per-DP slot size and this rank's slot offset.""" + if self.etp_data_parallel_size == 1: + return local_num_tokens, 0 + dp_metadata: DPMetadata | None = get_forward_context().dp_metadata + if dp_metadata is None: + raise RuntimeError("ETP spanning DP requires DP token metadata") + group_start = (self.data_parallel_rank // self.etp_data_parallel_size) * ( + self.etp_data_parallel_size + ) + group_end = group_start + self.etp_data_parallel_size + token_counts = dp_metadata.num_tokens_across_dp_cpu.tolist() + group_counts = token_counts[group_start:group_end] + slot_size = max(group_counts) + dp_rank = get_dp_group().rank_in_group + return slot_size, dp_rank * slot_size + + def _gather_dp_ids( + self, + ngram_ids: torch.Tensor, + slot_size: int, + ) -> torch.Tensor: + """Gather DP-local IDs that share one ETP-sharded PLE table.""" + if self.etp_data_parallel_size == 1: + return ngram_ids + if ngram_ids.shape[0] < slot_size: + padding = ngram_ids.new_zeros( + slot_size - ngram_ids.shape[0], ngram_ids.shape[1] + ) + ngram_ids = torch.cat((ngram_ids, padding), dim=0) + return get_dp_group().all_gather(ngram_ids, dim=0) + + def _select_embeddings( + self, + embeddings: torch.Tensor, + local_num_tokens: int, + slot_offset: int, + ) -> torch.Tensor: + """Select this DP rank's rows from the ETP-reduced embeddings.""" + if self.etp_data_parallel_size == 1: + return embeddings + return embeddings.narrow(0, slot_offset, local_num_tokens) + + @abstractmethod + def start_prefetch( + self, + hidden_states: torch.Tensor, + ngram_ids: torch.Tensor, + ) -> None: + """Start an asynchronous lookup when supported.""" + raise NotImplementedError + + +class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase): + """Quantization interface shared by resident and pinned PLE tables.""" + + # PLE post-load processing only validates scales in their current storage. + requires_device_loading: bool = False + + @staticmethod + def from_quant_config( + quant_config: QuantizationConfig | None, + prefix: str, + embedding_dtype: str | None = None, + ) -> "Qwen4ExpPLEEmbeddingMethod": + """Select the concrete PLE embedding format for a layer.""" + if embedding_dtype == "float8_e4m3fn": + return Qwen4ExpPLEFp8EmbeddingMethod() + if quant_config is None: + return Qwen4ExpPLEUnquantizedEmbeddingMethod() + if isinstance(quant_config, ModelOptMixedPrecisionConfig): + if quant_config._resolve_quant_algo(prefix) == "FP8": + return Qwen4ExpPLEFp8EmbeddingMethod() + return Qwen4ExpPLEUnquantizedEmbeddingMethod() + if isinstance( + quant_config, ModelOptQuantConfigBase + ) and quant_config.is_layer_excluded(prefix): + return Qwen4ExpPLEUnquantizedEmbeddingMethod() + if not isinstance(quant_config, Fp8Config): + raise NotImplementedError( + "Qwen4Exp PLE embedding does not support quantization config " + f"{type(quant_config).__name__}" + ) + + ignored_layers = quant_config.ignored_layers + if is_layer_skipped( + prefix, + ignored_layers, + quant_config.packed_modules_mapping, + match_mode=quant_config.ignored_layers_match_mode, + ): + return Qwen4ExpPLEUnquantizedEmbeddingMethod() + # PLE checkpoint shards form one runtime embedding parameter. + shard_prefix = f"{prefix}.shard_" + if any(name.startswith(shard_prefix) for name in ignored_layers): + return Qwen4ExpPLEUnquantizedEmbeddingMethod() + if not quant_config.is_checkpoint_fp8_serialized: + raise NotImplementedError( + "Qwen4Exp PLE embedding only supports serialized FP8 checkpoints" + ) + return Qwen4ExpPLEFp8EmbeddingMethod() + + def apply( + self, + layer: nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + raise NotImplementedError("PLE weights only support embedding lookup") + + def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor: + return F.embedding(input_, layer.weight) + + @abstractmethod + def dequantize( + self, + layer: nn.Module, + embeddings: torch.Tensor, + output_dtype: torch.dtype, + ) -> torch.Tensor: + """Convert looked-up PLE rows to the activation dtype.""" + raise NotImplementedError + + +class Qwen4ExpPLEUnquantizedEmbeddingMethod(Qwen4ExpPLEEmbeddingMethod): + """Unquantized PLE embedding storage and lookup semantics.""" + + def create_weights( + self, + layer: Qwen4ExpPLEEmbedding, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + del input_size, output_size + weight = nn.Parameter( + layer.allocate_embedding_weight( + sum(output_partition_sizes), + input_size_per_partition, + params_dtype, + ), + requires_grad=False, + ) + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) + set_weight_attrs(weight, extra_weight_attrs) + layer.register_parameter("weight", weight) + + def dequantize( + self, + layer: nn.Module, + embeddings: torch.Tensor, + output_dtype: torch.dtype, + ) -> torch.Tensor: + del layer, output_dtype + return embeddings + + +class Qwen4ExpPLEFp8EmbeddingMethod(Qwen4ExpPLEEmbeddingMethod): + """FP8 PLE embedding with one global checkpoint scale.""" + + def create_weights( + self, + layer: Qwen4ExpPLEEmbedding, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + del input_size, output_size, params_dtype + weight_loader = extra_weight_attrs.get("weight_loader") + weight = ModelWeightParameter( + data=layer.allocate_embedding_weight( + sum(output_partition_sizes), + input_size_per_partition, + torch.float8_e4m3fn, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + weight_scale = create_fp8_scale_parameter( + PerTensorScaleParameter, + output_partition_sizes, + input_size_per_partition, + None, + weight_loader, + scale_dtype=torch.float32, + ) + layer.register_parameter("weight_scale", weight_scale) + + def process_weights_after_loading(self, layer: nn.Module) -> None: + """Reject FP8 PLE checkpoints without a global scale.""" + sentinel = torch.finfo(torch.float32).min + if torch.any(layer.weight_scale == sentinel): + raise ValueError("FP8 PLE checkpoint is missing its global scale") + + def dequantize( + self, + layer: nn.Module, + embeddings: torch.Tensor, + output_dtype: torch.dtype, + ) -> torch.Tensor: + weight_scale = getattr(layer, "weight_scale", None) + if weight_scale is None: + raise RuntimeError("FP8 PLE embedding is missing its global scale") + if weight_scale.device != embeddings.device: + raise RuntimeError("FP8 PLE embedding scale must be on the output device") + return embeddings.to(output_dtype) * weight_scale.to(output_dtype) + + +class Qwen4ExpPLEDeviceEmbedding(Qwen4ExpPLEEmbedding): + """PLE table allocated on the active model device.""" + + def allocate_embedding_weight( + self, + num_embeddings: int, + embedding_dim: int, + dtype: torch.dtype, + ) -> torch.Tensor: + """Allocate the complete PLE weight on the active device.""" + return torch.empty(num_embeddings, embedding_dim, dtype=dtype) + + def start_prefetch( + self, + hidden_states: torch.Tensor, + ngram_ids: torch.Tensor, + ) -> None: + """Resident embedding prefetch is a no-op.""" + return None + + def forward(self, ngram_ids: torch.Tensor) -> torch.Tensor: + """Gather ETP inputs, look up embeddings, and select local rows.""" + slot_size, slot_offset = self._get_dp_gather_slot(ngram_ids.shape[0]) + gathered_ids = self._gather_dp_ids(ngram_ids, slot_size) + embeddings = super().forward(gathered_ids) + return self._select_embeddings( + embeddings, + ngram_ids.shape[0], + slot_offset, + ) + + +@triton.jit +def _lookup_ple_embedding_from_pinned_kernel( + weight_ptr, + ids_ptr, + output_ptr, + embedding_dim, + tp_vocab_start, + tp_vocab_end, + BLOCK_D: tl.constexpr, +): + """Look up TP-owned PLE rows through a CUDA view of pinned host memory.""" + row_id = tl.program_id(0) + global_idx = tl.load(ids_ptr + row_id) + in_range = (global_idx >= tp_vocab_start) & (global_idx < tp_vocab_end) + local_idx = tl.where(in_range, global_idx - tp_vocab_start, 0) + offsets = tl.arange(0, BLOCK_D) + store_mask = offsets < embedding_dim + load_mask = store_mask & in_range + values = tl.load( + weight_ptr + local_idx * embedding_dim + offsets, + mask=load_mask, + other=0.0, + ) + tl.store( + output_ptr + row_id * embedding_dim + offsets, + values, + mask=store_mask, + ) + + +class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): + """PLE table loaded into pinned CPU memory and looked up through UVA.""" + + supports_prefetch: ClassVar[bool] = True + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + *, + params_dtype: torch.dtype, + padding_size: int, + prefix: str, + embedding_method: Qwen4ExpPLEEmbeddingMethod, + num_ngram_heads: int = 1, + max_total_tokens: int = 0, + data_parallel_rank: int = 0, + ) -> None: + if not is_uva_available(): + raise RuntimeError("Engram CPU offload requires UVA support") + super().__init__( + num_embeddings, + embedding_dim, + params_dtype=params_dtype, + padding_size=padding_size, + prefix=prefix, + embedding_method=embedding_method, + num_ngram_heads=num_ngram_heads, + max_total_tokens=max_total_tokens, + data_parallel_rank=data_parallel_rank, + ) + self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight) + self._block_d = triton.next_power_of_2(self.embedding_dim) + self._prefetch_stream = torch.cuda.Stream(device=self._uva_weight.device) + self._prefetch_buffer = torch.empty( + max_total_tokens * self.etp_data_parallel_size, + num_ngram_heads, + self.embedding_dim, + dtype=self.weight.dtype, + device=self._uva_weight.device, + ) + self._output_dim = num_ngram_heads * self.embedding_dim + + def allocate_embedding_weight( + self, + num_embeddings: int, + embedding_dim: int, + dtype: torch.dtype, + ) -> torch.Tensor: + """Allocate the complete PLE weight directly in pinned CPU memory.""" + return torch.empty( + num_embeddings, + embedding_dim, + dtype=dtype, + device="cpu", + pin_memory=True, + ) + + def _lookup( + self, + input_ids: torch.Tensor, + output: torch.Tensor | None = None, + ) -> torch.Tensor: + """Look up local ETP rows while preserving the weight storage dtype.""" + expected_shape = (*input_ids.shape, self.embedding_dim) + if output is None: + output = torch.empty( + expected_shape, + dtype=self.weight.dtype, + device=input_ids.device, + ) + elif ( + tuple(output.shape) != expected_shape + or output.dtype != self.weight.dtype + or output.device != input_ids.device + ): + raise ValueError( + "PLE prefetch output must match the input shape, weight dtype, " + "and input device" + ) + + flat_ids = input_ids.reshape(-1).long() + if flat_ids.numel(): + _lookup_ple_embedding_from_pinned_kernel[(flat_ids.numel(),)]( + self._uva_weight, + flat_ids, + output, + self.embedding_dim, + self.shard_indices.org_vocab_start_index, + self.shard_indices.org_vocab_end_index, + BLOCK_D=self._block_d, + ) + return output + + def _reduce_etp_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor: + """Combine pinned lookup results owned by different ETP ranks.""" + if self.tp_size == 1: + return embeddings + assert self.parallel_group is not None + if embeddings.dtype in (torch.float8_e4m3fn, torch.float8_e5m2): + # Each vocabulary row has one owner, so reduce the raw FP8 bytes. + reduced = self.parallel_group.all_reduce(embeddings.view(torch.int8)) + return reduced.view(embeddings.dtype) + return self.parallel_group.all_reduce(embeddings) + + @eager_break_during_capture + def start_prefetch( + self, + hidden_states: torch.Tensor, + ngram_ids: torch.Tensor, + ) -> None: + """Gather ETP IDs and launch their UVA lookup on the side stream.""" + slot_size, _ = self._get_dp_gather_slot(ngram_ids.shape[0]) + gathered_ids = self._gather_dp_ids(ngram_ids, slot_size) + active_output = self._prefetch_buffer[: gathered_ids.shape[0]] + prefetch_stream = self._prefetch_stream + prefetch_stream.wait_stream(torch.cuda.current_stream()) + gathered_ids.record_stream(prefetch_stream) + with torch.cuda.stream(prefetch_stream): + self._lookup(gathered_ids, output=active_output) + + @eager_break_during_capture + def _finalize_prefetch( + self, + prefetch_output: torch.Tensor, + output: torch.Tensor, + ) -> None: + """Join the side stream, reduce ETP shards, and select local rows.""" + torch.cuda.current_stream().wait_stream(self._prefetch_stream) + slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0]) + active_output = prefetch_output[: slot_size * self.etp_data_parallel_size] + embeddings = self._reduce_etp_embeddings(active_output) + embeddings = self._select_embeddings( + embeddings, + output.shape[0], + slot_offset, + ) + output.copy_(embeddings.flatten(-2)) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Finish the pinned lookup into graph-owned output storage.""" + output = self._prefetch_buffer.new_empty( + (hidden_states.shape[0], self._output_dim) + ) + self._finalize_prefetch(self._prefetch_buffer, output) + return output + + +class Qwen4ExpNGramEmbedding(nn.Module): + _MASK64 = (1 << 64) - 1 + _SPLITMIX_GAMMA = 0x9E3779B97F4A7C15 + _SPLITMIX_M1 = 0xBF58476D1CE4E5B9 + _SPLITMIX_M2 = 0x94D049BB133111EB + _PLE_LAYER_PRIME = 10007 + + @classmethod + def _splitmix64(cls, value: int) -> int: + """Mix an integer into a deterministic unsigned 64-bit value.""" + value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64 + value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64 + value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64 + return (value ^ (value >> 31)) & cls._MASK64 + + @staticmethod + def _is_prime_64(value: int) -> bool: + """Return whether a 64-bit integer is prime.""" + if value < 2: + return False + for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37): + if value % prime == 0: + return value == prime + exponent = value - 1 + shifts = 0 + while exponent % 2 == 0: + exponent //= 2 + shifts += 1 + for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022): + if base % value == 0: + continue + witness = pow(base, exponent, value) + if witness in (1, value - 1): + continue + for _ in range(shifts - 1): + witness = pow(witness, 2, value) + if witness == value - 1: + break + else: + return False + return True + + @classmethod + def _nth_prime_after(cls, start: int, count: int) -> int: + """Return the ``count``-th prime strictly greater than ``start``.""" + prime = int(start) + for _ in range(count): + candidate = prime + 1 + if candidate <= 2: + prime = 2 + continue + if candidate % 2 == 0: + candidate += 1 + while not cls._is_prime_64(candidate): + candidate += 2 + prime = candidate + return prime + + @classmethod + def _make_layer_multipliers( + cls, + *, + ngram_size: int, + unigram_vocab_size: int, + seed: int, + ple_dense_layer_id: int, + ) -> list[int]: + """Build deterministic hash multipliers for one PLE layer.""" + max_multiplier = ((1 << 63) - 1) // unigram_vocab_size + half_bound = max(1, max_multiplier // 2) + base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id + multipliers = [] + for index in range(ngram_size): + value = base_seed + cls._SPLITMIX_GAMMA * (index + 1) + multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1) + return multipliers + + @classmethod + def _make_vocab_layout( + cls, + *, + ngram_vocab_size_base: int, + ngram_heads: int, + ple_dense_layer_id: int, + ) -> tuple[list[int], list[int], int]: + """Build per-head vocabulary sizes, offsets, and total row count.""" + sizes: list[int] = [] + offsets: list[int] = [] + offset = 0 + for local_head in range(ngram_heads): + global_head = ple_dense_layer_id * ngram_heads + local_head + size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1) + sizes.append(size) + offsets.append(offset) + offset += size + return sizes, offsets, offset + + def __init__( + self, + config: Qwen4ExpTextConfig, + embedding_dim: int, + ple_dense_layer_id: int, + max_total_tokens: int, + *, + data_parallel_rank: int, + prefix: str, + quant_config: QuantizationConfig | None = None, + params_dtype: torch.dtype | None = None, + ) -> None: + super().__init__() + self.embedding_dim = embedding_dim + self.ngram_size = int(config.ngram_size) + self.heads_per_ngram = int(config.heads_per_ngram) + self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram + if self.ngram_size < 2: + raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}") + if self.heads_per_ngram <= 0: + raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}") + if embedding_dim % self.ngram_heads: + raise ValueError( + "ple_embed_dim must be divisible by total ngram heads: " + f"{embedding_dim} % {self.ngram_heads} != 0" + ) + self.head_dim = embedding_dim // self.ngram_heads + self.eos_token_id = int(config.eos_token_id) + self.unigram_vocab_size = int(config.vocab_size) + self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512)) + if self.split_ngram_parts <= 0: + raise ValueError("split_ngram_parts must be positive") + + multipliers = self._make_layer_multipliers( + ngram_size=self.ngram_size, + unigram_vocab_size=self.unigram_vocab_size, + seed=int(getattr(config, "seed", 1234)), + ple_dense_layer_id=ple_dense_layer_id, + ) + self.register_buffer( + "layer_multipliers", + torch.tensor(multipliers, dtype=torch.long), + persistent=True, + ) + + sizes, offsets, total_vocab_size = self._make_vocab_layout( + ngram_vocab_size_base=int(config.ngram_vocab_size_base), + ngram_heads=self.ngram_heads, + ple_dense_layer_id=ple_dense_layer_id, + ) + self.register_buffer( + "ngram_heads_vocab_sizes", + torch.tensor(sizes, dtype=torch.long), + persistent=True, + ) + self.register_buffer( + "ngram_heads_offsets", + torch.tensor(offsets, dtype=torch.long), + persistent=True, + ) + divisor = int(config.make_ngram_vocab_size_divisible_by) + padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor + embedding_prefix = f"{prefix}.ngram_embedding" + embedding_quant_method = Qwen4ExpPLEEmbeddingMethod.from_quant_config( + quant_config, + embedding_prefix, + getattr(config, "ple_embedding_dtype", None), + ) + if params_dtype is None: + params_dtype = torch.get_default_dtype() + engram_config = get_current_vllm_config().engram_config + embedding_cls = ( + Qwen4ExpPLEPinnedHostEmbedding + if engram_config is not None and engram_config.cpu_offload + else Qwen4ExpPLEDeviceEmbedding + ) + self.ngram_embedding = embedding_cls( + padded_vocab_size, + self.head_dim, + params_dtype=params_dtype, + padding_size=divisor, + prefix=embedding_prefix, + embedding_method=embedding_quant_method, + num_ngram_heads=self.ngram_heads, + max_total_tokens=max_total_tokens, + data_parallel_rank=data_parallel_rank, + ) + weight = self.ngram_embedding.weight + logger.info( + "Initialized PLE embedding %s: quantization_method=%s, " + "weight_dtype=%s, weight_device=%s, pinned=%s", + embedding_prefix, + type(embedding_quant_method).__name__, + weight.dtype, + weight.device, + weight.is_pinned(), + ) + + @staticmethod + def _shift_precompute( + tokens: torch.Tensor, eos_token_id: int + ) -> tuple[torch.Tensor, torch.Tensor]: + if tokens.dim() != 2: + raise ValueError("tokens must be a 2D tensor") + batch_size, seq_len = tokens.shape + positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64) + eos_positions = torch.where(tokens == eos_token_id, positions, -1) + previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values + previous_eos = torch.cat( + [ + eos_positions.new_full((batch_size, 1), -1), + previous_eos_inclusive[:, :-1], + ], + dim=1, + ) + return positions, positions.unsqueeze(0) - previous_eos - 1 + + @staticmethod + def _shift_apply( + tokens: torch.Tensor, + positions: torch.Tensor, + position_in_segment: torch.Tensor, + shift: int, + eos_token_id: int, + ) -> torch.Tensor: + if shift == 0: + return tokens + source = positions - shift + gather_indices = source.clamp_min(0).unsqueeze(0).expand(tokens.shape[0], -1) + shifted = tokens.gather(1, gather_indices) + valid = (source.unsqueeze(0) >= 0) & (position_in_segment >= shift) + return torch.where(valid, shifted, tokens.new_full((), eos_token_id)) + + def compute_ngram_ids( + self, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + output: torch.Tensor | None = None, + ) -> torch.Tensor: + """Compute n-gram embedding indices for the current request layout.""" + input_ids = input_ids.reshape(-1) + num_reqs = query_start_loc.numel() - 1 + num_tokens = input_ids.shape[0] + + if input_ids.is_cuda: + return ple_ngram_ids( + input_ids=input_ids, + query_start_loc=query_start_loc, + ngram_context=ngram_context, + layer_multipliers=self.layer_multipliers, + ngram_heads_vocab_sizes=self.ngram_heads_vocab_sizes, + ngram_heads_offsets=self.ngram_heads_offsets, + eos_token_id=self.eos_token_id, + heads_per_ngram=self.heads_per_ngram, + output=output, + ) + input_ids = input_ids.long() + query_start_loc = query_start_loc.long() + positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64) + packed = torch.full( + (num_reqs, num_tokens), + self.eos_token_id, + device=input_ids.device, + dtype=torch.int64, + ) + request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1 + request_indices.clamp_(max=num_reqs - 1) + columns = (positions - query_start_loc[request_indices]).clamp( + 0, packed.shape[1] - 1 + ) + packed[request_indices, columns] = input_ids + ngram_context = ngram_context[:num_reqs].to( + device=input_ids.device, dtype=torch.long + ) + + context = torch.cat([ngram_context, packed], dim=-1) + positions_2d, position_in_segment = self._shift_precompute( + context, self.eos_token_id + ) + shifted = [context] + for shift in range(1, self.ngram_size): + shifted.append( + self._shift_apply( + context, + positions_2d, + position_in_segment, + shift, + self.eos_token_id, + ) + ) + adjusted_columns = columns + self.ngram_size - 1 + id_blocks = [] + for ngram in range(2, self.ngram_size + 1): + start = (ngram - 2) * self.heads_per_ngram + end = start + self.heads_per_ngram + mixed = shifted[0] * self.layer_multipliers[0] + for index in range(1, ngram): + mixed = torch.bitwise_xor( + mixed, shifted[index] * self.layer_multipliers[index] + ) + sizes = self.ngram_heads_vocab_sizes[start:end] + offsets = self.ngram_heads_offsets[start:end] + ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets + id_blocks.append(ids[request_indices, adjusted_columns]) + return torch.cat(id_blocks, dim=-1) + + def forward( + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> torch.Tensor: + embedding = self.ngram_embedding + if embedding.supports_prefetch: + return embedding(hidden_states) + ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context) + return self.ngram_embedding(ngram_ids).flatten(-2) + + def start_prefetch( + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> None: + """Start the pinned lookup while the preceding decoder layer runs.""" + embedding = self.ngram_embedding + if not embedding.supports_prefetch: + return + ngram_ids = self.compute_ngram_ids( + input_ids, + query_start_loc, + ngram_context, + ) + embedding.start_prefetch(hidden_states, ngram_ids) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load hash buffers and checkpoint-split embedding rows.""" + + persistent_buffers = { + "layer_multipliers": self.layer_multipliers, + "ngram_heads_offsets": self.ngram_heads_offsets, + "ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes, + } + loaded: set[str] = set() + regular_weights: list[tuple[str, torch.Tensor]] = [] + shard_prefix = "ngram_embedding.shard_" + + for name, loaded_weight in weights: + leaf_name = name.rsplit(".", 1)[-1] + if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup": + continue + if name in persistent_buffers: + buffer = persistent_buffers[name] + if buffer.shape != loaded_weight.shape: + raise ValueError( + f"Shape mismatch for {name}: expected " + f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}" + ) + buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype)) + loaded.add(name) + continue + if name.startswith(shard_prefix) and name.endswith(".weight"): + shard_text = name[len(shard_prefix) : -len(".weight")] + if not shard_text.isdigit(): + regular_weights.append((name, loaded_weight)) + continue + shard_index = int(shard_text) + if shard_index >= self.split_ngram_parts: + raise ValueError( + f"PLE embedding shard index {shard_index} exceeds " + f"split_ngram_parts={self.split_ngram_parts}" + ) + embedding = self.ngram_embedding + shard_size = ( + embedding.org_vocab_size + self.split_ngram_parts - 1 + ) // self.split_ngram_parts + checkpoint_start = shard_index * shard_size + expected_rows = max( + 0, + min(shard_size, embedding.org_vocab_size - checkpoint_start), + ) + expected_shape = (expected_rows, embedding.embedding_dim) + if tuple(loaded_weight.shape) != expected_shape: + raise ValueError( + f"Shape mismatch for PLE embedding shard {shard_index}: " + f"expected {expected_shape}, got " + f"{tuple(loaded_weight.shape)}" + ) + embedding.weight.weight_loader( + embedding.weight, + loaded_weight, + checkpoint_start=checkpoint_start, + ) + loaded.add("ngram_embedding.weight") + continue + regular_weights.append((name, loaded_weight)) + + if regular_weights: + loaded.update(AutoWeightsLoader(self).load_weights(regular_weights)) + return loaded + + +__all__ = [ + "Qwen4ExpNGramEmbedding", +] diff --git a/vllm/models/qwen4_exp/nvidia/ple_layer.py b/vllm/models/qwen4_exp/nvidia/ple_layer.py index 7650e2e3a9..26622357ca 100644 --- a/vllm/models/qwen4_exp/nvidia/ple_layer.py +++ b/vllm/models/qwen4_exp/nvidia/ple_layer.py @@ -2,10 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """GPU-resident Qwen4Exp position-learning enhancement layers.""" -from collections.abc import Iterable, Sequence +from collections.abc import Sequence import torch -import torch.nn.functional as F from torch import nn from vllm.compilation.breakable_cudagraph import eager_break_during_capture @@ -18,24 +17,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateShapeCalculator, is_conv_state_dim_first, ) -from vllm.model_executor.layers.quantization.base_config import ( - QuantizationConfig, - QuantizeMethodBase, -) -from vllm.model_executor.layers.quantization.fp8 import Fp8Config -from vllm.model_executor.layers.quantization.modelopt import ( - ModelOptMixedPrecisionConfig, -) -from vllm.model_executor.layers.quantization.utils.fp8_utils import ( - create_fp8_scale_parameter, - create_fp8_weight_parameter, - is_fp8, -) -from vllm.model_executor.layers.quantization.utils.quant_utils import ( - is_layer_skipped, -) -from vllm.model_executor.models.utils import AutoWeightsLoader -from vllm.model_executor.parameter import PerTensorScaleParameter from vllm.transformers_utils.configs.qwen4_exp import ( Qwen4ExpTextConfig, ) @@ -45,8 +26,8 @@ from vllm.v1.attention.backends.short_conv_attn import ( PleShortConvAttentionMetadata, ) -from ..common.ple import PLEVocabParallelEmbedding -from .ops.ple import ple_conv, ple_gate, ple_ngram_ids +from .ngram_embedding import Qwen4ExpNGramEmbedding +from .ops.ple import ple_conv, ple_gate class Qwen4ExpPLEGroupedNorm(nn.Module): @@ -82,437 +63,6 @@ class Qwen4ExpPLEGroupedNorm(nn.Module): return (normalized * (1.0 + self.weight.float())).to(input_dtype) -class Qwen4ExpPLEFp8EmbeddingMethod(QuantizeMethodBase): - """FP8 PLE embedding with one global checkpoint scale.""" - - def create_weights( - self, - layer: nn.Module, - input_size_per_partition: int, - output_partition_sizes: list[int], - input_size: int, - output_size: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ) -> None: - del input_size, output_size, params_dtype - weight_loader = extra_weight_attrs.get("weight_loader") - weight = create_fp8_weight_parameter( - sum(output_partition_sizes), input_size_per_partition, weight_loader - ) - layer.register_parameter("weight", weight) - - weight_scale = create_fp8_scale_parameter( - PerTensorScaleParameter, - output_partition_sizes, - input_size_per_partition, - None, - weight_loader, - scale_dtype=torch.float32, - ) - layer.register_parameter("weight_scale", weight_scale) - - def process_weights_after_loading(self, layer: nn.Module) -> None: - """Reject FP8 PLE checkpoints without a global scale.""" - sentinel = torch.finfo(torch.float32).min - if torch.any(layer.weight_scale == sentinel): - raise ValueError("FP8 PLE checkpoint is missing its global scale") - - def apply( - self, - layer: nn.Module, - x: torch.Tensor, - bias: torch.Tensor | None = None, - ) -> torch.Tensor: - raise NotImplementedError("PLE FP8 weights only support embedding lookup") - - def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor: - return F.embedding(input_, layer.weight) - - -def _get_ple_embedding_quant_method( - quant_config: QuantizationConfig | None, - prefix: str, -) -> QuantizeMethodBase | None: - """Select global-scale FP8 only for quantized PLE checkpoint shards.""" - - if isinstance(quant_config, ModelOptMixedPrecisionConfig): - if quant_config._resolve_quant_algo(prefix) == "FP8": - return Qwen4ExpPLEFp8EmbeddingMethod() - return None - - if not isinstance(quant_config, Fp8Config): - return None - if not quant_config.is_checkpoint_fp8_serialized: - return None - - ignored_layers = quant_config.ignored_layers - if is_layer_skipped( - prefix, - ignored_layers, - quant_config.packed_modules_mapping, - match_mode=quant_config.ignored_layers_match_mode, - ): - return None - # PLE checkpoint shards form one runtime embedding parameter. - shard_prefix = f"{prefix}.shard_" - if any(name.startswith(shard_prefix) for name in ignored_layers): - return None - return Qwen4ExpPLEFp8EmbeddingMethod() - - -class Qwen4ExpNGramEmbedding(nn.Module): - _MASK64 = (1 << 64) - 1 - _SPLITMIX_GAMMA = 0x9E3779B97F4A7C15 - _SPLITMIX_M1 = 0xBF58476D1CE4E5B9 - _SPLITMIX_M2 = 0x94D049BB133111EB - _PLE_LAYER_PRIME = 10007 - - @classmethod - def _splitmix64(cls, value: int) -> int: - """Mix an integer into a deterministic unsigned 64-bit value.""" - value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64 - value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64 - value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64 - return (value ^ (value >> 31)) & cls._MASK64 - - @staticmethod - def _is_prime_64(value: int) -> bool: - """Return whether a 64-bit integer is prime.""" - if value < 2: - return False - for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37): - if value % prime == 0: - return value == prime - exponent = value - 1 - shifts = 0 - while exponent % 2 == 0: - exponent //= 2 - shifts += 1 - for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022): - if base % value == 0: - continue - witness = pow(base, exponent, value) - if witness in (1, value - 1): - continue - for _ in range(shifts - 1): - witness = pow(witness, 2, value) - if witness == value - 1: - break - else: - return False - return True - - @classmethod - def _nth_prime_after(cls, start: int, count: int) -> int: - """Return the ``count``-th prime strictly greater than ``start``.""" - prime = int(start) - for _ in range(count): - candidate = prime + 1 - if candidate <= 2: - prime = 2 - continue - if candidate % 2 == 0: - candidate += 1 - while not cls._is_prime_64(candidate): - candidate += 2 - prime = candidate - return prime - - @classmethod - def _make_layer_multipliers( - cls, - *, - ngram_size: int, - unigram_vocab_size: int, - seed: int, - ple_dense_layer_id: int, - ) -> list[int]: - """Build deterministic hash multipliers for one PLE layer.""" - max_multiplier = ((1 << 63) - 1) // unigram_vocab_size - half_bound = max(1, max_multiplier // 2) - base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id - multipliers = [] - for index in range(ngram_size): - value = base_seed + cls._SPLITMIX_GAMMA * (index + 1) - multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1) - return multipliers - - @classmethod - def _make_vocab_layout( - cls, - *, - ngram_vocab_size_base: int, - ngram_heads: int, - ple_dense_layer_id: int, - ) -> tuple[list[int], list[int], int]: - """Build per-head vocabulary sizes, offsets, and total row count.""" - sizes: list[int] = [] - offsets: list[int] = [] - offset = 0 - for local_head in range(ngram_heads): - global_head = ple_dense_layer_id * ngram_heads + local_head - size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1) - sizes.append(size) - offsets.append(offset) - offset += size - return sizes, offsets, offset - - def __init__( - self, - config: Qwen4ExpTextConfig, - embedding_dim: int, - ple_dense_layer_id: int, - prefix: str, - quant_config: QuantizationConfig | None = None, - params_dtype: torch.dtype | None = None, - ) -> None: - super().__init__() - self.embedding_dim = embedding_dim - self.ngram_size = int(config.ngram_size) - self.heads_per_ngram = int(config.heads_per_ngram) - self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram - if self.ngram_size < 2: - raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}") - if self.heads_per_ngram <= 0: - raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}") - if embedding_dim % self.ngram_heads: - raise ValueError( - "ple_embed_dim must be divisible by total ngram heads: " - f"{embedding_dim} % {self.ngram_heads} != 0" - ) - self.head_dim = embedding_dim // self.ngram_heads - self.eos_token_id = int(config.eos_token_id) - self.unigram_vocab_size = int(config.vocab_size) - self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512)) - if self.split_ngram_parts <= 0: - raise ValueError("split_ngram_parts must be positive") - - multipliers = self._make_layer_multipliers( - ngram_size=self.ngram_size, - unigram_vocab_size=self.unigram_vocab_size, - seed=int(getattr(config, "seed", 1234)), - ple_dense_layer_id=ple_dense_layer_id, - ) - self.register_buffer( - "layer_multipliers", - torch.tensor(multipliers, dtype=torch.long), - persistent=True, - ) - - sizes, offsets, total_vocab_size = self._make_vocab_layout( - ngram_vocab_size_base=int(config.ngram_vocab_size_base), - ngram_heads=self.ngram_heads, - ple_dense_layer_id=ple_dense_layer_id, - ) - self.register_buffer( - "ngram_heads_vocab_sizes", - torch.tensor(sizes, dtype=torch.long), - persistent=True, - ) - self.register_buffer( - "ngram_heads_offsets", - torch.tensor(offsets, dtype=torch.long), - persistent=True, - ) - divisor = int(config.make_ngram_vocab_size_divisible_by) - padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor - self.ngram_embedding = PLEVocabParallelEmbedding( - padded_vocab_size, - self.head_dim, - params_dtype=params_dtype, - padding_size=divisor, - prefix=f"{prefix}.ngram_embedding", - quant_method=_get_ple_embedding_quant_method( - quant_config, f"{prefix}.ngram_embedding" - ), - ) - - @staticmethod - def _shift_precompute( - tokens: torch.Tensor, eos_token_id: int - ) -> tuple[torch.Tensor, torch.Tensor]: - if tokens.dim() != 2: - raise ValueError("tokens must be a 2D tensor") - batch_size, seq_len = tokens.shape - positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64) - eos_positions = torch.where(tokens == eos_token_id, positions, -1) - previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values - previous_eos = torch.cat( - [ - eos_positions.new_full((batch_size, 1), -1), - previous_eos_inclusive[:, :-1], - ], - dim=1, - ) - return positions, positions.unsqueeze(0) - previous_eos - 1 - - @staticmethod - def _shift_apply( - tokens: torch.Tensor, - positions: torch.Tensor, - position_in_segment: torch.Tensor, - shift: int, - eos_token_id: int, - ) -> torch.Tensor: - if shift == 0: - return tokens - source = positions - shift - gather_indices = source.clamp_min(0).unsqueeze(0).expand(tokens.shape[0], -1) - shifted = tokens.gather(1, gather_indices) - valid = (source.unsqueeze(0) >= 0) & (position_in_segment >= shift) - return torch.where(valid, shifted, tokens.new_full((), eos_token_id)) - - def compute_ngram_ids( - self, - input_ids: torch.Tensor, - query_start_loc: torch.Tensor, - ngram_context: torch.Tensor, - output: torch.Tensor | None = None, - ) -> torch.Tensor: - """Compute n-gram embedding indices for the current request layout.""" - input_ids = input_ids.reshape(-1) - num_reqs = query_start_loc.numel() - 1 - num_tokens = input_ids.shape[0] - - if input_ids.is_cuda: - return ple_ngram_ids( - input_ids=input_ids, - query_start_loc=query_start_loc, - ngram_context=ngram_context, - layer_multipliers=self.layer_multipliers, - ngram_heads_vocab_sizes=self.ngram_heads_vocab_sizes, - ngram_heads_offsets=self.ngram_heads_offsets, - eos_token_id=self.eos_token_id, - heads_per_ngram=self.heads_per_ngram, - output=output, - ) - input_ids = input_ids.long() - query_start_loc = query_start_loc.long() - positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64) - packed = torch.full( - (num_reqs, num_tokens), - self.eos_token_id, - device=input_ids.device, - dtype=torch.int64, - ) - request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1 - request_indices.clamp_(max=num_reqs - 1) - columns = (positions - query_start_loc[request_indices]).clamp( - 0, packed.shape[1] - 1 - ) - packed[request_indices, columns] = input_ids - ngram_context = ngram_context[:num_reqs].to( - device=input_ids.device, dtype=torch.long - ) - - context = torch.cat([ngram_context, packed], dim=-1) - positions_2d, position_in_segment = self._shift_precompute( - context, self.eos_token_id - ) - shifted = [context] - for shift in range(1, self.ngram_size): - shifted.append( - self._shift_apply( - context, - positions_2d, - position_in_segment, - shift, - self.eos_token_id, - ) - ) - adjusted_columns = columns + self.ngram_size - 1 - id_blocks = [] - for ngram in range(2, self.ngram_size + 1): - start = (ngram - 2) * self.heads_per_ngram - end = start + self.heads_per_ngram - mixed = shifted[0] * self.layer_multipliers[0] - for index in range(1, ngram): - mixed = torch.bitwise_xor( - mixed, shifted[index] * self.layer_multipliers[index] - ) - sizes = self.ngram_heads_vocab_sizes[start:end] - offsets = self.ngram_heads_offsets[start:end] - ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets - id_blocks.append(ids[request_indices, adjusted_columns]) - return torch.cat(id_blocks, dim=-1) - - def forward( - self, - input_ids: torch.Tensor, - query_start_loc: torch.Tensor, - ngram_context: torch.Tensor, - ) -> torch.Tensor: - ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context) - return self.ngram_embedding(ngram_ids).flatten(-2) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Load hash buffers and checkpoint-split embedding rows.""" - - persistent_buffers = { - "layer_multipliers": self.layer_multipliers, - "ngram_heads_offsets": self.ngram_heads_offsets, - "ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes, - } - loaded: set[str] = set() - regular_weights: list[tuple[str, torch.Tensor]] = [] - shard_prefix = "ngram_embedding.shard_" - - for name, loaded_weight in weights: - leaf_name = name.rsplit(".", 1)[-1] - if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup": - continue - if name in persistent_buffers: - buffer = persistent_buffers[name] - if buffer.shape != loaded_weight.shape: - raise ValueError( - f"Shape mismatch for {name}: expected " - f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}" - ) - buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype)) - loaded.add(name) - continue - if name.startswith(shard_prefix) and name.endswith(".weight"): - shard_text = name[len(shard_prefix) : -len(".weight")] - if not shard_text.isdigit(): - regular_weights.append((name, loaded_weight)) - continue - shard_index = int(shard_text) - if shard_index >= self.split_ngram_parts: - raise ValueError( - f"PLE embedding shard index {shard_index} exceeds " - f"split_ngram_parts={self.split_ngram_parts}" - ) - embedding = self.ngram_embedding - shard_size = ( - embedding.org_vocab_size + self.split_ngram_parts - 1 - ) // self.split_ngram_parts - checkpoint_start = shard_index * shard_size - expected_rows = max( - 0, - min(shard_size, embedding.org_vocab_size - checkpoint_start), - ) - expected_shape = (expected_rows, embedding.embedding_dim) - if tuple(loaded_weight.shape) != expected_shape: - raise ValueError( - f"Shape mismatch for PLE embedding shard {shard_index}: " - f"expected {expected_shape}, got " - f"{tuple(loaded_weight.shape)}" - ) - embedding.weight.weight_loader( - embedding.weight, - loaded_weight, - checkpoint_start=checkpoint_start, - ) - loaded.add("ngram_embedding.weight") - continue - regular_weights.append((name, loaded_weight)) - - if regular_weights: - loaded.update(AutoWeightsLoader(self).load_weights(regular_weights)) - return loaded - - class Qwen4ExpPLELayer(nn.Module, MambaBase): def __init__( self, @@ -547,6 +97,8 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): config, int(config.ple_embed_dim), self.ple_dense_layer_id, + vllm_config.scheduler_config.max_num_batched_tokens, + data_parallel_rank=vllm_config.parallel_config.data_parallel_rank, prefix=f"{prefix}.ple_embedding", quant_config=quant_config, params_dtype=model_config.dtype, @@ -588,10 +140,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self - def _get_embedding_weight_scale(self) -> torch.Tensor | None: - embedding = getattr(self.ple_embedding, "ngram_embedding", None) - return getattr(embedding, "weight_scale", None) - def _dequantize_embeddings( self, embeddings: torch.Tensor, @@ -599,14 +147,25 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): ) -> torch.Tensor: """Dequantize PLE lookup output.""" - if not is_fp8(embeddings): - return embeddings - weight_scale = self._get_embedding_weight_scale() - if weight_scale is None: - raise RuntimeError("FP8 PLE embedding is missing its global scale") - if weight_scale.device != embeddings.device: - raise RuntimeError("FP8 PLE embedding scale must be on the output device") - return embeddings.to(output_dtype) * weight_scale.to(output_dtype) + return self.ple_embedding.ngram_embedding.dequantize( + embeddings, + output_dtype, + ) + + def start_prefetch( + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> None: + """Start the pinned PLE lookup while the preceding decoder layer runs.""" + self.ple_embedding.start_prefetch( + hidden_states, + input_ids, + query_start_loc, + ngram_context, + ) @property def mamba_type(self) -> MambaAttentionBackendEnum: @@ -827,7 +386,12 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): f"token length, got {input_ids.shape[0]} and " f"{hidden_states.shape[0]}" ) - embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context) + embeddings = self.ple_embedding( + hidden_states, + input_ids, + query_start_loc, + ngram_context, + ) embeddings = self._dequantize_embeddings(embeddings, hidden_states.dtype) kv, _ = self.kv_proj(embeddings) key, value = kv.split(self.kv_proj.output_sizes, dim=-1) @@ -845,7 +409,6 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): __all__ = [ - "Qwen4ExpNGramEmbedding", "Qwen4ExpPLEGroupedNorm", "Qwen4ExpPLELayer", ]