[Quantization] Support packed NVFP4 Qwen4Exp PLE embeddings (#56273)
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: Kartikey Vyas <kartikey@artificialanalysis.ai>
This commit is contained in:
co-authored by
Kartikey Vyas
parent
87e7dce3a7
commit
4e0681ed56
@@ -41,6 +41,7 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
Qwen4ExpPLEEmbeddingMethod,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLENvFp4EmbeddingMethod,
|
||||
Qwen4ExpPLEPinnedHostEmbedding,
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
@@ -174,9 +175,10 @@ def _mock_etp_group(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
world_size: int = 1,
|
||||
all_reduce=lambda tensor: tensor,
|
||||
rank: int = 0,
|
||||
) -> None:
|
||||
group = SimpleNamespace(
|
||||
rank_in_group=0,
|
||||
rank_in_group=rank,
|
||||
world_size=world_size,
|
||||
all_reduce=all_reduce,
|
||||
)
|
||||
@@ -325,6 +327,162 @@ def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
|
||||
)
|
||||
|
||||
|
||||
def _make_nvfp4_ngram_embedding(monkeypatch, *, rank=0, device="cpu", pinned=False):
|
||||
_mock_etp_group(monkeypatch, world_size=2, rank=rank)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_rank", lambda: rank
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
)
|
||||
cls = Qwen4ExpPLEPinnedHostEmbedding if pinned else Qwen4ExpPLEDeviceEmbedding
|
||||
with torch.device(device):
|
||||
embedding = cls(
|
||||
8,
|
||||
160,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=2,
|
||||
prefix="test.ple_embedding.ngram_embedding",
|
||||
embedding_method=Qwen4ExpPLENvFp4EmbeddingMethod(),
|
||||
num_ngram_heads=2,
|
||||
max_total_tokens=4,
|
||||
)
|
||||
module = _make_ngram_embedding_for_load_test()
|
||||
module.ngram_embedding = embedding
|
||||
module.split_ngram_parts = 3
|
||||
codes = torch.arange(8 * 80).reshape(8, 80).to(torch.uint8)
|
||||
scales = ((torch.arange(80).reshape(8, 10) + 1) / 4).to(torch.float8_e4m3fn)
|
||||
tensors = []
|
||||
# Checkpoint shards cross ETP boundaries; scales can arrive before weights.
|
||||
for shard in (2, 0, 1):
|
||||
for suffix, tensor in (("weight_scale", scales), ("weight", codes)):
|
||||
tensors.append(
|
||||
(f"ngram_embedding.shard_{shard}.{suffix}", tensor[shard * 3 :][:3])
|
||||
)
|
||||
tensors.append(("ngram_embedding.weight_scale_2", torch.tensor(0.37)))
|
||||
return module, tensors, codes, scales
|
||||
|
||||
|
||||
def _reference_nvfp4_ple(codes, scales):
|
||||
nibbles = torch.stack((codes & 15, codes >> 4), dim=-1).reshape(8, 160)
|
||||
values = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
|
||||
decoded = values[(nibbles & 7).long()] * torch.where(nibbles < 8, 1.0, -1.0)
|
||||
return (decoded * scales.float().repeat_interleave(16, -1) * 0.37).bfloat16()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rank", [0, 1])
|
||||
def test_nvfp4_ple_loads_streamed_shards_across_etp_boundaries(monkeypatch, rank):
|
||||
module, tensors, codes, scales = _make_nvfp4_ngram_embedding(monkeypatch, rank=rank)
|
||||
loaded = set()
|
||||
for start in range(0, len(tensors), 3):
|
||||
loaded.update(module.load_weights(iter(tensors[start : start + 3])))
|
||||
layer = module.ngram_embedding
|
||||
layer.embedding_method.process_weights_after_loading(layer)
|
||||
|
||||
assert loaded == {
|
||||
"ngram_embedding.weight",
|
||||
"ngram_embedding.weight_scale",
|
||||
"ngram_embedding.weight_scale_2",
|
||||
}
|
||||
assert layer.weight.dtype == torch.uint8
|
||||
assert layer.weight_scale.dtype == torch.float8_e4m3fn
|
||||
assert layer.weight.numel() + layer.weight_scale.numel() == 4 * (80 + 10)
|
||||
torch.testing.assert_close(layer.weight, codes[rank * 4 :][:4])
|
||||
torch.testing.assert_close(
|
||||
layer.weight_scale.float(), scales[rank * 4 :][:4].float()
|
||||
)
|
||||
ids = torch.tensor([[3, 4], [7, 0], [4, 3]])
|
||||
output = layer(ids)
|
||||
expected = _reference_nvfp4_ple(codes, scales)[ids]
|
||||
expected[(ids < rank * 4) | (ids >= (rank + 1) * 4)] = 0
|
||||
torch.testing.assert_close(output, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_nvfp4_ple_rejects_missing_global_scale(monkeypatch):
|
||||
module, tensors, _, _ = _make_nvfp4_ngram_embedding(monkeypatch)
|
||||
module.load_weights(tensors[:-1])
|
||||
layer = module.ngram_embedding
|
||||
with pytest.raises(ValueError, match="missing its global scale"):
|
||||
layer.embedding_method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
def test_nvfp4_ple_resolves_from_mixed_precision_quantized_layers():
|
||||
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
config = ModelOptMixedPrecisionConfig.from_config(
|
||||
{
|
||||
"quantization": {
|
||||
"quant_algo": "MIXED_PRECISION",
|
||||
"kv_cache_quant_algo": None,
|
||||
"quantized_layers": {
|
||||
prefix: {"quant_algo": "NVFP4", "group_size": 16},
|
||||
"model.language_model.layers.0.mlp.experts": {
|
||||
"quant_algo": "NVFP4",
|
||||
"group_size": 16,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(config, prefix),
|
||||
Qwen4ExpPLENvFp4EmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required")
|
||||
@pytest.mark.parametrize("pinned", [False, True])
|
||||
def test_nvfp4_ple_cuda_lookup_and_graph_replay(monkeypatch, pinned):
|
||||
# Rank 1 exercises both the vocab offset and out-of-shard masking.
|
||||
rank = 1
|
||||
module, tensors, codes, scales = _make_nvfp4_ngram_embedding(
|
||||
monkeypatch, rank=rank, device="cuda", pinned=pinned
|
||||
)
|
||||
module.load_weights(iter(tensors))
|
||||
layer = module.ngram_embedding
|
||||
layer.embedding_method.process_weights_after_loading(layer)
|
||||
ids = torch.tensor([[3, 4], [7, 0]], device="cuda")
|
||||
lookup_fn = layer._lookup if pinned else torch.compile(layer, fullgraph=True)
|
||||
|
||||
def lookup():
|
||||
return lookup_fn(ids)
|
||||
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
for _ in range(3):
|
||||
lookup()
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = lookup()
|
||||
for new_ids in ([[3, 4], [7, 0]], [[4, 3], [0, 7]]):
|
||||
ids.copy_(torch.tensor(new_ids, device="cuda"))
|
||||
graph.replay()
|
||||
cpu_ids = ids.cpu()
|
||||
expected = _reference_nvfp4_ple(codes, scales)[cpu_ids]
|
||||
expected[(cpu_ids < rank * 4) | (cpu_ids >= (rank + 1) * 4)] = 0
|
||||
torch.testing.assert_close(output.cpu(), expected, atol=0, rtol=0)
|
||||
if pinned:
|
||||
layer.start_prefetch(ids)
|
||||
torch.testing.assert_close(
|
||||
layer.finalize_prefetch(total_tokens=ids.shape[0]).cpu(),
|
||||
expected.flatten(-2),
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
assert layer.weight.device.type == ("cpu" if pinned else "cuda")
|
||||
assert layer.weight_scale.device == layer.weight.device
|
||||
assert layer.weight_scale_2.device.type == "cuda"
|
||||
assert lookup_fn(ids[:0]).shape == (0, 2, 160)
|
||||
padded_ids = torch.tensor([[8, -1]], device="cuda")
|
||||
torch.testing.assert_close(
|
||||
lookup_fn(padded_ids),
|
||||
torch.zeros(1, 2, 160, device="cuda", dtype=torch.bfloat16),
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("dp_rank", "local_tokens"), [(0, 2), (1, 3)])
|
||||
def test_etp_lookup_gathers_and_returns_dp_local_rows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -865,7 +1023,7 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
"quantized_layers": {
|
||||
prefix: {"quant_algo": "FP8"},
|
||||
"model.language_model.layers.2.moe.gate_proj": {
|
||||
"quant_algo": "NVFP4"
|
||||
"quant_algo": "MXFP8"
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -876,13 +1034,12 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
# A quant_algo the PLE table cannot load must not fall back to unquantized.
|
||||
with pytest.raises(ValueError, match="Unsupported quant_algo 'MXFP8'"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
"model.language_model.layers.2.moe.gate_proj",
|
||||
),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
|
||||
|
||||
Reference in New Issue
Block a user