[Qwen4Exp] Serve PLE rows from the loader's mapped checkpoint shards (PR #58815)

Adds an opt-in backend, --engram-config '{"host_file_gather": true}', that
serves Qwen4Exp PLE rows from the checkpoint files through the page cache
instead of allocating the 47.7 GiB table on GPU or in pinned host memory.

- Qwen4ExpPLEFileGatherEmbedding allocates no table rows; load_weights keeps
  the default loader's zero-copy safetensors views of each shard. At bind
  (post-load hook) every view must be file-backed per /proc/self/maps (HF
  cache symlinks work; heap/pinned/anonymous memory is rejected) and the
  shards must cover the whole table.
- Each step, Qwen4ExpModelState.prepare_inputs computes the PLE n-gram ids,
  copies them to pinned host memory, syncs the stream once, issues
  POSIX_FADV_WILLNEED over the coalesced page ranges of the step's rows,
  gathers the rows on the host, and copies them into a fixed device staging
  buffer. The forward reads only the staging buffer, so FULL CUDA graph
  capture is unaffected.
- Rows stay in the checkpoint dtype (FP8 table stays FP8, dequantized on the
  GPU); dummy loads serve zeros; weight reload is rejected; takes precedence
  over cpu_offload; rejected on ROCm and with elastic EP.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-09-27 09:54:42 +07:00
co-authored by Hinata
parent 5af48b498c
commit 43fbd6510b
8 changed files with 684 additions and 15 deletions
+35
View File
@@ -21,6 +21,7 @@ from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from ...utils import spawn_new_process_for_each_test
from .test_ple import _make_file_gather_embedding
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
@@ -175,6 +176,7 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
model_state.file_gather_modules = []
input_batch = SimpleNamespace(
num_reqs=2,
@@ -224,6 +226,39 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr()
def test_qwen4_exp_model_state_stages_file_gather_rows(monkeypatch, tmp_path) -> None:
"""Rows are staged from the same PLE inputs the forward receives."""
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_eos_token_id = 0
model_state.ngram_context = torch.empty((4, 1), dtype=torch.int32)
model_state.ngram_context_offsets = torch.arange(-1, 0, dtype=torch.int64)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
model_state.file_gather_modules = [module]
input_batch = SimpleNamespace(
num_reqs=2,
num_reqs_after_padding=2,
idx_mapping=torch.tensor([0, 1]),
query_start_loc=torch.tensor([0, 4, 6], dtype=torch.int32),
input_ids=torch.tensor([5, 9, 3, 7, 2, 11]),
)
req_states = SimpleNamespace(
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([2, 1])),
all_token_ids=SimpleNamespace(gpu=torch.tensor([[1, 8, 6], [4, 3, 3]])),
)
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
ids = module.compute_ngram_ids(
input_batch.input_ids,
model_inputs["query_start_loc"],
model_inputs["ngram_context"],
)
assert torch.equal(module.ngram_embedding._staging[:6], table[ids])
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
+351
View File
@@ -1,19 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import mmap
import os
from dataclasses import dataclass
from functools import partial
from itertools import accumulate
from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
from safetensors import safe_open
from safetensors.torch import load, load_file, save_file
from torch import nn
from torch.nn import functional as F
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
import vllm.models.qwen4_exp.common.ngram_embedding as ngram_embedding_module
import vllm.models.qwen4_exp.nvidia.ngram_embedding as nvidia_ngram_embedding_module
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.modelopt import (
@@ -35,6 +41,7 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpNGramEmbedding,
Qwen4ExpPLEDeviceEmbedding,
Qwen4ExpPLEEmbeddingMethod,
Qwen4ExpPLEFileGatherEmbedding,
Qwen4ExpPLEFp8EmbeddingMethod,
Qwen4ExpPLEPinnedHostEmbedding,
Qwen4ExpPLEUnquantizedEmbeddingMethod,
@@ -634,6 +641,350 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
)
def _patch_ple_construction(
monkeypatch: pytest.MonkeyPatch,
*,
engram_config: SimpleNamespace | None = None,
load_format: str = "auto",
) -> None:
_mock_etp_group(monkeypatch)
for module in (embedding_module, parameter_module):
monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 1)
vllm_config = SimpleNamespace(
load_config=SimpleNamespace(load_format=load_format),
engram_config=engram_config,
)
monkeypatch.setattr(
nvidia_ngram_embedding_module, "get_current_vllm_config", lambda: vllm_config
)
def _make_file_gather_embedding(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
*,
fp8: bool = False,
load_format: str = "auto",
bind: bool = False,
) -> tuple[Qwen4ExpNGramEmbedding, torch.Tensor]:
"""Build a two-head, 12-token host-file-gather embedding and its saved table.
The 21-row table is saved to ``tmp_path / "model.safetensors"`` as four
checkpoint shards of stride 6, so the last shard holds 3 rows.
"""
_patch_ple_construction(monkeypatch, load_format=load_format)
method = (
Qwen4ExpPLEFp8EmbeddingMethod()
if fp8
else Qwen4ExpPLEUnquantizedEmbeddingMethod()
)
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
nn.Module.__init__(module)
module.split_ngram_parts = 4
module.eos_token_id = 0
module.ngram_size = 2
module.heads_per_ngram = 2
module.register_buffer("layer_multipliers", torch.tensor([3, 5]))
module.register_buffer("ngram_heads_vocab_sizes", torch.tensor([13, 7]))
module.register_buffer("ngram_heads_offsets", torch.tensor([0, 13]))
module.ngram_embedding = Qwen4ExpPLEFileGatherEmbedding(
21,
2,
params_dtype=torch.bfloat16,
padding_size=1,
prefix="test.ple_embedding",
embedding_method=method,
num_ngram_heads=2,
max_total_tokens=12,
)
dtype = torch.float8_e4m3fn if fp8 else torch.bfloat16
table = torch.arange(42, dtype=torch.float32).reshape(21, 2).to(dtype)
checkpoint = {
f"ngram_embedding.shard_{index}.weight": shard.clone()
for index, shard in enumerate(table.split(6))
}
if fp8:
checkpoint["ngram_embedding.weight_scale"] = torch.tensor([0.25]).bfloat16()
save_file(checkpoint, tmp_path / "model.safetensors")
if bind:
_load_file_shards(module, tmp_path / "model.safetensors")
module.ngram_embedding.bind_file_shards()
return module, table
def _load_file_shards(module: Qwen4ExpNGramEmbedding, path: Path) -> set[str]:
"""Load ``path`` through ``load_weights`` as the lazy safetensors iterator does."""
with safe_open(path, framework="pt") as checkpoint:
return module.load_weights(
(name, checkpoint.get_tensor(name))
for name in checkpoint.keys() # noqa: SIM118
)
def _dev_zero_shards() -> list[tuple[str, torch.Tensor]]:
"""Return shards in a ``/dev/zero`` mapping, which is how pinned memory maps."""
with open("/dev/zero", "r+b") as dev_zero:
mapping = mmap.mmap(dev_zero.fileno(), mmap.PAGESIZE)
table = torch.frombuffer(mapping, dtype=torch.uint8)[:84].view(torch.bfloat16)
return [
(f"ngram_embedding.shard_{index}.weight", shard)
for index, shard in enumerate(table.reshape(21, 2).split(6))
]
@pytest.mark.parametrize(
("engram_config", "backend", "rows"),
[
(
SimpleNamespace(host_file_gather=True, cpu_offload=True),
Qwen4ExpPLEFileGatherEmbedding,
0,
),
(
SimpleNamespace(host_file_gather=False, cpu_offload=False),
Qwen4ExpPLEDeviceEmbedding,
24,
),
(None, Qwen4ExpPLEDeviceEmbedding, 24),
],
)
def test_ngram_embedding_selects_backend_from_engram_config(
monkeypatch, engram_config, backend, rows
) -> None:
"""File gather wins with no rows; ``cpu_offload=False`` keeps rows on device."""
_patch_ple_construction(monkeypatch, engram_config=engram_config)
config = SimpleNamespace(
ngram_size=2,
heads_per_ngram=2,
eos_token_id=0,
vocab_size=64,
ngram_vocab_size_base=8,
make_ngram_vocab_size_divisible_by=1,
)
module = Qwen4ExpNGramEmbedding(config, 4, 0, 4, data_parallel_rank=0, prefix="t")
assert tuple(module.ngram_embedding.weight.shape) == (rows, 2)
assert isinstance(module.ngram_embedding, backend)
def test_ngram_embedding_default_cpu_offload_selects_pinned_backend(
monkeypatch,
) -> None:
"""Default ``cpu_offload=True`` selects pinned storage and requires UVA."""
_patch_ple_construction(
monkeypatch,
engram_config=SimpleNamespace(host_file_gather=False, cpu_offload=True),
)
monkeypatch.setattr(ngram_embedding_module, "is_uva_available", lambda: False)
config = SimpleNamespace(
ngram_size=2,
heads_per_ngram=2,
eos_token_id=0,
vocab_size=64,
ngram_vocab_size_base=8,
make_ngram_vocab_size_divisible_by=1,
)
with pytest.raises(RuntimeError, match="CPU offload requires UVA"):
Qwen4ExpNGramEmbedding(config, 4, 0, 4, data_parallel_rank=0, prefix="t")
@pytest.mark.parametrize(
("read_shards", "error"),
[
pytest.param(
lambda path: [("ngram_embedding.shard_0.weight", torch.zeros(6, 2))],
"dtype torch.float32 must match the embedding dtype torch.bfloat16",
id="wrong_dtype",
),
pytest.param(
lambda path: [*load_file(path).items()] * 2,
"Duplicate PLE embedding shard 0",
id="duplicate_shard",
),
pytest.param(
lambda path: [s for s in load_file(path).items() if "_2." not in s[0]],
"PLE shards cover 15 of 21 rows",
id="missing_shard",
),
pytest.param(lambda path: [], "PLE shards cover 0 of 21 rows", id="no_shards"),
pytest.param(
lambda path: [(n, t.clone()) for n, t in load_file(path).items()],
"not a file-backed view",
id="anonymous",
),
pytest.param(
lambda path: load(path.read_bytes()).items(),
"not a file-backed view",
id="heap",
),
pytest.param(
lambda path: _dev_zero_shards(), "not a file-backed view", id="dev_zero"
),
],
)
def test_file_gather_rejects_shards_that_cannot_serve_every_row_from_a_file(
monkeypatch, tmp_path, read_shards, error
) -> None:
"""A real load must fail instead of serving zeros or a copied table."""
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path)
with pytest.raises(ValueError, match=error):
module.load_weights(read_shards(tmp_path / "model.safetensors"))
module.ngram_embedding.bind_file_shards()
@pytest.mark.parametrize("fp8", [False, True])
def test_file_gather_binds_zero_copy_shards_through_loader_hook(
monkeypatch, tmp_path, fp8
) -> None:
"""Fails if safetensors stops returning zero-copy views of the file."""
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, fp8=fp8)
embedding = module.ngram_embedding
loaded = _load_file_shards(module, tmp_path / "model.safetensors")
embedding.quant_method.process_weights_after_loading(embedding)
embedding.quant_method.process_weights_after_loading(embedding)
assert loaded == {"ngram_embedding.weight"} | (
{"ngram_embedding.weight_scale"} if fp8 else set()
)
assert embedding._bound
stored = torch.cat([embedding._shards[index] for index in range(4)])
assert torch.equal(stored, table.view(torch.uint8))
def test_file_gather_binds_shards_opened_through_a_symlink(
monkeypatch, tmp_path
) -> None:
"""HF cache snapshots link to suffix-less blobs, which is the mapped path."""
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path)
blob = (tmp_path / "model.safetensors").rename(tmp_path / "blob")
snapshot = tmp_path / "snapshot.safetensors"
snapshot.symlink_to(blob)
_load_file_shards(module, snapshot)
module.ngram_embedding.bind_file_shards()
assert module.ngram_embedding._bound
def test_file_gather_rejects_reload_and_stays_poisoned(monkeypatch, tmp_path) -> None:
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
with pytest.raises(RuntimeError, match="does not support weight reload"):
module.load_weights([("ngram_embedding.shard_0.weight", table[:6])])
with pytest.raises(RuntimeError, match="unbound or was reloaded"):
module.ngram_embedding.stage_rows(1)
@pytest.mark.parametrize(
("bad_id", "error"),
[(21, IndexError), (24, IndexError), (-1, RuntimeError)],
ids=["past_last_shard_rows", "past_last_shard", "negative"],
)
def test_file_gather_out_of_range_id_keeps_previous_staged_rows(
monkeypatch, tmp_path, bad_id, error
) -> None:
"""Out-of-range ids fail before the H2D copy, so no wrong row is served."""
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
embedding = module.ngram_embedding
ids = torch.tensor(
[
[20, 0],
[19, 1],
[18, 2],
[17, 3],
[16, 4],
[15, 5],
[14, 6],
[13, 7],
[12, 8],
[11, 9],
[10, 10],
]
)
embedding._host_ids[:11] = ids
embedding.stage_rows(11)
embedding._host_ids[5, 1] = bad_id
with pytest.raises(error):
embedding.stage_rows(11)
assert torch.equal(embedding._staging[:11], table[ids])
def test_file_gather_reads_ahead_the_coalesced_pages_of_staged_rows(
monkeypatch, tmp_path
) -> None:
"""Rows start 34 bytes before page 2 of a file mapped from page 1.
Rows 0-7 are in page 1, row 8 straddles pages 1 and 2, and the rows of shard 1
arrive unsorted.
"""
module, table = _make_file_gather_embedding(monkeypatch, tmp_path)
embedding, page = module.ngram_embedding, mmap.PAGESIZE
start = 2 * page - 34
path = tmp_path / "table.bin"
path.write_bytes(bytes(start) + table.view(torch.uint8).numpy().tobytes())
with open(path, "rb") as file:
mapping = mmap.mmap(file.fileno(), 0, offset=page, access=mmap.ACCESS_COPY)
rows = torch.frombuffer(mapping, dtype=torch.uint8)[start - page :]
module.load_weights(
(f"ngram_embedding.shard_{index}.weight", shard)
for index, shard in enumerate(rows.view(table.dtype).reshape(21, 2).split(6))
)
embedding.bind_file_shards()
calls = []
monkeypatch.setattr(
nvidia_ngram_embedding_module.os,
"posix_fadvise",
lambda fd, *args: calls.append((os.fstat(fd).st_ino, *args)),
)
ids = torch.tensor([[0, 1], [8, 7]])
embedding._host_ids[:2] = ids
embedding.stage_rows(2)
inode, willneed = path.stat().st_ino, os.POSIX_FADV_WILLNEED
assert calls == [(inode, page, page, willneed), (inode, page, 2 * page, willneed)]
assert torch.equal(embedding._staging[:2], table[ids])
def test_file_gather_dummy_load_stages_zeros(monkeypatch, tmp_path) -> None:
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path, load_format="dummy")
embedding = module.ngram_embedding
embedding.quant_method.process_weights_after_loading(embedding)
embedding._host_rows.view(torch.uint8).fill_(0xFF)
embedding._staging.view(torch.uint8).fill_(0xFF)
embedding.stage_rows(4)
staged = embedding._staging.view(torch.uint8)
assert torch.all(staged[:4] == 0)
assert torch.all(staged[4:] == 0xFF)
def test_file_gather_forward_reads_staging_without_host_work(
monkeypatch, tmp_path
) -> None:
"""The forward must be a fixed-address read so FULL CUDA graphs replay it."""
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path)
staging = module.ngram_embedding._staging
staging.view(torch.uint8).fill_(0xFF)
step = (torch.arange(6), torch.tensor([0, 6]), torch.tensor([[1]]))
hidden_states = torch.zeros(6, 8, dtype=torch.bfloat16)
assert module.start_prefetch(hidden_states, *step) is None
output = module(hidden_states, *step)
assert torch.all(staging.view(torch.uint8) == 0xFF)
assert output.data_ptr() == staging.data_ptr()
assert output.shape == (6, 4)
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
conv_state_len = 6
dilation = 2