[Qwen4Exp] Serve PLE rows from the loader's mapped checkpoint shards (PR #58815)
Adds an opt-in backend, --engram-config '{"host_file_gather": true}', that
serves Qwen4Exp PLE rows from the checkpoint files through the page cache
instead of allocating the 47.7 GiB table on GPU or in pinned host memory.
- Qwen4ExpPLEFileGatherEmbedding allocates no table rows; load_weights keeps
the default loader's zero-copy safetensors views of each shard. At bind
(post-load hook) every view must be file-backed per /proc/self/maps (HF
cache symlinks work; heap/pinned/anonymous memory is rejected) and the
shards must cover the whole table.
- Each step, Qwen4ExpModelState.prepare_inputs computes the PLE n-gram ids,
copies them to pinned host memory, syncs the stream once, issues
POSIX_FADV_WILLNEED over the coalesced page ranges of the step's rows,
gathers the rows on the host, and copies them into a fixed device staging
buffer. The forward reads only the staging buffer, so FULL CUDA graph
capture is unaffected.
- Rows stay in the checkpoint dtype (FP8 table stays FP8, dequantized on the
GPU); dummy loads serve zeros; weight reload is rejected; takes precedence
over cpu_offload; rejected on ROCm and with elastic EP.
Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -21,6 +21,7 @@ from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
from ...utils import spawn_new_process_for_each_test
|
||||
from .test_ple import _make_file_gather_embedding
|
||||
|
||||
|
||||
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
||||
@@ -175,6 +176,7 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
||||
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
|
||||
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
|
||||
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
|
||||
model_state.file_gather_modules = []
|
||||
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
@@ -224,6 +226,39 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
||||
assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr()
|
||||
|
||||
|
||||
def test_qwen4_exp_model_state_stages_file_gather_rows(monkeypatch, tmp_path) -> None:
|
||||
"""Rows are staged from the same PLE inputs the forward receives."""
|
||||
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
|
||||
model_state = object.__new__(Qwen4ExpModelState)
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_eos_token_id = 0
|
||||
model_state.ngram_context = torch.empty((4, 1), dtype=torch.int32)
|
||||
model_state.ngram_context_offsets = torch.arange(-1, 0, dtype=torch.int64)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
model_state.file_gather_modules = [module]
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
num_reqs_after_padding=2,
|
||||
idx_mapping=torch.tensor([0, 1]),
|
||||
query_start_loc=torch.tensor([0, 4, 6], dtype=torch.int32),
|
||||
input_ids=torch.tensor([5, 9, 3, 7, 2, 11]),
|
||||
)
|
||||
req_states = SimpleNamespace(
|
||||
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([2, 1])),
|
||||
all_token_ids=SimpleNamespace(gpu=torch.tensor([[1, 8, 6], [4, 3, 3]])),
|
||||
)
|
||||
|
||||
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
||||
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
||||
|
||||
ids = module.compute_ngram_ids(
|
||||
input_batch.input_ids,
|
||||
model_inputs["query_start_loc"],
|
||||
model_inputs["ngram_context"],
|
||||
)
|
||||
assert torch.equal(module.ngram_embedding._staging[:6], table[ids])
|
||||
|
||||
|
||||
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
||||
model_state = object.__new__(Qwen4ExpModelState)
|
||||
model_state.uses_ngram_embedding = True
|
||||
|
||||
@@ -1,19 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import mmap
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import load, load_file, save_file
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
import vllm.models.qwen4_exp.common.ngram_embedding as ngram_embedding_module
|
||||
import vllm.models.qwen4_exp.nvidia.ngram_embedding as nvidia_ngram_embedding_module
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
@@ -35,6 +41,7 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
Qwen4ExpPLEEmbeddingMethod,
|
||||
Qwen4ExpPLEFileGatherEmbedding,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLEPinnedHostEmbedding,
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
@@ -634,6 +641,350 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
)
|
||||
|
||||
|
||||
def _patch_ple_construction(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
engram_config: SimpleNamespace | None = None,
|
||||
load_format: str = "auto",
|
||||
) -> None:
|
||||
_mock_etp_group(monkeypatch)
|
||||
for module in (embedding_module, parameter_module):
|
||||
monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
vllm_config = SimpleNamespace(
|
||||
load_config=SimpleNamespace(load_format=load_format),
|
||||
engram_config=engram_config,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
nvidia_ngram_embedding_module, "get_current_vllm_config", lambda: vllm_config
|
||||
)
|
||||
|
||||
|
||||
def _make_file_gather_embedding(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
*,
|
||||
fp8: bool = False,
|
||||
load_format: str = "auto",
|
||||
bind: bool = False,
|
||||
) -> tuple[Qwen4ExpNGramEmbedding, torch.Tensor]:
|
||||
"""Build a two-head, 12-token host-file-gather embedding and its saved table.
|
||||
|
||||
The 21-row table is saved to ``tmp_path / "model.safetensors"`` as four
|
||||
checkpoint shards of stride 6, so the last shard holds 3 rows.
|
||||
"""
|
||||
_patch_ple_construction(monkeypatch, load_format=load_format)
|
||||
method = (
|
||||
Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
if fp8
|
||||
else Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
)
|
||||
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
|
||||
nn.Module.__init__(module)
|
||||
module.split_ngram_parts = 4
|
||||
module.eos_token_id = 0
|
||||
module.ngram_size = 2
|
||||
module.heads_per_ngram = 2
|
||||
module.register_buffer("layer_multipliers", torch.tensor([3, 5]))
|
||||
module.register_buffer("ngram_heads_vocab_sizes", torch.tensor([13, 7]))
|
||||
module.register_buffer("ngram_heads_offsets", torch.tensor([0, 13]))
|
||||
module.ngram_embedding = Qwen4ExpPLEFileGatherEmbedding(
|
||||
21,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
prefix="test.ple_embedding",
|
||||
embedding_method=method,
|
||||
num_ngram_heads=2,
|
||||
max_total_tokens=12,
|
||||
)
|
||||
dtype = torch.float8_e4m3fn if fp8 else torch.bfloat16
|
||||
table = torch.arange(42, dtype=torch.float32).reshape(21, 2).to(dtype)
|
||||
checkpoint = {
|
||||
f"ngram_embedding.shard_{index}.weight": shard.clone()
|
||||
for index, shard in enumerate(table.split(6))
|
||||
}
|
||||
if fp8:
|
||||
checkpoint["ngram_embedding.weight_scale"] = torch.tensor([0.25]).bfloat16()
|
||||
save_file(checkpoint, tmp_path / "model.safetensors")
|
||||
if bind:
|
||||
_load_file_shards(module, tmp_path / "model.safetensors")
|
||||
module.ngram_embedding.bind_file_shards()
|
||||
return module, table
|
||||
|
||||
|
||||
def _load_file_shards(module: Qwen4ExpNGramEmbedding, path: Path) -> set[str]:
|
||||
"""Load ``path`` through ``load_weights`` as the lazy safetensors iterator does."""
|
||||
with safe_open(path, framework="pt") as checkpoint:
|
||||
return module.load_weights(
|
||||
(name, checkpoint.get_tensor(name))
|
||||
for name in checkpoint.keys() # noqa: SIM118
|
||||
)
|
||||
|
||||
|
||||
def _dev_zero_shards() -> list[tuple[str, torch.Tensor]]:
|
||||
"""Return shards in a ``/dev/zero`` mapping, which is how pinned memory maps."""
|
||||
with open("/dev/zero", "r+b") as dev_zero:
|
||||
mapping = mmap.mmap(dev_zero.fileno(), mmap.PAGESIZE)
|
||||
table = torch.frombuffer(mapping, dtype=torch.uint8)[:84].view(torch.bfloat16)
|
||||
return [
|
||||
(f"ngram_embedding.shard_{index}.weight", shard)
|
||||
for index, shard in enumerate(table.reshape(21, 2).split(6))
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("engram_config", "backend", "rows"),
|
||||
[
|
||||
(
|
||||
SimpleNamespace(host_file_gather=True, cpu_offload=True),
|
||||
Qwen4ExpPLEFileGatherEmbedding,
|
||||
0,
|
||||
),
|
||||
(
|
||||
SimpleNamespace(host_file_gather=False, cpu_offload=False),
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
24,
|
||||
),
|
||||
(None, Qwen4ExpPLEDeviceEmbedding, 24),
|
||||
],
|
||||
)
|
||||
def test_ngram_embedding_selects_backend_from_engram_config(
|
||||
monkeypatch, engram_config, backend, rows
|
||||
) -> None:
|
||||
"""File gather wins with no rows; ``cpu_offload=False`` keeps rows on device."""
|
||||
_patch_ple_construction(monkeypatch, engram_config=engram_config)
|
||||
config = SimpleNamespace(
|
||||
ngram_size=2,
|
||||
heads_per_ngram=2,
|
||||
eos_token_id=0,
|
||||
vocab_size=64,
|
||||
ngram_vocab_size_base=8,
|
||||
make_ngram_vocab_size_divisible_by=1,
|
||||
)
|
||||
|
||||
module = Qwen4ExpNGramEmbedding(config, 4, 0, 4, data_parallel_rank=0, prefix="t")
|
||||
|
||||
assert tuple(module.ngram_embedding.weight.shape) == (rows, 2)
|
||||
assert isinstance(module.ngram_embedding, backend)
|
||||
|
||||
|
||||
def test_ngram_embedding_default_cpu_offload_selects_pinned_backend(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Default ``cpu_offload=True`` selects pinned storage and requires UVA."""
|
||||
_patch_ple_construction(
|
||||
monkeypatch,
|
||||
engram_config=SimpleNamespace(host_file_gather=False, cpu_offload=True),
|
||||
)
|
||||
monkeypatch.setattr(ngram_embedding_module, "is_uva_available", lambda: False)
|
||||
config = SimpleNamespace(
|
||||
ngram_size=2,
|
||||
heads_per_ngram=2,
|
||||
eos_token_id=0,
|
||||
vocab_size=64,
|
||||
ngram_vocab_size_base=8,
|
||||
make_ngram_vocab_size_divisible_by=1,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="CPU offload requires UVA"):
|
||||
Qwen4ExpNGramEmbedding(config, 4, 0, 4, data_parallel_rank=0, prefix="t")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("read_shards", "error"),
|
||||
[
|
||||
pytest.param(
|
||||
lambda path: [("ngram_embedding.shard_0.weight", torch.zeros(6, 2))],
|
||||
"dtype torch.float32 must match the embedding dtype torch.bfloat16",
|
||||
id="wrong_dtype",
|
||||
),
|
||||
pytest.param(
|
||||
lambda path: [*load_file(path).items()] * 2,
|
||||
"Duplicate PLE embedding shard 0",
|
||||
id="duplicate_shard",
|
||||
),
|
||||
pytest.param(
|
||||
lambda path: [s for s in load_file(path).items() if "_2." not in s[0]],
|
||||
"PLE shards cover 15 of 21 rows",
|
||||
id="missing_shard",
|
||||
),
|
||||
pytest.param(lambda path: [], "PLE shards cover 0 of 21 rows", id="no_shards"),
|
||||
pytest.param(
|
||||
lambda path: [(n, t.clone()) for n, t in load_file(path).items()],
|
||||
"not a file-backed view",
|
||||
id="anonymous",
|
||||
),
|
||||
pytest.param(
|
||||
lambda path: load(path.read_bytes()).items(),
|
||||
"not a file-backed view",
|
||||
id="heap",
|
||||
),
|
||||
pytest.param(
|
||||
lambda path: _dev_zero_shards(), "not a file-backed view", id="dev_zero"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_file_gather_rejects_shards_that_cannot_serve_every_row_from_a_file(
|
||||
monkeypatch, tmp_path, read_shards, error
|
||||
) -> None:
|
||||
"""A real load must fail instead of serving zeros or a copied table."""
|
||||
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match=error):
|
||||
module.load_weights(read_shards(tmp_path / "model.safetensors"))
|
||||
module.ngram_embedding.bind_file_shards()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fp8", [False, True])
|
||||
def test_file_gather_binds_zero_copy_shards_through_loader_hook(
|
||||
monkeypatch, tmp_path, fp8
|
||||
) -> None:
|
||||
"""Fails if safetensors stops returning zero-copy views of the file."""
|
||||
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, fp8=fp8)
|
||||
embedding = module.ngram_embedding
|
||||
|
||||
loaded = _load_file_shards(module, tmp_path / "model.safetensors")
|
||||
embedding.quant_method.process_weights_after_loading(embedding)
|
||||
embedding.quant_method.process_weights_after_loading(embedding)
|
||||
|
||||
assert loaded == {"ngram_embedding.weight"} | (
|
||||
{"ngram_embedding.weight_scale"} if fp8 else set()
|
||||
)
|
||||
assert embedding._bound
|
||||
stored = torch.cat([embedding._shards[index] for index in range(4)])
|
||||
assert torch.equal(stored, table.view(torch.uint8))
|
||||
|
||||
|
||||
def test_file_gather_binds_shards_opened_through_a_symlink(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
"""HF cache snapshots link to suffix-less blobs, which is the mapped path."""
|
||||
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path)
|
||||
blob = (tmp_path / "model.safetensors").rename(tmp_path / "blob")
|
||||
snapshot = tmp_path / "snapshot.safetensors"
|
||||
snapshot.symlink_to(blob)
|
||||
_load_file_shards(module, snapshot)
|
||||
|
||||
module.ngram_embedding.bind_file_shards()
|
||||
|
||||
assert module.ngram_embedding._bound
|
||||
|
||||
|
||||
def test_file_gather_rejects_reload_and_stays_poisoned(monkeypatch, tmp_path) -> None:
|
||||
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
|
||||
|
||||
with pytest.raises(RuntimeError, match="does not support weight reload"):
|
||||
module.load_weights([("ngram_embedding.shard_0.weight", table[:6])])
|
||||
with pytest.raises(RuntimeError, match="unbound or was reloaded"):
|
||||
module.ngram_embedding.stage_rows(1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("bad_id", "error"),
|
||||
[(21, IndexError), (24, IndexError), (-1, RuntimeError)],
|
||||
ids=["past_last_shard_rows", "past_last_shard", "negative"],
|
||||
)
|
||||
def test_file_gather_out_of_range_id_keeps_previous_staged_rows(
|
||||
monkeypatch, tmp_path, bad_id, error
|
||||
) -> None:
|
||||
"""Out-of-range ids fail before the H2D copy, so no wrong row is served."""
|
||||
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
|
||||
embedding = module.ngram_embedding
|
||||
ids = torch.tensor(
|
||||
[
|
||||
[20, 0],
|
||||
[19, 1],
|
||||
[18, 2],
|
||||
[17, 3],
|
||||
[16, 4],
|
||||
[15, 5],
|
||||
[14, 6],
|
||||
[13, 7],
|
||||
[12, 8],
|
||||
[11, 9],
|
||||
[10, 10],
|
||||
]
|
||||
)
|
||||
embedding._host_ids[:11] = ids
|
||||
embedding.stage_rows(11)
|
||||
embedding._host_ids[5, 1] = bad_id
|
||||
|
||||
with pytest.raises(error):
|
||||
embedding.stage_rows(11)
|
||||
|
||||
assert torch.equal(embedding._staging[:11], table[ids])
|
||||
|
||||
|
||||
def test_file_gather_reads_ahead_the_coalesced_pages_of_staged_rows(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
"""Rows start 34 bytes before page 2 of a file mapped from page 1.
|
||||
|
||||
Rows 0-7 are in page 1, row 8 straddles pages 1 and 2, and the rows of shard 1
|
||||
arrive unsorted.
|
||||
"""
|
||||
module, table = _make_file_gather_embedding(monkeypatch, tmp_path)
|
||||
embedding, page = module.ngram_embedding, mmap.PAGESIZE
|
||||
start = 2 * page - 34
|
||||
path = tmp_path / "table.bin"
|
||||
path.write_bytes(bytes(start) + table.view(torch.uint8).numpy().tobytes())
|
||||
with open(path, "rb") as file:
|
||||
mapping = mmap.mmap(file.fileno(), 0, offset=page, access=mmap.ACCESS_COPY)
|
||||
rows = torch.frombuffer(mapping, dtype=torch.uint8)[start - page :]
|
||||
module.load_weights(
|
||||
(f"ngram_embedding.shard_{index}.weight", shard)
|
||||
for index, shard in enumerate(rows.view(table.dtype).reshape(21, 2).split(6))
|
||||
)
|
||||
embedding.bind_file_shards()
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
nvidia_ngram_embedding_module.os,
|
||||
"posix_fadvise",
|
||||
lambda fd, *args: calls.append((os.fstat(fd).st_ino, *args)),
|
||||
)
|
||||
ids = torch.tensor([[0, 1], [8, 7]])
|
||||
embedding._host_ids[:2] = ids
|
||||
|
||||
embedding.stage_rows(2)
|
||||
|
||||
inode, willneed = path.stat().st_ino, os.POSIX_FADV_WILLNEED
|
||||
assert calls == [(inode, page, page, willneed), (inode, page, 2 * page, willneed)]
|
||||
assert torch.equal(embedding._staging[:2], table[ids])
|
||||
|
||||
|
||||
def test_file_gather_dummy_load_stages_zeros(monkeypatch, tmp_path) -> None:
|
||||
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path, load_format="dummy")
|
||||
embedding = module.ngram_embedding
|
||||
embedding.quant_method.process_weights_after_loading(embedding)
|
||||
embedding._host_rows.view(torch.uint8).fill_(0xFF)
|
||||
embedding._staging.view(torch.uint8).fill_(0xFF)
|
||||
|
||||
embedding.stage_rows(4)
|
||||
|
||||
staged = embedding._staging.view(torch.uint8)
|
||||
assert torch.all(staged[:4] == 0)
|
||||
assert torch.all(staged[4:] == 0xFF)
|
||||
|
||||
|
||||
def test_file_gather_forward_reads_staging_without_host_work(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
"""The forward must be a fixed-address read so FULL CUDA graphs replay it."""
|
||||
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path)
|
||||
staging = module.ngram_embedding._staging
|
||||
staging.view(torch.uint8).fill_(0xFF)
|
||||
step = (torch.arange(6), torch.tensor([0, 6]), torch.tensor([[1]]))
|
||||
hidden_states = torch.zeros(6, 8, dtype=torch.bfloat16)
|
||||
|
||||
assert module.start_prefetch(hidden_states, *step) is None
|
||||
output = module(hidden_states, *step)
|
||||
|
||||
assert torch.all(staging.view(torch.uint8) == 0xFF)
|
||||
assert output.data_ptr() == staging.data_ptr()
|
||||
assert output.shape == (6, 4)
|
||||
|
||||
|
||||
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
|
||||
conv_state_len = 6
|
||||
dilation = 2
|
||||
|
||||
Reference in New Issue
Block a user