[Qwen4Exp] Serve PLE rows from the loader's mapped checkpoint shards (PR #58815)

Adds an opt-in backend, --engram-config '{"host_file_gather": true}', that
serves Qwen4Exp PLE rows from the checkpoint files through the page cache
instead of allocating the 47.7 GiB table on GPU or in pinned host memory.

- Qwen4ExpPLEFileGatherEmbedding allocates no table rows; load_weights keeps
  the default loader's zero-copy safetensors views of each shard. At bind
  (post-load hook) every view must be file-backed per /proc/self/maps (HF
  cache symlinks work; heap/pinned/anonymous memory is rejected) and the
  shards must cover the whole table.
- Each step, Qwen4ExpModelState.prepare_inputs computes the PLE n-gram ids,
  copies them to pinned host memory, syncs the stream once, issues
  POSIX_FADV_WILLNEED over the coalesced page ranges of the step's rows,
  gathers the rows on the host, and copies them into a fixed device staging
  buffer. The forward reads only the staging buffer, so FULL CUDA graph
  capture is unaffected.
- Rows stay in the checkpoint dtype (FP8 table stays FP8, dequantized on the
  GPU); dummy loads serve zeros; weight reload is rejected; takes precedence
  over cpu_offload; rejected on ROCm and with elastic EP.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-09-27 09:54:42 +07:00
co-authored by Hinata
parent 5af48b498c
commit 43fbd6510b
8 changed files with 684 additions and 15 deletions
+38 -1
View File
@@ -1478,7 +1478,9 @@ def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: in
assert config.get_parallel_size(parallel) == expected
@pytest.mark.parametrize("option", ["embedding_across_dp", "dp_shared_memory"])
@pytest.mark.parametrize(
"option", ["embedding_across_dp", "dp_shared_memory", "host_file_gather"]
)
def test_engram_rejects_elastic_cross_dp(option):
parallel = ParallelConfig(
tensor_parallel_size=4,
@@ -1681,12 +1683,47 @@ def test_engram_draft_config_validates_target(monkeypatch, target_has_ple, expli
assert config.engram_config.cpu_offload is True
@pytest.mark.parametrize(
("architecture", "layer_field", "rocm", "expect_error"),
[
pytest.param(
"DeepseekV41ForCausalLM", "engram_layer_ids", False, True, id="deepseek"
),
pytest.param(
"Qwen4ExpForCausalLM", "ple_layer_ids", True, True, id="qwen4exp_rocm"
),
pytest.param(
"Qwen4ExpForCausalLM", "ple_layer_ids", False, False, id="qwen4exp_cuda"
),
],
)
def test_engram_host_file_gather_requires_cuda_qwen4exp(
monkeypatch, architecture, layer_field, rocm, expect_error
):
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True)
monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm)
model = cast(
ModelConfig,
SimpleNamespace(
architecture=architecture,
hf_text_config=SimpleNamespace(**{layer_field: [1]}),
),
)
config = EngramConfig(host_file_gather=True)
if expect_error:
with pytest.raises(ValueError, match="host_file_gather"):
config.verify_model_config(model)
else:
config.verify_model_config(model)
def test_engram_hash_tracks_execution_options():
configs = [
EngramConfig(cpu_offload=True),
EngramConfig(cpu_offload=False),
EngramConfig(cpu_offload=True, embedding_across_dp=True),
EngramConfig(cpu_offload=True, dp_shared_memory=True),
EngramConfig(host_file_gather=True),
]
assert len({config.compute_hash() for config in configs}) == len(configs)