[Qwen4Exp] Serve PLE rows from the loader's mapped checkpoint shards (PR #58815)
Adds an opt-in backend, --engram-config '{"host_file_gather": true}', that
serves Qwen4Exp PLE rows from the checkpoint files through the page cache
instead of allocating the 47.7 GiB table on GPU or in pinned host memory.
- Qwen4ExpPLEFileGatherEmbedding allocates no table rows; load_weights keeps
the default loader's zero-copy safetensors views of each shard. At bind
(post-load hook) every view must be file-backed per /proc/self/maps (HF
cache symlinks work; heap/pinned/anonymous memory is rejected) and the
shards must cover the whole table.
- Each step, Qwen4ExpModelState.prepare_inputs computes the PLE n-gram ids,
copies them to pinned host memory, syncs the stream once, issues
POSIX_FADV_WILLNEED over the coalesced page ranges of the step's rows,
gathers the rows on the host, and copies them into a fixed device staging
buffer. The forward reads only the staging buffer, so FULL CUDA graph
capture is unaffected.
- Rows stay in the checkpoint dtype (FP8 table stays FP8, dequantized on the
GPU); dummy loads serve zeros; weight reload is rejected; takes precedence
over cpu_offload; rejected on ROCm and with elastic EP.
Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+38
-1
@@ -1478,7 +1478,9 @@ def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: in
|
||||
assert config.get_parallel_size(parallel) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("option", ["embedding_across_dp", "dp_shared_memory"])
|
||||
@pytest.mark.parametrize(
|
||||
"option", ["embedding_across_dp", "dp_shared_memory", "host_file_gather"]
|
||||
)
|
||||
def test_engram_rejects_elastic_cross_dp(option):
|
||||
parallel = ParallelConfig(
|
||||
tensor_parallel_size=4,
|
||||
@@ -1681,12 +1683,47 @@ def test_engram_draft_config_validates_target(monkeypatch, target_has_ple, expli
|
||||
assert config.engram_config.cpu_offload is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("architecture", "layer_field", "rocm", "expect_error"),
|
||||
[
|
||||
pytest.param(
|
||||
"DeepseekV41ForCausalLM", "engram_layer_ids", False, True, id="deepseek"
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen4ExpForCausalLM", "ple_layer_ids", True, True, id="qwen4exp_rocm"
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen4ExpForCausalLM", "ple_layer_ids", False, False, id="qwen4exp_cuda"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_engram_host_file_gather_requires_cuda_qwen4exp(
|
||||
monkeypatch, architecture, layer_field, rocm, expect_error
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm)
|
||||
model = cast(
|
||||
ModelConfig,
|
||||
SimpleNamespace(
|
||||
architecture=architecture,
|
||||
hf_text_config=SimpleNamespace(**{layer_field: [1]}),
|
||||
),
|
||||
)
|
||||
config = EngramConfig(host_file_gather=True)
|
||||
if expect_error:
|
||||
with pytest.raises(ValueError, match="host_file_gather"):
|
||||
config.verify_model_config(model)
|
||||
else:
|
||||
config.verify_model_config(model)
|
||||
|
||||
|
||||
def test_engram_hash_tracks_execution_options():
|
||||
configs = [
|
||||
EngramConfig(cpu_offload=True),
|
||||
EngramConfig(cpu_offload=False),
|
||||
EngramConfig(cpu_offload=True, embedding_across_dp=True),
|
||||
EngramConfig(cpu_offload=True, dp_shared_memory=True),
|
||||
EngramConfig(host_file_gather=True),
|
||||
]
|
||||
assert len({config.compute_hash() for config in configs}) == len(configs)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user