Files
MOVZXandHinata 43fbd6510b [Qwen4Exp] Serve PLE rows from the loader's mapped checkpoint shards (PR #58815)
Adds an opt-in backend, --engram-config '{"host_file_gather": true}', that
serves Qwen4Exp PLE rows from the checkpoint files through the page cache
instead of allocating the 47.7 GiB table on GPU or in pinned host memory.

- Qwen4ExpPLEFileGatherEmbedding allocates no table rows; load_weights keeps
  the default loader's zero-copy safetensors views of each shard. At bind
  (post-load hook) every view must be file-backed per /proc/self/maps (HF
  cache symlinks work; heap/pinned/anonymous memory is rejected) and the
  shards must cover the whole table.
- Each step, Qwen4ExpModelState.prepare_inputs computes the PLE n-gram ids,
  copies them to pinned host memory, syncs the stream once, issues
  POSIX_FADV_WILLNEED over the coalesced page ranges of the step's rows,
  gathers the rows on the host, and copies them into a fixed device staging
  buffer. The forward reads only the staging buffer, so FULL CUDA graph
  capture is unaffected.
- Rows stay in the checkpoint dtype (FP8 table stays FP8, dequantized on the
  GPU); dummy loads serve zeros; weight reload is rejected; takes precedence
  over cpu_offload; rejected on ROCm and with elastic EP.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
2026-09-27 09:54:42 +07:00

168 lines
5.8 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Model-runner state for Qwen4Exp PLE inputs."""
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from vllm.v1.worker.gpu.states import RequestState
from .ngram_embedding import (
Qwen4ExpNGramEmbedding,
Qwen4ExpPLEFileGatherEmbedding,
stage_checkpoint_rows,
)
class Qwen4ExpModelState(MambaHybridModelState):
"""Add rollback-safe PLE n-gram context to the model inputs."""
def __init__(
self,
vllm_config: VllmConfig,
model: nn.Module,
encoder_cache: EncoderCache | None,
device: torch.device,
) -> None:
super().__init__(vllm_config, model, encoder_cache, device)
config = self.model_config.hf_text_config
self.uses_ngram_embedding = bool(config.ple_layer_ids)
if not self.uses_ngram_embedding:
self.ngram_context_len = 0
self.ngram_eos_token_id = 0
return
if vllm_config.parallel_config.pipeline_parallel_size > 1:
raise RuntimeError(
"N-gram PLE embedding currently requires "
"pipeline_parallel_size=1 because non-first pipeline ranks do "
"not receive the raw input_ids required by PLE. Please run "
"with PP=1."
)
self.ngram_context_len = int(config.ngram_size) - 1
if self.ngram_context_len <= 0:
raise ValueError("N-gram embedding requires context length >= 1.")
self.ngram_eos_token_id = int(config.eos_token_id)
# PLE runs inside captured regions, so these buffers keep a fixed shape
# and address as the active request count changes between replays.
self.ngram_context = torch.full(
(self.max_num_reqs, self.ngram_context_len),
self.ngram_eos_token_id,
dtype=torch.int32,
device=self.device,
)
self.ngram_context_offsets = torch.arange(
-self.ngram_context_len,
0,
dtype=torch.int64,
device=self.device,
)
self.ple_query_start_loc = torch.zeros(
self.max_num_reqs + 1,
dtype=torch.int32,
device=self.device,
)
self.file_gather_modules = [
module
for module in model.modules()
if isinstance(module, Qwen4ExpNGramEmbedding)
and isinstance(module.ngram_embedding, Qwen4ExpPLEFileGatherEmbedding)
]
def _prepare_ngram_context(
self,
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
context = self.ngram_context
context.fill_(self.ngram_eos_token_id)
if num_reqs == 0:
return context
request_indices = input_batch.idx_mapping[:num_reqs].long()
context_end = req_states.num_computed_tokens.gpu[request_indices].long()
token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets
valid_tokens = token_indices >= 0
token_indices.clamp_min_(0)
context_tokens = req_states.all_token_ids.gpu[
request_indices.unsqueeze(1), token_indices
]
context[:num_reqs].copy_(
torch.where(
valid_tokens,
context_tokens,
context_tokens.new_full((), self.ngram_eos_token_id),
)
)
return context
def prepare_inputs(
self,
input_batch: InputBatch,
req_states: RequestState,
) -> dict[str, Any]:
model_inputs = super().prepare_inputs(input_batch, req_states)
if not self.uses_ngram_embedding:
return model_inputs
num_reqs_padded = input_batch.num_reqs_after_padding
query_start_loc = self.ple_query_start_loc
query_start_loc[: num_reqs_padded + 1].copy_(input_batch.query_start_loc)
# Represent unused capacity as trailing zero-length requests.
query_start_loc[num_reqs_padded + 1 :].copy_(input_batch.query_start_loc[-1])
ngram_context = self._prepare_ngram_context(input_batch, req_states)
if self.file_gather_modules:
stage_checkpoint_rows(
self.file_gather_modules,
input_batch.input_ids,
query_start_loc,
ngram_context,
)
model_inputs.update(
query_start_loc=query_start_loc,
ngram_context=ngram_context,
)
return model_inputs
def prepare_dummy_inputs(
self,
num_reqs: int,
num_tokens: int,
) -> dict[str, Any]:
model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens)
if not self.uses_ngram_embedding:
return model_inputs
query_start_loc = self.ple_query_start_loc
query_start_loc[0] = 0
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
query_lens = torch.full(
(num_reqs,),
tokens_per_req,
dtype=query_start_loc.dtype,
device=query_start_loc.device,
)
if num_extra_tokens > 0:
query_lens[-num_extra_tokens:] += 1
torch.cumsum(query_lens, dim=0, out=query_start_loc[1 : num_reqs + 1])
query_start_loc[num_reqs + 1 :].fill_(num_tokens)
ngram_context = self.ngram_context
ngram_context.fill_(self.ngram_eos_token_id)
model_inputs.update(
query_start_loc=query_start_loc,
ngram_context=ngram_context,
)
return model_inputs
__all__ = ["Qwen4ExpModelState"]