Adds an opt-in backend, --engram-config '{"host_file_gather": true}', that
serves Qwen4Exp PLE rows from the checkpoint files through the page cache
instead of allocating the 47.7 GiB table on GPU or in pinned host memory.
- Qwen4ExpPLEFileGatherEmbedding allocates no table rows; load_weights keeps
the default loader's zero-copy safetensors views of each shard. At bind
(post-load hook) every view must be file-backed per /proc/self/maps (HF
cache symlinks work; heap/pinned/anonymous memory is rejected) and the
shards must cover the whole table.
- Each step, Qwen4ExpModelState.prepare_inputs computes the PLE n-gram ids,
copies them to pinned host memory, syncs the stream once, issues
POSIX_FADV_WILLNEED over the coalesced page ranges of the step's rows,
gathers the rows on the host, and copies them into a fixed device staging
buffer. The forward reads only the staging buffer, so FULL CUDA graph
capture is unaffected.
- Rows stay in the checkpoint dtype (FP8 table stays FP8, dequantized on the
GPU); dummy loads serve zeros; weight reload is rejected; takes precedence
over cpu_offload; rejected on ROCm and with elastic EP.
Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
168 lines
5.8 KiB
Python
168 lines
5.8 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Model-runner state for Qwen4Exp PLE inputs."""
|
|
|
|
from typing import Any
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
|
|
from vllm.config import VllmConfig
|
|
from vllm.v1.worker.gpu.input_batch import InputBatch
|
|
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
|
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
|
from vllm.v1.worker.gpu.states import RequestState
|
|
|
|
from .ngram_embedding import (
|
|
Qwen4ExpNGramEmbedding,
|
|
Qwen4ExpPLEFileGatherEmbedding,
|
|
stage_checkpoint_rows,
|
|
)
|
|
|
|
|
|
class Qwen4ExpModelState(MambaHybridModelState):
|
|
"""Add rollback-safe PLE n-gram context to the model inputs."""
|
|
|
|
def __init__(
|
|
self,
|
|
vllm_config: VllmConfig,
|
|
model: nn.Module,
|
|
encoder_cache: EncoderCache | None,
|
|
device: torch.device,
|
|
) -> None:
|
|
super().__init__(vllm_config, model, encoder_cache, device)
|
|
config = self.model_config.hf_text_config
|
|
self.uses_ngram_embedding = bool(config.ple_layer_ids)
|
|
if not self.uses_ngram_embedding:
|
|
self.ngram_context_len = 0
|
|
self.ngram_eos_token_id = 0
|
|
return
|
|
|
|
if vllm_config.parallel_config.pipeline_parallel_size > 1:
|
|
raise RuntimeError(
|
|
"N-gram PLE embedding currently requires "
|
|
"pipeline_parallel_size=1 because non-first pipeline ranks do "
|
|
"not receive the raw input_ids required by PLE. Please run "
|
|
"with PP=1."
|
|
)
|
|
|
|
self.ngram_context_len = int(config.ngram_size) - 1
|
|
if self.ngram_context_len <= 0:
|
|
raise ValueError("N-gram embedding requires context length >= 1.")
|
|
self.ngram_eos_token_id = int(config.eos_token_id)
|
|
# PLE runs inside captured regions, so these buffers keep a fixed shape
|
|
# and address as the active request count changes between replays.
|
|
self.ngram_context = torch.full(
|
|
(self.max_num_reqs, self.ngram_context_len),
|
|
self.ngram_eos_token_id,
|
|
dtype=torch.int32,
|
|
device=self.device,
|
|
)
|
|
self.ngram_context_offsets = torch.arange(
|
|
-self.ngram_context_len,
|
|
0,
|
|
dtype=torch.int64,
|
|
device=self.device,
|
|
)
|
|
self.ple_query_start_loc = torch.zeros(
|
|
self.max_num_reqs + 1,
|
|
dtype=torch.int32,
|
|
device=self.device,
|
|
)
|
|
self.file_gather_modules = [
|
|
module
|
|
for module in model.modules()
|
|
if isinstance(module, Qwen4ExpNGramEmbedding)
|
|
and isinstance(module.ngram_embedding, Qwen4ExpPLEFileGatherEmbedding)
|
|
]
|
|
|
|
def _prepare_ngram_context(
|
|
self,
|
|
input_batch: InputBatch,
|
|
req_states: RequestState,
|
|
) -> torch.Tensor:
|
|
num_reqs = input_batch.num_reqs
|
|
context = self.ngram_context
|
|
context.fill_(self.ngram_eos_token_id)
|
|
if num_reqs == 0:
|
|
return context
|
|
|
|
request_indices = input_batch.idx_mapping[:num_reqs].long()
|
|
context_end = req_states.num_computed_tokens.gpu[request_indices].long()
|
|
token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets
|
|
valid_tokens = token_indices >= 0
|
|
token_indices.clamp_min_(0)
|
|
context_tokens = req_states.all_token_ids.gpu[
|
|
request_indices.unsqueeze(1), token_indices
|
|
]
|
|
context[:num_reqs].copy_(
|
|
torch.where(
|
|
valid_tokens,
|
|
context_tokens,
|
|
context_tokens.new_full((), self.ngram_eos_token_id),
|
|
)
|
|
)
|
|
return context
|
|
|
|
def prepare_inputs(
|
|
self,
|
|
input_batch: InputBatch,
|
|
req_states: RequestState,
|
|
) -> dict[str, Any]:
|
|
model_inputs = super().prepare_inputs(input_batch, req_states)
|
|
if not self.uses_ngram_embedding:
|
|
return model_inputs
|
|
|
|
num_reqs_padded = input_batch.num_reqs_after_padding
|
|
query_start_loc = self.ple_query_start_loc
|
|
query_start_loc[: num_reqs_padded + 1].copy_(input_batch.query_start_loc)
|
|
# Represent unused capacity as trailing zero-length requests.
|
|
query_start_loc[num_reqs_padded + 1 :].copy_(input_batch.query_start_loc[-1])
|
|
ngram_context = self._prepare_ngram_context(input_batch, req_states)
|
|
if self.file_gather_modules:
|
|
stage_checkpoint_rows(
|
|
self.file_gather_modules,
|
|
input_batch.input_ids,
|
|
query_start_loc,
|
|
ngram_context,
|
|
)
|
|
model_inputs.update(
|
|
query_start_loc=query_start_loc,
|
|
ngram_context=ngram_context,
|
|
)
|
|
return model_inputs
|
|
|
|
def prepare_dummy_inputs(
|
|
self,
|
|
num_reqs: int,
|
|
num_tokens: int,
|
|
) -> dict[str, Any]:
|
|
model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens)
|
|
if not self.uses_ngram_embedding:
|
|
return model_inputs
|
|
|
|
query_start_loc = self.ple_query_start_loc
|
|
query_start_loc[0] = 0
|
|
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
|
|
query_lens = torch.full(
|
|
(num_reqs,),
|
|
tokens_per_req,
|
|
dtype=query_start_loc.dtype,
|
|
device=query_start_loc.device,
|
|
)
|
|
if num_extra_tokens > 0:
|
|
query_lens[-num_extra_tokens:] += 1
|
|
torch.cumsum(query_lens, dim=0, out=query_start_loc[1 : num_reqs + 1])
|
|
query_start_loc[num_reqs + 1 :].fill_(num_tokens)
|
|
|
|
ngram_context = self.ngram_context
|
|
ngram_context.fill_(self.ngram_eos_token_id)
|
|
model_inputs.update(
|
|
query_start_loc=query_start_loc,
|
|
ngram_context=ngram_context,
|
|
)
|
|
return model_inputs
|
|
|
|
|
|
__all__ = ["Qwen4ExpModelState"]
|