[Model] Support DeepSeek-V4.1-Flash (#56214)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jee Jee Li
andyluo7
OpenAI Codex
zjy0516
Claude Opus 5
parent
e6821ceac9
commit
e77daef89e
@@ -813,6 +813,9 @@ class GPUModelRunner(
|
||||
|
||||
# Persistent buffers for CUDA graphs.
|
||||
self.input_ids = self._make_buffer(self.max_num_tokens, dtype=torch.int32)
|
||||
# [max_num_reqs, depth] prompt token ids preceding each request's chunk
|
||||
# start, for models exposing `token_lookback_depth`. Sized after load.
|
||||
self.lookback_token_ids: CpuGpuBuffer | None = None
|
||||
self.positions = torch.zeros(
|
||||
self.max_num_tokens, dtype=torch.int64, device=self.device
|
||||
)
|
||||
@@ -1065,9 +1068,36 @@ class GPUModelRunner(
|
||||
)
|
||||
return self._mamba_bufs
|
||||
|
||||
def _init_model_kwargs(self):
|
||||
def _prepare_lookback_token_ids(self, num_reqs: int) -> torch.Tensor:
|
||||
"""Gather, per request, the `depth` prompt token ids preceding its
|
||||
first scheduled token (column j is position start - 1 - j); -1 where
|
||||
the position is before the prompt or already past it. Generated
|
||||
positions are left to the model: under async scheduling the CPU token
|
||||
table holds placeholders for them."""
|
||||
buf = self.lookback_token_ids
|
||||
assert buf is not None
|
||||
buf.np.fill(-1)
|
||||
if num_reqs > 0:
|
||||
depth = buf.np.shape[1]
|
||||
starts = self.input_batch.num_computed_tokens_cpu[:num_reqs, None]
|
||||
pos = starts - np.arange(1, depth + 1)
|
||||
num_prompt = self.input_batch.num_prompt_tokens[:num_reqs, None]
|
||||
valid = (pos >= 0) & (pos < num_prompt)
|
||||
rows = np.arange(num_reqs)[:, None]
|
||||
ids = self.input_batch.token_ids_cpu[rows, np.clip(pos, 0, None)]
|
||||
buf.np[:num_reqs] = np.where(valid, ids, -1)
|
||||
return buf.copy_to_gpu()
|
||||
|
||||
def _init_model_kwargs(self, num_reqs: int | None = None):
|
||||
model_kwargs = dict[str, Any]()
|
||||
|
||||
if self.lookback_token_ids is not None:
|
||||
if num_reqs is None:
|
||||
num_reqs = self.input_batch.num_reqs
|
||||
model_kwargs["lookback_token_ids"] = self._prepare_lookback_token_ids(
|
||||
num_reqs
|
||||
)
|
||||
|
||||
if not self.is_pooling_model:
|
||||
return model_kwargs
|
||||
|
||||
@@ -5314,6 +5344,11 @@ class GPUModelRunner(
|
||||
self.model = model_loader.load_model(
|
||||
vllm_config=self.vllm_config, model_config=self.model_config
|
||||
)
|
||||
lookback_depth = getattr(self.model, "token_lookback_depth", 0)
|
||||
if lookback_depth > 0:
|
||||
self.lookback_token_ids = self._make_buffer(
|
||||
self.max_num_reqs, lookback_depth, dtype=torch.int32
|
||||
)
|
||||
if self.lora_config:
|
||||
self.model = self.load_lora_model(
|
||||
self.model, self.vllm_config, self.device
|
||||
@@ -6076,7 +6111,7 @@ class GPUModelRunner(
|
||||
):
|
||||
# Make sure padding doesn't exceed max_num_tokens
|
||||
assert num_tokens_padded <= self.max_num_tokens
|
||||
model_kwargs = self._init_model_kwargs()
|
||||
model_kwargs = self._init_model_kwargs(num_reqs=0)
|
||||
if self.supports_mm_inputs and not self.model_config.is_encoder_decoder:
|
||||
input_ids, inputs_embeds = self._prepare_mm_inputs(num_tokens_padded)
|
||||
|
||||
@@ -6087,7 +6122,7 @@ class GPUModelRunner(
|
||||
elif self.enable_prompt_embeds:
|
||||
input_ids = None
|
||||
inputs_embeds = self.inputs_embeds.gpu[:num_tokens_padded]
|
||||
model_kwargs = self._init_model_kwargs()
|
||||
model_kwargs = self._init_model_kwargs(num_reqs=0)
|
||||
else:
|
||||
input_ids = self.input_ids.gpu[:num_tokens_padded]
|
||||
inputs_embeds = None
|
||||
|
||||
Reference in New Issue
Block a user