[Bugfix] Re-enable prefill of max model length (#24446)

Signed-off-by: Yannick Schnider <yannick.schnider1@ibm.com>
This commit is contained in:
Yannick Schnider
2025-10-03 14:13:34 +02:00
committed by GitHub
parent 812b7f54a8
commit 8ee846c27c
2 changed files with 112 additions and 7 deletions
+21 -7
View File
@@ -2247,14 +2247,28 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
start_idx = self.input_batch.num_tokens_no_spec[req_idx]
end_idx = start_idx + len(sampled_ids)
assert end_idx <= self.max_model_len, (
"Sampled token IDs exceed the max model length. "
f"Total number of tokens: {end_idx} > max_model_len: "
f"{self.max_model_len}")
assert end_idx <= self.max_model_len + 1, (
"Sampled token IDs exceed the max model length + 1. "
f"Total number of tokens: {end_idx} > max_model_len + 1: "
f"{self.max_model_len + 1}")
n_tokens_cache = len(sampled_ids)
# Sampled token IDs exceed the max model length by 1. This is
# legitimate as we can still sample 1 last token when the context
# length equals the max model length. Note that we do not need to
# cache this token ID as the sequence finishes after this step.
# Additionally, the buffers token_ids_cpu and is_token_ids are of
# size max model length only.
if end_idx == self.max_model_len + 1:
n_tokens_cache -= 1
self.input_batch.token_ids_cpu[req_idx, start_idx:(
start_idx + n_tokens_cache)] = sampled_ids[:n_tokens_cache]
self.input_batch.is_token_ids[req_idx,
start_idx:(start_idx +
n_tokens_cache)] = True
self.input_batch.token_ids_cpu[req_idx,
start_idx:end_idx] = sampled_ids
self.input_batch.is_token_ids[req_idx, start_idx:end_idx] = True
self.input_batch.num_tokens_no_spec[req_idx] = end_idx
self.input_batch.num_tokens[req_idx] = end_idx