[Bugfix] Re-enable prefill of max model length (#24446)
Signed-off-by: Yannick Schnider <yannick.schnider1@ibm.com>
This commit is contained in:
@@ -2247,14 +2247,28 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
|
||||
start_idx = self.input_batch.num_tokens_no_spec[req_idx]
|
||||
end_idx = start_idx + len(sampled_ids)
|
||||
assert end_idx <= self.max_model_len, (
|
||||
"Sampled token IDs exceed the max model length. "
|
||||
f"Total number of tokens: {end_idx} > max_model_len: "
|
||||
f"{self.max_model_len}")
|
||||
assert end_idx <= self.max_model_len + 1, (
|
||||
"Sampled token IDs exceed the max model length + 1. "
|
||||
f"Total number of tokens: {end_idx} > max_model_len + 1: "
|
||||
f"{self.max_model_len + 1}")
|
||||
|
||||
n_tokens_cache = len(sampled_ids)
|
||||
|
||||
# Sampled token IDs exceed the max model length by 1. This is
|
||||
# legitimate as we can still sample 1 last token when the context
|
||||
# length equals the max model length. Note that we do not need to
|
||||
# cache this token ID as the sequence finishes after this step.
|
||||
# Additionally, the buffers token_ids_cpu and is_token_ids are of
|
||||
# size max model length only.
|
||||
if end_idx == self.max_model_len + 1:
|
||||
n_tokens_cache -= 1
|
||||
|
||||
self.input_batch.token_ids_cpu[req_idx, start_idx:(
|
||||
start_idx + n_tokens_cache)] = sampled_ids[:n_tokens_cache]
|
||||
self.input_batch.is_token_ids[req_idx,
|
||||
start_idx:(start_idx +
|
||||
n_tokens_cache)] = True
|
||||
|
||||
self.input_batch.token_ids_cpu[req_idx,
|
||||
start_idx:end_idx] = sampled_ids
|
||||
self.input_batch.is_token_ids[req_idx, start_idx:end_idx] = True
|
||||
self.input_batch.num_tokens_no_spec[req_idx] = end_idx
|
||||
self.input_batch.num_tokens[req_idx] = end_idx
|
||||
|
||||
|
||||
Reference in New Issue
Block a user