[BugFix][MRV2] Don't create dummy requests longer than max_model_len (#49751)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -59,8 +59,12 @@ def create_lora_capture_hook(
|
||||
return None
|
||||
|
||||
def hook(num_active_loras: int, num_reqs: int, num_tokens: int) -> None:
|
||||
# Match InputBatch.make_dummy: distribute the remainder evenly so no
|
||||
# dummy request exceeds ceil(num_tokens / num_reqs) tokens.
|
||||
num_scheduled = np.full(num_reqs, num_tokens // num_reqs, dtype=np.int32)
|
||||
num_scheduled[-1] += num_tokens % num_reqs
|
||||
num_extra = num_tokens % num_reqs
|
||||
if num_extra > 0:
|
||||
num_scheduled[-num_extra:] += 1
|
||||
with runner.maybe_select_dummy_loras(
|
||||
lora_config, num_scheduled, num_active_loras=num_active_loras
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user