[BugFix][MRV2] Don't create dummy requests longer than max_model_len (#49751)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -115,13 +115,18 @@ class InputBatch:
|
||||
expanded_idx_mapping = idx_mapping
|
||||
expanded_local_pos = torch.zeros(num_reqs, dtype=torch.int32, device=device)
|
||||
|
||||
num_scheduled_tokens = np.full(num_reqs, num_tokens // num_reqs, dtype=np.int32)
|
||||
num_scheduled_tokens[-1] += num_tokens % num_reqs
|
||||
# Distribute the remainder evenly so that no dummy request exceeds
|
||||
# ceil(num_tokens / num_reqs) <= max_model_len tokens.
|
||||
base_tokens = num_tokens // num_reqs
|
||||
num_extra = num_tokens % num_reqs
|
||||
num_scheduled_tokens = np.full(num_reqs, base_tokens, dtype=np.int32)
|
||||
if num_extra > 0:
|
||||
num_scheduled_tokens[-num_extra:] += 1
|
||||
assert int(num_scheduled_tokens.sum()) == num_tokens
|
||||
|
||||
# seq_len equals to query_len
|
||||
input_buffers.seq_lens[:num_reqs] = num_tokens // num_reqs
|
||||
input_buffers.seq_lens[num_reqs - 1] += num_tokens % num_reqs
|
||||
input_buffers.seq_lens[: num_reqs - num_extra] = base_tokens
|
||||
input_buffers.seq_lens[num_reqs - num_extra : num_reqs] = base_tokens + 1
|
||||
# Pad for full CUDA graph mode.
|
||||
input_buffers.seq_lens[num_reqs:] = 0
|
||||
seq_lens = input_buffers.seq_lens[:num_reqs]
|
||||
|
||||
@@ -59,8 +59,12 @@ def create_lora_capture_hook(
|
||||
return None
|
||||
|
||||
def hook(num_active_loras: int, num_reqs: int, num_tokens: int) -> None:
|
||||
# Match InputBatch.make_dummy: distribute the remainder evenly so no
|
||||
# dummy request exceeds ceil(num_tokens / num_reqs) tokens.
|
||||
num_scheduled = np.full(num_reqs, num_tokens // num_reqs, dtype=np.int32)
|
||||
num_scheduled[-1] += num_tokens % num_reqs
|
||||
num_extra = num_tokens % num_reqs
|
||||
if num_extra > 0:
|
||||
num_scheduled[-num_extra:] += 1
|
||||
with runner.maybe_select_dummy_loras(
|
||||
lora_config, num_scheduled, num_active_loras=num_active_loras
|
||||
):
|
||||
|
||||
@@ -564,8 +564,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_tokens = max(num_tokens, self.decode_query_len)
|
||||
num_reqs = num_tokens // self.decode_query_len
|
||||
assert num_tokens % self.decode_query_len == 0
|
||||
num_tokens_per_request = [num_tokens // num_reqs] * num_reqs
|
||||
num_tokens_per_request[-1] += num_tokens % num_reqs
|
||||
# Distribute the remainder evenly so no dummy request exceeds
|
||||
# ceil(num_tokens / num_reqs) <= max_model_len tokens.
|
||||
num_tokens_per_request = [
|
||||
num_tokens // num_reqs + (i >= num_reqs - num_tokens % num_reqs)
|
||||
for i in range(num_reqs)
|
||||
]
|
||||
|
||||
assert sum(num_tokens_per_request) == num_tokens
|
||||
num_scheduled_tokens = {
|
||||
|
||||
Reference in New Issue
Block a user