[Bugfix][MRV2] Match fast-prefill padding to active LoRA batches (#56456)
Signed-off-by: waizuichougou <2082431897@qq.com> Signed-off-by: waizuichougou <105572299+waizuichougou@users.noreply.github.com>
This commit is contained in:
@@ -1271,6 +1271,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
scheduler_output: SchedulerOutput,
|
||||
batch_req_state: "BatchReqState",
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
num_active_loras: int,
|
||||
) -> InputBatch:
|
||||
num_tokens = batch_req_state.num_tokens
|
||||
num_tokens_after_padding = max(num_tokens, batch_desc.num_tokens)
|
||||
@@ -1405,6 +1406,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cu_num_logits_np,
|
||||
batch_req_state.has_prefill,
|
||||
batch_desc,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
|
||||
# CPU upper bound on seq_lens; padded entries left at zero.
|
||||
@@ -1710,7 +1712,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cudagraph_stats = make_cudagraph_stats(batch_desc, num_toks)
|
||||
assert batch_req_state is not None
|
||||
input_batch = self.prepare_inputs(
|
||||
scheduler_output, batch_req_state, batch_desc
|
||||
scheduler_output, batch_req_state, batch_desc, num_active_loras
|
||||
)
|
||||
block_tables, slot_mappings = self.prepare_attn(input_batch)
|
||||
# Mamba "align" pre-copy: migrate recurrent state across block
|
||||
|
||||
Reference in New Issue
Block a user