[Bugfix][MRV2] Match fast-prefill padding to active LoRA batches (#56456)
Signed-off-by: waizuichougou <2082431897@qq.com> Signed-off-by: waizuichougou <105572299+waizuichougou@users.noreply.github.com>
This commit is contained in:
@@ -95,6 +95,7 @@ class FastPrefillHelper:
|
||||
cu_num_logits_np: np.ndarray,
|
||||
has_prefill: bool,
|
||||
batch_desc: "BatchExecutionDescriptor",
|
||||
num_active_loras: int = 0,
|
||||
) -> FastPrefillBatchMetadata | None:
|
||||
if (
|
||||
not has_prefill
|
||||
@@ -116,7 +117,7 @@ class FastPrefillHelper:
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_logits,
|
||||
uniform_token_count=None,
|
||||
num_active_loras=0,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
num_logits_padded = min(desc.num_tokens, self.max_num_tokens)
|
||||
return FastPrefillBatchMetadata(
|
||||
|
||||
@@ -1271,6 +1271,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
scheduler_output: SchedulerOutput,
|
||||
batch_req_state: "BatchReqState",
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
num_active_loras: int,
|
||||
) -> InputBatch:
|
||||
num_tokens = batch_req_state.num_tokens
|
||||
num_tokens_after_padding = max(num_tokens, batch_desc.num_tokens)
|
||||
@@ -1405,6 +1406,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cu_num_logits_np,
|
||||
batch_req_state.has_prefill,
|
||||
batch_desc,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
|
||||
# CPU upper bound on seq_lens; padded entries left at zero.
|
||||
@@ -1710,7 +1712,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cudagraph_stats = make_cudagraph_stats(batch_desc, num_toks)
|
||||
assert batch_req_state is not None
|
||||
input_batch = self.prepare_inputs(
|
||||
scheduler_output, batch_req_state, batch_desc
|
||||
scheduler_output, batch_req_state, batch_desc, num_active_loras
|
||||
)
|
||||
block_tables, slot_mappings = self.prepare_attn(input_batch)
|
||||
# Mamba "align" pre-copy: migrate recurrent state across block
|
||||
|
||||
@@ -149,7 +149,9 @@ class MMEncoderModelRunner(GPUModelRunner):
|
||||
num_tokens=batch_req_state.num_tokens,
|
||||
num_reqs=None,
|
||||
)
|
||||
self.prepare_inputs(scheduler_output, batch_req_state, batch_desc)
|
||||
self.prepare_inputs(
|
||||
scheduler_output, batch_req_state, batch_desc, num_active_loras=0
|
||||
)
|
||||
|
||||
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
|
||||
if self.lora_config is not None:
|
||||
|
||||
Reference in New Issue
Block a user