[Model Runner V2] Fix seq_lens_cpu_upper_bound (#42202)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-11 10:37:50 -07:00
committed by GitHub
parent 7863fff6e5
commit 9af6a5ed75
2 changed files with 18 additions and 35 deletions
+15 -31
View File
@@ -712,15 +712,26 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.sampler.apply_staged_writes() self.sampler.apply_staged_writes()
def update_requests(self, scheduler_output: SchedulerOutput) -> None: def update_requests(self, scheduler_output: SchedulerOutput) -> None:
# Add new blocks for the existing requests. # Add new blocks and update num_computed_tokens for the existing requests.
reqs = scheduler_output.scheduled_cached_reqs reqs = scheduler_output.scheduled_cached_reqs
for req_new_block_ids, req_id in zip(reqs.new_block_ids, reqs.req_ids): num_computed_tokens_np = self.req_states.num_computed_tokens_np
for req_id, num_computed_tokens, req_new_block_ids in zip(
reqs.req_ids, reqs.num_computed_tokens, reqs.new_block_ids
):
req_index = self.req_states.req_id_to_index[req_id]
num_computed_tokens_np[req_index] = num_computed_tokens
if req_new_block_ids is not None: if req_new_block_ids is not None:
req_index = self.req_states.req_id_to_index[req_id]
self.block_tables.append_block_ids( self.block_tables.append_block_ids(
req_index, req_new_block_ids, overwrite=False req_index, req_new_block_ids, overwrite=False
) )
# Update num_computed_prefill_tokens.
np.minimum(
self.req_states.num_computed_tokens_np,
self.req_states.prefill_len.np,
out=self.req_states.num_computed_prefill_tokens,
)
def prepare_inputs( def prepare_inputs(
self, scheduler_output: SchedulerOutput, batch_desc: BatchExecutionDescriptor self, scheduler_output: SchedulerOutput, batch_desc: BatchExecutionDescriptor
) -> InputBatch: ) -> InputBatch:
@@ -787,10 +798,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc) async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1] query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1] query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
is_prefilling_np = ( is_prefilling_np = self.req_states.is_prefilling(idx_mapping_np)
self.req_states.num_computed_prefill_tokens[idx_mapping_np]
< self.req_states.prefill_len.np[idx_mapping_np]
)
# Get prefill tokens if any. # Get prefill tokens if any.
if np.any(is_prefilling_np): if np.any(is_prefilling_np):
@@ -973,18 +981,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.total_len.gpu, self.req_states.total_len.gpu,
) )
# Update the number of computed prefill tokens.
idx_mapping_np = input_batch.idx_mapping_np
computed_prefill = self.req_states.num_computed_prefill_tokens
computed_prefill[idx_mapping_np] += input_batch.num_scheduled_tokens
np.minimum(
computed_prefill, self.req_states.prefill_len.np, out=computed_prefill
)
# Advance the CPU mirror optimistically (assume all scheduled accepted).
self.req_states.num_computed_tokens_np[idx_mapping_np] += (
input_batch.num_scheduled_tokens
)
self.model_state.postprocess_state(input_batch, num_sampled) self.model_state.postprocess_state(input_batch, num_sampled)
@torch.inference_mode() @torch.inference_mode()
@@ -1357,18 +1353,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch.query_start_loc, input_batch.query_start_loc,
) )
# Update the number of computed prefill tokens.
idx_mapping_np = input_batch.idx_mapping_np
computed_prefill = self.req_states.num_computed_prefill_tokens
computed_prefill[idx_mapping_np] += input_batch.num_scheduled_tokens
np.minimum(
computed_prefill, self.req_states.prefill_len.np, out=computed_prefill
)
# Advance the CPU mirror optimistically (assume all scheduled accepted).
self.req_states.num_computed_tokens_np[idx_mapping_np] += (
input_batch.num_scheduled_tokens
)
def shutdown(self) -> None: def shutdown(self) -> None:
"""Release GPU tensors (model weights, KV caches, workspace) so that """Release GPU tensors (model weights, KV caches, workspace) so that
memory is reclaimable when running in the same process.""" memory is reclaimable when running in the same process."""
+3 -4
View File
@@ -104,9 +104,8 @@ class RequestState:
self.num_computed_prefill_tokens[req_idx] = num_computed_tokens self.num_computed_prefill_tokens[req_idx] = num_computed_tokens
self.num_computed_tokens_np[req_idx] = num_computed_tokens self.num_computed_tokens_np[req_idx] = num_computed_tokens
self.num_computed_tokens.stage_write_elem(req_idx, num_computed_tokens) self.num_computed_tokens.stage_write_elem(req_idx, num_computed_tokens)
self.num_computed_tokens_np[req_idx] = num_computed_tokens
if num_computed_tokens > 0 and num_computed_tokens <= prefill_len: if 0 < num_computed_tokens <= prefill_len:
# For PD disagg or resumed requests: set last_sampled to the last # For PD disagg or resumed requests: set last_sampled to the last
# computed token so the first decode step gets the right input_id. # computed token so the first decode step gets the right input_id.
# For fresh prefill requests (num_computed_tokens == 0) the tensor # For fresh prefill requests (num_computed_tokens == 0) the tensor
@@ -134,8 +133,8 @@ class RequestState:
self.free_indices.append(req_idx) self.free_indices.append(req_idx)
return True return True
def any_prefills(self, idx_mapping_np: np.ndarray) -> bool: def is_prefilling(self, idx_mapping_np: np.ndarray) -> np.ndarray:
return np.any( return (
self.num_computed_prefill_tokens[idx_mapping_np] self.num_computed_prefill_tokens[idx_mapping_np]
< self.prefill_len.np[idx_mapping_np] < self.prefill_len.np[idx_mapping_np]
) )