[ModelRunner V2] Misc minor simplifications and optimizations (#33467)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -339,10 +339,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
gc.collect()
|
||||
|
||||
def reset_mm_cache(self) -> None:
|
||||
self.encoder_runner.reset_mm_cache()
|
||||
if self.supports_mm_inputs:
|
||||
self.encoder_runner.reset_mm_cache()
|
||||
|
||||
def reset_encoder_cache(self) -> None:
|
||||
self.encoder_runner.reset_encoder_cache()
|
||||
if self.supports_mm_inputs:
|
||||
self.encoder_runner.reset_encoder_cache()
|
||||
|
||||
def _get_num_input_tokens(self, num_scheduled_tokens: int) -> int:
|
||||
# SP is not supported yet.
|
||||
@@ -402,10 +404,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
def finish_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
finished_req_ids = scheduler_output.finished_req_ids
|
||||
if scheduler_output.preempted_req_ids:
|
||||
finished_req_ids = finished_req_ids.union(
|
||||
scheduler_output.preempted_req_ids
|
||||
)
|
||||
preempted_req_ids = scheduler_output.preempted_req_ids
|
||||
if preempted_req_ids:
|
||||
finished_req_ids = finished_req_ids.union(preempted_req_ids)
|
||||
for req_id in finished_req_ids:
|
||||
self.req_states.remove_request(req_id)
|
||||
if self.supports_mm_inputs:
|
||||
@@ -477,28 +478,21 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
|
||||
def prepare_inputs(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
num_tokens_after_padding: int,
|
||||
self, scheduler_output: SchedulerOutput, num_tokens_after_padding: int
|
||||
) -> InputBatch:
|
||||
num_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
assert num_tokens > 0
|
||||
num_reqs = len(scheduler_output.num_scheduled_tokens)
|
||||
num_tokens_per_req = scheduler_output.num_scheduled_tokens
|
||||
num_reqs = len(num_tokens_per_req)
|
||||
|
||||
# Decode first, then prefill.
|
||||
# batch_idx -> req_id
|
||||
req_ids = sorted(
|
||||
scheduler_output.num_scheduled_tokens.keys(),
|
||||
key=lambda k: scheduler_output.num_scheduled_tokens[k],
|
||||
)
|
||||
num_scheduled_tokens = np.array(
|
||||
[scheduler_output.num_scheduled_tokens[i] for i in req_ids], dtype=np.int32
|
||||
)
|
||||
req_ids = sorted(num_tokens_per_req, key=num_tokens_per_req.get) # type: ignore[arg-type]
|
||||
numtoks_iter = map(num_tokens_per_req.get, req_ids)
|
||||
num_scheduled_tokens = np.fromiter(numtoks_iter, dtype=np.int32, count=num_reqs)
|
||||
|
||||
idx_mapping_list = [
|
||||
self.req_states.req_id_to_index[req_id] for req_id in req_ids
|
||||
]
|
||||
idx_mapping_np = np.array(idx_mapping_list, dtype=np.int32)
|
||||
idx_mapping_iter = map(self.req_states.req_id_to_index.get, req_ids)
|
||||
idx_mapping_np = np.fromiter(idx_mapping_iter, dtype=np.int32, count=num_reqs)
|
||||
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
|
||||
|
||||
# Get the number of draft tokens for each request.
|
||||
@@ -889,8 +883,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
@torch.inference_mode()
|
||||
def sample_tokens(
|
||||
self,
|
||||
grammar_output: GrammarOutput | None,
|
||||
self, grammar_output: GrammarOutput | None
|
||||
) -> AsyncOutput | ModelRunnerOutput:
|
||||
assert self.execute_model_state is not None
|
||||
hidden_states, input_batch, kv_connector_output = self.execute_model_state
|
||||
|
||||
Reference in New Issue
Block a user