[ModelRunner V2] Simplify recent UnlimitedOCR-related changes (#46975)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-29 09:56:17 -07:00
committed by GitHub
parent 7be582697b
commit 8ad4a01825
5 changed files with 11 additions and 21 deletions
+3 -2
View File
@@ -96,8 +96,8 @@ class InputBatch:
# Whether any requests in batch use structured output.
has_structured_output_reqs: bool
# [num_reqs_after_padding] per-request prompt length for R-SWA (optional).
rswa_prefix_lens: torch.Tensor | None = None
# [num_reqs] per-request prompt length, only populated for R-SWA.
prompt_lens: torch.Tensor | None
@classmethod
def make_dummy(
@@ -178,6 +178,7 @@ class InputBatch:
cu_num_logits=cu_num_logits,
cu_num_logits_np=cu_num_logits_np,
has_structured_output_reqs=False,
prompt_lens=None,
)
+5 -16
View File
@@ -223,13 +223,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_num_tokens=self.max_num_tokens,
device=self.device,
)
# R-SWA: persistent GPU buffer for per-request prefix lengths (CUDA-graph safe).
self.rswa_prefix_lens_buffer: torch.Tensor | None = None
if self.model_config.rswa_window is not None:
self.rswa_prefix_lens_buffer = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=self.device
)
if self.use_pp:
self.pp_handler = PPHandler(
max_num_reqs=self.max_num_reqs,
@@ -992,14 +985,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# max_seq_len is only consumed by the PP `compute_need_sampled_mask`
max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np]
rswa_prefix_lens = None
if self.rswa_prefix_lens_buffer is not None:
rswa_prefix_lens = self.rswa_prefix_lens_buffer[:num_reqs_padded]
rswa_prefix_lens[:num_reqs] = self.req_states.prompt_len.gpu[
idx_mapping[:num_reqs]
]
if num_reqs_padded > num_reqs:
rswa_prefix_lens[num_reqs:].zero_()
prompt_lens = None
if self.model_config.rswa_window is not None:
# prompt_lens is only used in R-SWA case.
prompt_lens = self.req_states.prompt_len.gpu[idx_mapping]
return InputBatch(
req_ids=req_ids,
@@ -1031,7 +1020,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
cu_num_logits=cu_num_logits,
cu_num_logits_np=cu_num_logits_np,
has_structured_output_reqs=scheduler_output.has_structured_output_requests,
rswa_prefix_lens=rswa_prefix_lens,
prompt_lens=prompt_lens,
)
def prepare_attn(
+1 -1
View File
@@ -168,6 +168,6 @@ class DefaultModelState(ModelState):
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
positions=input_batch.positions,
for_cudagraph_capture=for_capture,
rswa_prefix_lens=input_batch.rswa_prefix_lens,
rswa_prefix_lens=input_batch.prompt_lens,
)
return attn_metadata
@@ -146,7 +146,7 @@ class EncoderDecoderModelState(ModelState):
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
model_specific_attn_metadata=enc_dec_attn_metadata,
for_cudagraph_capture=for_capture,
rswa_prefix_lens=input_batch.rswa_prefix_lens,
rswa_prefix_lens=input_batch.prompt_lens,
)
return attn_metadata
@@ -141,7 +141,7 @@ class MambaHybridModelState(DefaultModelState):
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
model_specific_attn_metadata=mamba_attn_metadata,
for_cudagraph_capture=for_capture,
rswa_prefix_lens=input_batch.rswa_prefix_lens,
rswa_prefix_lens=input_batch.prompt_lens,
)
def postprocess_state(