diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index a6a2b296e3..006e11e450 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -96,8 +96,8 @@ class InputBatch: # Whether any requests in batch use structured output. has_structured_output_reqs: bool - # [num_reqs_after_padding] per-request prompt length for R-SWA (optional). - rswa_prefix_lens: torch.Tensor | None = None + # [num_reqs] per-request prompt length, only populated for R-SWA. + prompt_lens: torch.Tensor | None @classmethod def make_dummy( @@ -178,6 +178,7 @@ class InputBatch: cu_num_logits=cu_num_logits, cu_num_logits_np=cu_num_logits_np, has_structured_output_reqs=False, + prompt_lens=None, ) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 0f57e8a31c..8869e93f9f 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -223,13 +223,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): max_num_tokens=self.max_num_tokens, device=self.device, ) - # R-SWA: persistent GPU buffer for per-request prefix lengths (CUDA-graph safe). - self.rswa_prefix_lens_buffer: torch.Tensor | None = None - if self.model_config.rswa_window is not None: - self.rswa_prefix_lens_buffer = torch.zeros( - self.max_num_reqs, dtype=torch.int32, device=self.device - ) - if self.use_pp: self.pp_handler = PPHandler( max_num_reqs=self.max_num_reqs, @@ -992,14 +985,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): # max_seq_len is only consumed by the PP `compute_need_sampled_mask` max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np] - rswa_prefix_lens = None - if self.rswa_prefix_lens_buffer is not None: - rswa_prefix_lens = self.rswa_prefix_lens_buffer[:num_reqs_padded] - rswa_prefix_lens[:num_reqs] = self.req_states.prompt_len.gpu[ - idx_mapping[:num_reqs] - ] - if num_reqs_padded > num_reqs: - rswa_prefix_lens[num_reqs:].zero_() + prompt_lens = None + if self.model_config.rswa_window is not None: + # prompt_lens is only used in R-SWA case. + prompt_lens = self.req_states.prompt_len.gpu[idx_mapping] return InputBatch( req_ids=req_ids, @@ -1031,7 +1020,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): cu_num_logits=cu_num_logits, cu_num_logits_np=cu_num_logits_np, has_structured_output_reqs=scheduler_output.has_structured_output_requests, - rswa_prefix_lens=rswa_prefix_lens, + prompt_lens=prompt_lens, ) def prepare_attn( diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index f760fc36de..e5e89da2b2 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -168,6 +168,6 @@ class DefaultModelState(ModelState): dcp_local_seq_lens=input_batch.dcp_local_seq_lens, positions=input_batch.positions, for_cudagraph_capture=for_capture, - rswa_prefix_lens=input_batch.rswa_prefix_lens, + rswa_prefix_lens=input_batch.prompt_lens, ) return attn_metadata diff --git a/vllm/v1/worker/gpu/model_states/encoder_decoder.py b/vllm/v1/worker/gpu/model_states/encoder_decoder.py index 9edda27538..f759c0b1e1 100644 --- a/vllm/v1/worker/gpu/model_states/encoder_decoder.py +++ b/vllm/v1/worker/gpu/model_states/encoder_decoder.py @@ -146,7 +146,7 @@ class EncoderDecoderModelState(ModelState): dcp_local_seq_lens=input_batch.dcp_local_seq_lens, model_specific_attn_metadata=enc_dec_attn_metadata, for_cudagraph_capture=for_capture, - rswa_prefix_lens=input_batch.rswa_prefix_lens, + rswa_prefix_lens=input_batch.prompt_lens, ) return attn_metadata diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index e08b09f189..a0f5968361 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -141,7 +141,7 @@ class MambaHybridModelState(DefaultModelState): dcp_local_seq_lens=input_batch.dcp_local_seq_lens, model_specific_attn_metadata=mamba_attn_metadata, for_cudagraph_capture=for_capture, - rswa_prefix_lens=input_batch.rswa_prefix_lens, + rswa_prefix_lens=input_batch.prompt_lens, ) def postprocess_state(