[Bugfix][MRV2] Keep input_ids on later PP stages for raw-token models (#61016)
Signed-off-by: booth-algo <kevinlauofficial01@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kevin H. Luu
parent
d486219fc2
commit
d12ea8e309
@@ -33,6 +33,7 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -690,7 +691,8 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
}
|
||||
if not self.is_first_pp_rank:
|
||||
# Update for non-first PP ranks.
|
||||
model_inputs["input_ids"] = None
|
||||
if not requires_raw_input_tokens(model):
|
||||
model_inputs["input_ids"] = None
|
||||
model_inputs["inputs_embeds"] = None
|
||||
assert intermediate_tensors is not None
|
||||
model_inputs["intermediate_tensors"] = intermediate_tensors[:num_tokens]
|
||||
|
||||
@@ -1970,7 +1970,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
}
|
||||
if not self.is_first_pp_rank:
|
||||
# Update for non-first PP ranks.
|
||||
model_inputs["input_ids"] = None
|
||||
if not requires_raw_input_tokens(self.model):
|
||||
model_inputs["input_ids"] = None
|
||||
model_inputs["inputs_embeds"] = None
|
||||
|
||||
# Prepare the intermediate tensors.
|
||||
|
||||
Reference in New Issue
Block a user