[Bugfix][MRV2] Keep input_ids on later PP stages for raw-token models (#61016)

Signed-off-by: booth-algo <kevinlauofficial01@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Kevin
2026-10-15 12:54:21 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Kevin H. Luu
parent d486219fc2
commit d12ea8e309
6 changed files with 51 additions and 5 deletions
@@ -6,11 +6,16 @@ from vllm import ModelRegistry
def register():
# Test directly passing the model
from .my_opt import MyOPTForCausalLM
from .my_opt import MyOPTForCausalLM, MyOPTRawInputTokensForCausalLM
if "MyOPTForCausalLM" not in ModelRegistry.get_supported_archs():
ModelRegistry.register_model("MyOPTForCausalLM", MyOPTForCausalLM)
if "MyOPTRawInputTokensForCausalLM" not in ModelRegistry.get_supported_archs():
ModelRegistry.register_model(
"MyOPTRawInputTokensForCausalLM", MyOPTRawInputTokensForCausalLM
)
# Test passing lazy model
if "MyGemma2Embedding" not in ModelRegistry.get_supported_archs():
ModelRegistry.register_model(
@@ -5,6 +5,7 @@
import torch
from vllm.model_executor.models.opt import OPTForCausalLM
from vllm.sequence import IntermediateTensors
class MyOPTForCausalLM(OPTForCausalLM):
@@ -15,3 +16,21 @@ class MyOPTForCausalLM(OPTForCausalLM):
logits.zero_()
logits[:, 0] += 1.0
return logits
class MyOPTRawInputTokensForCausalLM(MyOPTForCausalLM):
# Like models whose decoder layers read token ids, this model needs
# input_ids on every pipeline-parallel stage, not only the first.
requires_raw_input_tokens = True
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
assert input_ids is not None, "requires_raw_input_tokens model got no input_ids"
return super().forward(
input_ids, positions, intermediate_tensors, inputs_embeds
)