[Bugfix][MRV2] Keep input_ids on later PP stages for raw-token models (#61016)
Signed-off-by: booth-algo <kevinlauofficial01@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kevin H. Luu
parent
d486219fc2
commit
d12ea8e309
@@ -6,11 +6,16 @@ from vllm import ModelRegistry
|
||||
|
||||
def register():
|
||||
# Test directly passing the model
|
||||
from .my_opt import MyOPTForCausalLM
|
||||
from .my_opt import MyOPTForCausalLM, MyOPTRawInputTokensForCausalLM
|
||||
|
||||
if "MyOPTForCausalLM" not in ModelRegistry.get_supported_archs():
|
||||
ModelRegistry.register_model("MyOPTForCausalLM", MyOPTForCausalLM)
|
||||
|
||||
if "MyOPTRawInputTokensForCausalLM" not in ModelRegistry.get_supported_archs():
|
||||
ModelRegistry.register_model(
|
||||
"MyOPTRawInputTokensForCausalLM", MyOPTRawInputTokensForCausalLM
|
||||
)
|
||||
|
||||
# Test passing lazy model
|
||||
if "MyGemma2Embedding" not in ModelRegistry.get_supported_archs():
|
||||
ModelRegistry.register_model(
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.opt import OPTForCausalLM
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
|
||||
class MyOPTForCausalLM(OPTForCausalLM):
|
||||
@@ -15,3 +16,21 @@ class MyOPTForCausalLM(OPTForCausalLM):
|
||||
logits.zero_()
|
||||
logits[:, 0] += 1.0
|
||||
return logits
|
||||
|
||||
|
||||
class MyOPTRawInputTokensForCausalLM(MyOPTForCausalLM):
|
||||
# Like models whose decoder layers read token ids, this model needs
|
||||
# input_ids on every pipeline-parallel stage, not only the first.
|
||||
requires_raw_input_tokens = True
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
assert input_ids is not None, "requires_raw_input_tokens model got no input_ids"
|
||||
return super().forward(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user