[Bugfix] Fix weight loading for Chameleon when TP>1 (#7410)
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import sys
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
from openai import OpenAI, OpenAIError
|
||||
@@ -17,8 +18,11 @@ assert chatml_jinja_path.exists()
|
||||
|
||||
class MyOPTForCausalLM(OPTForCausalLM):
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata) -> torch.Tensor:
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> Optional[torch.Tensor]:
|
||||
# this dummy model always predicts the first token
|
||||
logits = super().compute_logits(hidden_states, sampling_metadata)
|
||||
logits.zero_()
|
||||
|
||||
Reference in New Issue
Block a user