[V0 Deprecation] Remove V0 sampling metadata (#25345)
Signed-off-by: Woosuk Kwon <woosuk@thinkingmachines.ai>
This commit is contained in:
@@ -239,7 +239,7 @@ class EagleProposer:
|
||||
else:
|
||||
last_hidden_states, hidden_states = ret_hidden_states
|
||||
sample_hidden_states = last_hidden_states[last_token_indices]
|
||||
logits = self.model.compute_logits(sample_hidden_states, None)
|
||||
logits = self.model.compute_logits(sample_hidden_states)
|
||||
|
||||
# Early exit if there is only one draft token to be generated.
|
||||
if self.num_speculative_tokens == 1:
|
||||
@@ -367,8 +367,7 @@ class EagleProposer:
|
||||
else:
|
||||
last_hidden_states, hidden_states = ret_hidden_states
|
||||
hidden_states = hidden_states[:batch_size]
|
||||
logits = self.model.compute_logits(last_hidden_states[:batch_size],
|
||||
None)
|
||||
logits = self.model.compute_logits(last_hidden_states[:batch_size])
|
||||
draft_token_ids = logits.argmax(dim=-1)
|
||||
draft_token_ids_list.append(draft_token_ids)
|
||||
|
||||
@@ -678,9 +677,7 @@ class EagleProposer:
|
||||
# Get the output logits for the draft tokens.
|
||||
logits = self.model.compute_logits(
|
||||
draft_last_hidden_states.reshape(batch_size * level_num_drafts,
|
||||
-1),
|
||||
None,
|
||||
)
|
||||
-1))
|
||||
|
||||
# Sample a draft token for each child at the next tree level.
|
||||
num_children = self.child_drafts_per_level[level + 1]
|
||||
|
||||
@@ -41,7 +41,7 @@ class MedusaProposer:
|
||||
) -> list[list[int]]:
|
||||
# Generate blocks and compute logits
|
||||
blocks = self.model(target_hidden_states)
|
||||
logits = self.model.compute_logits(blocks, None)
|
||||
logits = self.model.compute_logits(blocks)
|
||||
|
||||
# Get draft tokens and transpose the result
|
||||
# TODO(woosuk): OPTIMIZATION: Return GPU tensor without GPU-CPU
|
||||
|
||||
@@ -2240,7 +2240,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
return output
|
||||
|
||||
sample_hidden_states = hidden_states[logits_indices]
|
||||
logits = self.model.compute_logits(sample_hidden_states, None)
|
||||
logits = self.model.compute_logits(sample_hidden_states)
|
||||
else:
|
||||
# Rare case.
|
||||
assert not self.is_pooling_model
|
||||
@@ -2258,8 +2258,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
logits = None
|
||||
else:
|
||||
sample_hidden_states = hidden_states[logits_indices]
|
||||
logits = self.model.compute_logits(sample_hidden_states,
|
||||
None)
|
||||
logits = self.model.compute_logits(sample_hidden_states)
|
||||
|
||||
model_output_broadcast_data = {}
|
||||
if logits is not None:
|
||||
@@ -2706,7 +2705,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
req_idx = self.input_batch.req_id_to_index[req_id]
|
||||
offset = self.query_start_loc.np[req_idx].item()
|
||||
prompt_hidden_states = hidden_states[offset:offset + num_logits]
|
||||
logits = self.model.compute_logits(prompt_hidden_states, None)
|
||||
logits = self.model.compute_logits(prompt_hidden_states)
|
||||
|
||||
# Get the "target" tokens for each index. For prompt at index i,
|
||||
# the token at prompt index i+1 is the "sampled" token we want
|
||||
@@ -3105,7 +3104,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
# To avoid breaking the sampler, we use a random tensor here instead.
|
||||
hidden_states = torch.rand_like(hidden_states)
|
||||
|
||||
logits = self.model.compute_logits(hidden_states, None)
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
num_reqs = logits.size(0)
|
||||
|
||||
dummy_tensors = lambda v: torch.full(
|
||||
|
||||
@@ -1692,7 +1692,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
@torch.compile(backend="openxla", fullgraph=True, dynamic=False)
|
||||
def compute_logits(self,
|
||||
sample_hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.compute_logits(sample_hidden_states, None)
|
||||
return self.model.compute_logits(sample_hidden_states)
|
||||
|
||||
# TODO: Under SPMD mode, sample_from_logits has correctness issue.
|
||||
# Re-enable the torch.compile once the issue is fixed in torchxla.
|
||||
|
||||
Reference in New Issue
Block a user