[Hardware][Intel] OpenVINO vLLM backend (#5379)
This commit is contained in:
@@ -679,7 +679,7 @@ def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
Returns:
|
||||
torch.Tensor: 1D tensor of shape (N,) where N is the no. of tokens.
|
||||
Each element in the returned tensor represents the rank
|
||||
Each element in the returned tensor represents the rank
|
||||
of the chosen token in the input logprob tensor.
|
||||
"""
|
||||
vals = x[torch.arange(0, len(x), device=x.device, dtype=indices.dtype),
|
||||
@@ -965,7 +965,7 @@ def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
|
||||
distribution.
|
||||
- Greedy sampling performs `argmax` to obtain the token with the
|
||||
highest likelihood.
|
||||
|
||||
|
||||
Ignoring greedy sampling for a moment, we find that the computed probability
|
||||
distribution has the following property: we can sample from it independently
|
||||
and find that the token sampled by the Sampler has a frequency corresponding
|
||||
|
||||
Reference in New Issue
Block a user