[Perf] Avoid more GPU<->CPU syncs on the model execution path (#51738)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -69,6 +69,7 @@ from vllm.multimodal.processing.processor import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -2039,7 +2040,20 @@ class Gemma4ForConditionalGeneration(
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.language_model.compute_logits(hidden_states)
|
||||
if logits is not None and self._suppress_token_ids:
|
||||
logits[:, self._suppress_token_ids] = -float("inf")
|
||||
# Cache a per-device index tensor for the (static) suppressed-token
|
||||
# set and use `index_fill_`, so neither the Python-list indices nor
|
||||
# the scalar fill value take a host roundtrip per call.
|
||||
cache = getattr(self, "_suppress_token_ids_cache", None)
|
||||
if cache is None:
|
||||
cache = {}
|
||||
self._suppress_token_ids_cache = cache
|
||||
suppress_idx = cache.get(logits.device)
|
||||
if suppress_idx is None:
|
||||
suppress_idx = async_tensor_h2d(
|
||||
self._suppress_token_ids, dtype=torch.long, device=logits.device
|
||||
)
|
||||
cache[logits.device] = suppress_idx
|
||||
logits.index_fill_(1, suppress_idx, -float("inf"))
|
||||
return logits
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
Reference in New Issue
Block a user