[Perf] Avoid more GPU<->CPU syncs on the model execution path (#51738)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-11 19:30:34 -07:00
committed by GitHub
parent 23f25aac14
commit f97e502969
15 changed files with 116 additions and 56 deletions
+15 -1
View File
@@ -69,6 +69,7 @@ from vllm.multimodal.processing.processor import (
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -2039,7 +2040,20 @@ class Gemma4ForConditionalGeneration(
) -> torch.Tensor | None:
logits = self.language_model.compute_logits(hidden_states)
if logits is not None and self._suppress_token_ids:
logits[:, self._suppress_token_ids] = -float("inf")
# Cache a per-device index tensor for the (static) suppressed-token
# set and use `index_fill_`, so neither the Python-list indices nor
# the scalar fill value take a host roundtrip per call.
cache = getattr(self, "_suppress_token_ids_cache", None)
if cache is None:
cache = {}
self._suppress_token_ids_cache = cache
suppress_idx = cache.get(logits.device)
if suppress_idx is None:
suppress_idx = async_tensor_h2d(
self._suppress_token_ids, dtype=torch.long, device=logits.device
)
cache[logits.device] = suppress_idx
logits.index_fill_(1, suppress_idx, -float("inf"))
return logits
# ------------------------------------------------------------------ #