[Model Runner V2] batch-sharded sample (#50465)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-08-24 15:49:57 -07:00
committed by GitHub
parent 0d7d5ed0b2
commit d154d90d6c
19 changed files with 1781 additions and 22 deletions
+7
View File
@@ -155,7 +155,14 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
hidden_states: torch.Tensor,
lm_head: VocabParallelEmbedding,
embedding_bias: torch.Tensor | None = None,
skip_gather: bool = False,
) -> torch.Tensor | None:
# The LoRA delta is accumulated into the full gathered logits, so the
# TP gather cannot be skipped here.
if skip_gather:
raise NotImplementedError(
"Skipping the logits TP gather is not supported with an lm_head LoRA."
)
# Get the logits for the next tokens.
if hasattr(lm_head, "base_layer"):
actual_lm_head = lm_head.base_layer