[Model Runner V2] Do not initialize sampler for non-last PP ranks (#36824)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-03-12 03:55:28 +00:00
committed by GitHub
parent 2ef69456f5
commit 2f8b4ce0c0
3 changed files with 75 additions and 50 deletions
+4 -3
View File
@@ -19,10 +19,11 @@ class PoolingRunner:
def __init__(self, model: nn.Module):
self.model = cast(VllmModelForPooling, model)
def get_supported_pooling_tasks(self) -> list[PoolingTask]:
if not is_pooling_model(self.model):
@staticmethod
def get_supported_tasks(model: nn.Module) -> list[PoolingTask]:
if not is_pooling_model(model):
return []
assert "embed" in self.model.pooler.get_supported_tasks()
assert "embed" in model.pooler.get_supported_tasks()
return ["embed"]
def pool(