[Frontend] Parallelize preprocessing within the same request for pooling models online serving. (#49153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -161,7 +161,9 @@ def _make_pooling_serving(lora_name: str) -> _ConcretePoolingServing:
|
||||
return serving
|
||||
|
||||
|
||||
def _make_pooling_ctx(model_name: str) -> PoolingServeContext:
|
||||
def _make_pooling_ctx(
|
||||
model_name: str, serving: PoolingBaseServing
|
||||
) -> PoolingServeContext:
|
||||
mock_request = MagicMock()
|
||||
mock_request.model = model_name
|
||||
return PoolingServeContext(
|
||||
@@ -169,6 +171,9 @@ def _make_pooling_ctx(model_name: str) -> PoolingServeContext:
|
||||
model_name=MODEL_NAME,
|
||||
request_id="test-id",
|
||||
pooling_params=PoolingParams(),
|
||||
lora_request=serving._maybe_get_adapters(mock_request),
|
||||
priorities=0,
|
||||
prompt_extras=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -176,9 +181,7 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
|
||||
"""LoRA adapter name must populate ctx.lora_request without raising."""
|
||||
lora_name = "bot-embed-lora"
|
||||
serving = _make_pooling_serving(lora_name)
|
||||
ctx = _make_pooling_ctx(lora_name)
|
||||
|
||||
ctx.lora_request = serving._maybe_get_adapters(ctx.request)
|
||||
ctx = _make_pooling_ctx(lora_name, serving)
|
||||
|
||||
assert ctx.lora_request is not None
|
||||
assert ctx.lora_request.lora_name == lora_name
|
||||
@@ -187,7 +190,6 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
|
||||
def test_pooling_maybe_get_adapters_unknown_model_raises():
|
||||
"""An unrecognised model name must still raise VLLMNotFoundError."""
|
||||
serving = _make_pooling_serving("some-lora")
|
||||
ctx = _make_pooling_ctx("unknown-model")
|
||||
|
||||
with pytest.raises(VLLMNotFoundError):
|
||||
serving._maybe_get_adapters(ctx.request)
|
||||
_make_pooling_ctx("unknown-model", serving)
|
||||
|
||||
Reference in New Issue
Block a user