Allow AsyncLLMEngine.generate to target a specific DP rank (#19102)

Signed-off-by: Jon Swenson <jmswen@gmail.com>
This commit is contained in:
jmswen
2025-06-04 08:26:47 -07:00
committed by GitHub
parent 8f4ffbd373
commit c8dcc15921
10 changed files with 97 additions and 5 deletions
+2 -1
View File
@@ -70,7 +70,8 @@ def _run_incremental_decode(tokenizer,
None,
0.0,
None,
cache_salt=None)
cache_salt=None,
data_parallel_rank=None)
if fast is None:
detokenizer = IncrementalDetokenizer.from_new_request(
+1
View File
@@ -42,6 +42,7 @@ def make_request() -> EngineCoreRequest:
arrival_time=time.time(),
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
)
@@ -56,6 +56,7 @@ def make_request(
arrival_time=time.time(),
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
)
+5
View File
@@ -59,6 +59,7 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind,
eos_token_id=None,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=SamplingParams(
skip_special_tokens=False,
spaces_between_special_tokens=False,
@@ -406,6 +407,7 @@ def test_logprobs_processor(request_output_kind: RequestOutputKind,
eos_token_id=None,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=SamplingParams(
skip_special_tokens=False,
spaces_between_special_tokens=False,
@@ -569,6 +571,7 @@ def test_stop_token(include_stop_str_in_output: bool,
eos_token_id=eos_token_id,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=SamplingParams(
skip_special_tokens=False,
spaces_between_special_tokens=False,
@@ -666,6 +669,7 @@ def test_stop_string(include_stop_str_in_output: bool,
eos_token_id=None,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=SamplingParams(
skip_special_tokens=False,
spaces_between_special_tokens=False,
@@ -780,6 +784,7 @@ def test_iteration_stats(dummy_test_vectors):
eos_token_id=None,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=SamplingParams(),
) for idx, prompt_tokens in enumerate(dummy_test_vectors.prompt_tokens)
]