[ROCm][CI] Align spec decode logprob test prefill settings (#41335)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -33,11 +33,10 @@ PROMPT = BatchLogprobsComposition.PROMPT
|
|||||||
SAMPLE_PROMPT = BatchLogprobsComposition.SAMPLE_PROMPT
|
SAMPLE_PROMPT = BatchLogprobsComposition.SAMPLE_PROMPT
|
||||||
|
|
||||||
# On ROCm, floating-point reductions in attention and GEMM kernels are
|
# On ROCm, floating-point reductions in attention and GEMM kernels are
|
||||||
# non-associative and sensitive to batch geometry. The ref LLM (no spec
|
# non-associative and sensitive to batch geometry. If the ref LLM and
|
||||||
# decode, default scheduling) and the spec-decode LLM (chunked prefill,
|
# spec-decode LLM use different scheduling or batch geometry, they can
|
||||||
# different effective batch sizes) follow different reduction orders,
|
# follow different reduction orders and produce numerically divergent
|
||||||
# producing numerically divergent logprobs that get misattributed to
|
# logprobs that get misattributed to spec-decode incorrectness.
|
||||||
# spec-decode incorrectness.
|
|
||||||
#
|
#
|
||||||
# Force LLM instances into an identical, deterministic execution
|
# Force LLM instances into an identical, deterministic execution
|
||||||
# mode so the test isolates spec-decode correctness only:
|
# mode so the test isolates spec-decode correctness only:
|
||||||
@@ -1086,18 +1085,25 @@ def test_spec_decode_logprobs(
|
|||||||
)
|
)
|
||||||
|
|
||||||
max_model_len = 256
|
max_model_len = 256
|
||||||
|
llm_kwargs = dict(
|
||||||
# Run base LLM.
|
|
||||||
ref_llm = LLM(
|
|
||||||
model=model_name,
|
|
||||||
max_logprobs=5,
|
max_logprobs=5,
|
||||||
max_model_len=max_model_len,
|
max_model_len=max_model_len,
|
||||||
seed=42,
|
seed=42,
|
||||||
logprobs_mode=logprobs_mode,
|
logprobs_mode=logprobs_mode,
|
||||||
gpu_memory_utilization=0.4,
|
gpu_memory_utilization=0.4,
|
||||||
|
# Force the same prefill chunking for both the base model and
|
||||||
|
# spec decode model so the comparison isolates spec decode.
|
||||||
|
enable_chunked_prefill=True,
|
||||||
|
max_num_batched_tokens=32,
|
||||||
enable_prefix_caching=False,
|
enable_prefix_caching=False,
|
||||||
**ROCM_DETERMINISM_KWARGS,
|
**ROCM_DETERMINISM_KWARGS,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Run base LLM.
|
||||||
|
ref_llm = LLM(
|
||||||
|
model=model_name,
|
||||||
|
**llm_kwargs,
|
||||||
|
)
|
||||||
ref_results = ref_llm.generate(
|
ref_results = ref_llm.generate(
|
||||||
[prompt, prompt], [sampling_params, penalty_sampling_params]
|
[prompt, prompt], [sampling_params, penalty_sampling_params]
|
||||||
)
|
)
|
||||||
@@ -1117,16 +1123,7 @@ def test_spec_decode_logprobs(
|
|||||||
spec_llm = LLM(
|
spec_llm = LLM(
|
||||||
model_name,
|
model_name,
|
||||||
speculative_config=spec_config_with_len,
|
speculative_config=spec_config_with_len,
|
||||||
max_logprobs=5,
|
**llm_kwargs,
|
||||||
max_model_len=max_model_len,
|
|
||||||
seed=42,
|
|
||||||
logprobs_mode=logprobs_mode,
|
|
||||||
gpu_memory_utilization=0.4,
|
|
||||||
# Force prefill chunking
|
|
||||||
enable_chunked_prefill=True,
|
|
||||||
max_num_batched_tokens=32,
|
|
||||||
enable_prefix_caching=False,
|
|
||||||
**ROCM_DETERMINISM_KWARGS,
|
|
||||||
)
|
)
|
||||||
spec_results = spec_llm.generate(
|
spec_results = spec_llm.generate(
|
||||||
[prompt, prompt], [sampling_params, penalty_sampling_params]
|
[prompt, prompt], [sampling_params, penalty_sampling_params]
|
||||||
|
|||||||
Reference in New Issue
Block a user