[BugFix][Graph] fix: handle empty sym_shape_indices in PiecewiseBackend. (#39395)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -222,3 +222,47 @@ def test_model_specialization_with_evaluate_guards(
|
||||
torch.randn(1, 10).cuda(),
|
||||
is_01_specialization=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||
def test_piecewise_backend_empty_sym_shape_indices():
|
||||
"""Test that PiecewiseBackend handles empty sym_shape_indices correctly.
|
||||
|
||||
When all inputs have static shapes (no torch.SymInt), sym_shape_indices
|
||||
will be empty. The fix in PiecewiseBackend.__call__ handles this case
|
||||
by using the first compiled range_entry.
|
||||
"""
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Use small max_model_len and max_num_batched_tokens to encourage
|
||||
# static shape compilation with empty sym_shape_indices
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
max_num_batched_tokens=1,
|
||||
compilation_config={
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"dynamic_shapes_config": {
|
||||
"type": DynamicShapesType.BACKED.value,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
sampling_params = SamplingParams(temperature=0, top_p=0.95, max_tokens=10)
|
||||
|
||||
# Generate with static shape inputs
|
||||
output = llm.generate("Hello, my name is", sampling_params=sampling_params)
|
||||
result = output[0].outputs[0].text
|
||||
assert len(result) > 0, "Should generate non-empty output"
|
||||
|
||||
# Generate again to verify compilation works with empty sym_shape_indices
|
||||
output = llm.generate("The capital of France is", sampling_params=sampling_params)
|
||||
result = output[0].outputs[0].text
|
||||
assert len(result) > 0, "Should generate non-empty output on second run"
|
||||
|
||||
del llm
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
Reference in New Issue
Block a user