We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 7107502 commit 4e04eceCopy full SHA for 4e04ece
vllm/v1/worker/gpu_model_runner.py
@@ -1721,6 +1721,10 @@ def _dummy_sampler_run(
1721
self,
1722
hidden_states: torch.Tensor,
1723
) -> torch.Tensor:
1724
+ # The dummy hidden states may contain special values,
1725
+ # like `inf` or `nan`.
1726
+ # To avoid breaking the sampler, we use a random tensor here instead.
1727
+ hidden_states = torch.rand_like(hidden_states)
1728
1729
logits = self.model.compute_logits(hidden_states, None)
1730
num_reqs = logits.size(0)
0 commit comments