fix token

Signed-off-by: guangli.bao <[email protected]>

Author: tukwila

src/guidellm/dataset/synthetic.py

@@ -204,11 +204,12 @@ def _create_prompt(
 
         left = start_index
         right = start_index + 4 * prompt_tokens
+        start_tokens = []
 
         while left < right:
             mid = (left + right) // 2
             test_prompt = self.text_creator.create_text(start_index, mid - start_index)
-            test_tokens = self.processor.encode(test_prompt)
+            test_tokens = start_tokens + self.processor.encode(test_prompt)
 
             if len(test_tokens) == prompt_tokens:
                 return test_tokens
@@ -218,7 +219,7 @@ def _create_prompt(
                 right = mid
 
         final_text = self.text_creator.create_text(start_index, left - start_index)
-        return self.processor.encode(final_text)
+        return start_tokens + self.processor.encode(final_text)
 
 
 class SyntheticDatasetCreator(DatasetCreator):

tests/unit/dataset/test_synthetic.py

@@ -415,7 +415,7 @@ def test_create_prompt_method(
 
         # Test normal case
         result = generator._create_prompt(5, 0, 42)
-        assert result == [42, 1, 2, 3]
+        assert result == [1, 2, 3]
 
         # Test zero tokens
         result = generator._create_prompt(0, 0, 42)