File tree Expand file tree Collapse file tree 1 file changed +2
-3
lines changed Expand file tree Collapse file tree 1 file changed +2
-3
lines changed Original file line number Diff line number Diff line change @@ -204,12 +204,11 @@ def _create_prompt(
204204
205205 left = start_index
206206 right = start_index + 4 * prompt_tokens
207- start_tokens = list [int ]
208207
209208 while left < right :
210209 mid = (left + right ) // 2
211210 test_prompt = self .text_creator .create_text (start_index , mid - start_index )
212- test_tokens = start_tokens + self .processor .encode (test_prompt )
211+ test_tokens = self .processor .encode (test_prompt )
213212
214213 if len (test_tokens ) == prompt_tokens :
215214 return test_tokens
@@ -219,7 +218,7 @@ def _create_prompt(
219218 right = mid
220219
221220 final_text = self .text_creator .create_text (start_index , left - start_index )
222- return start_tokens + self .processor .encode (final_text )
221+ return self .processor .encode (final_text )
223222
224223
225224class SyntheticDatasetCreator (DatasetCreator ):
You can’t perform that action at this time.
0 commit comments