File tree Expand file tree Collapse file tree 1 file changed +6
-4
lines changed Expand file tree Collapse file tree 1 file changed +6
-4
lines changed Original file line number Diff line number Diff line change @@ -194,19 +194,21 @@ def __iter__(
194194 }
195195
196196 def _create_prompt (
197- self , prompt_tokens : int , start_index : int , unique_prefix : Optional [int ] = None
197+ self ,
198+ prompt_tokens : int ,
199+ start_index : int ,
200+ unique_prefix : Optional [int ] = None , # noqa: ARG002
198201 ) -> list [int ]:
199202 if prompt_tokens <= 0 :
200203 return []
201204
202205 left = start_index
203206 right = start_index + 4 * prompt_tokens
204- start_tokens = [unique_prefix ] if unique_prefix else []
205207
206208 while left < right :
207209 mid = (left + right ) // 2
208210 test_prompt = self .text_creator .create_text (start_index , mid - start_index )
209- test_tokens = start_tokens + self .processor .encode (test_prompt )
211+ test_tokens = self .processor .encode (test_prompt )
210212
211213 if len (test_tokens ) == prompt_tokens :
212214 return test_tokens
@@ -216,7 +218,7 @@ def _create_prompt(
216218 right = mid
217219
218220 final_text = self .text_creator .create_text (start_index , left - start_index )
219- return start_tokens + self .processor .encode (final_text )
221+ return self .processor .encode (final_text )
220222
221223
222224class SyntheticDatasetCreator (DatasetCreator ):
You can’t perform that action at this time.
0 commit comments