File tree Expand file tree Collapse file tree 1 file changed +4
-3
lines changed Expand file tree Collapse file tree 1 file changed +4
-3
lines changed Original file line number Diff line number Diff line change @@ -2979,13 +2979,14 @@ def _generate_exact_length_tokens(target_length: int) -> list[int]:
29792979 requests = []
29802980 token_mismatch_total = 0
29812981 for _ in range (num_prefixes ):
2982- prefix_tokens = _generate_exact_length_tokens (prefix_len )
2982+ prefix_tokens , prefix_mismatch = _generate_exact_length_tokens (prefix_len )
2983+ token_mismatch_total += prefix_mismatch
29832984
29842985 for _ in range (prompts_per_prefix ):
2985- suffix_tokens , token_mistmatch = _generate_exact_length_tokens (
2986+ suffix_tokens , suffix_mismatch = _generate_exact_length_tokens (
29862987 suffix_len
29872988 )
2988- token_mismatch_total += token_mistmatch
2989+ token_mismatch_total += suffix_mismatch
29892990 combined_tokens = prefix_tokens + suffix_tokens
29902991 prompt = tokenizer .decode (combined_tokens )
29912992 prompt_len = len (combined_tokens )
You can’t perform that action at this time.
0 commit comments