File tree Expand file tree Collapse file tree 1 file changed +3
-3
lines changed Expand file tree Collapse file tree 1 file changed +3
-3
lines changed Original file line number Diff line number Diff line change @@ -201,9 +201,9 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool:
201
201
if special_last != special_eos :
202
202
logger .warning (f'Unknown trailing special token { special_last !r} in TemplateProcessing<single>' )
203
203
if tmpl_pair :
204
- seq_start = 1 if tmpl_pair [0 ].get ('SpecialToken' , {}).get ('id' ) == special_first else 0
205
- seq_stop = - 1 if tmpl_pair [- 1 ].get ('SpecialToken' , {}).get ('id' ) == special_last else None
206
- if seq_start == 0 or seq_stop is None :
204
+ seq_start = 1 if special_first and tmpl_pair [0 ].get ('SpecialToken' , {}).get ('id' ) == special_first else 0
205
+ seq_stop = - 1 if special_last and tmpl_pair [- 1 ].get ('SpecialToken' , {}).get ('id' ) == special_last else None
206
+ if ( special_first and seq_start == 0 ) or ( special_last and seq_stop is None ) :
207
207
logger .warning ('TemplateProcessing<single> leading/trailing special tokens do not match TemplateProcessing<pair>' )
208
208
if tmpl_pair := tmpl_pair [slice (seq_start , seq_stop )]:
209
209
tmpl_a = tmpl_pair [0 ].get ('Sequence' , {}).get ('id' )
You can’t perform that action at this time.
0 commit comments