| { |
| "add_bos_token": false, |
| "add_prefix_space": false, |
| "added_tokens_decoder": { |
| "50256": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "50257": { |
| "content": "<|extratoken_1|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50258": { |
| "content": "<|extratoken_2|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50259": { |
| "content": "<|extratoken_3|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50260": { |
| "content": "<|extratoken_4|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50261": { |
| "content": "<|extratoken_5|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50262": { |
| "content": "<|extratoken_6|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50263": { |
| "content": "<|extratoken_7|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50264": { |
| "content": "<|extratoken_8|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50265": { |
| "content": "<|extratoken_9|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50266": { |
| "content": "<|extratoken_10|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50267": { |
| "content": "<|extratoken_11|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50268": { |
| "content": "<|extratoken_12|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50269": { |
| "content": "<|extratoken_13|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50270": { |
| "content": "<|extratoken_14|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50271": { |
| "content": "<|extratoken_15|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50272": { |
| "content": "<|extratoken_16|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50273": { |
| "content": "<|extratoken_17|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50274": { |
| "content": "<|extratoken_18|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50275": { |
| "content": "<|extratoken_19|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50276": { |
| "content": "<|extratoken_20|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50277": { |
| "content": "<|extratoken_21|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50278": { |
| "content": "<|extratoken_22|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50279": { |
| "content": "<|extratoken_23|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50280": { |
| "content": "<|extratoken_24|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50281": { |
| "content": "<|extratoken_25|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50282": { |
| "content": "<|extratoken_26|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50283": { |
| "content": "<|extratoken_27|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50284": { |
| "content": "<|extratoken_28|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50285": { |
| "content": "<|extratoken_29|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50286": { |
| "content": "<|extratoken_30|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50287": { |
| "content": "<|extratoken_31|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50288": { |
| "content": "<|extratoken_32|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50289": { |
| "content": "<|extratoken_33|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50290": { |
| "content": "<|extratoken_34|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50291": { |
| "content": "<|extratoken_35|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50292": { |
| "content": "<|extratoken_36|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50293": { |
| "content": "<|extratoken_37|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50294": { |
| "content": "<|extratoken_38|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50295": { |
| "content": "<|extratoken_39|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50296": { |
| "content": "<|extratoken_40|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50297": { |
| "content": "<|extratoken_41|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50298": { |
| "content": "<|extratoken_42|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50299": { |
| "content": "<|extratoken_43|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50300": { |
| "content": "<|extratoken_44|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50301": { |
| "content": "<|extratoken_45|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50302": { |
| "content": "<|extratoken_46|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50303": { |
| "content": "<|extratoken_47|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50304": { |
| "content": "<|extratoken_48|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50305": { |
| "content": "<|extratoken_49|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50306": { |
| "content": "<|extratoken_50|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50307": { |
| "content": "<|extratoken_51|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50308": { |
| "content": "<|extratoken_52|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50309": { |
| "content": "<|extratoken_53|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50310": { |
| "content": "<|extratoken_54|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50311": { |
| "content": "<|extratoken_55|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50312": { |
| "content": "<|extratoken_56|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50313": { |
| "content": "<|extratoken_57|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50314": { |
| "content": "<|extratoken_58|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50315": { |
| "content": "<|extratoken_59|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50316": { |
| "content": "<|extratoken_60|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50317": { |
| "content": "<|extratoken_61|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50318": { |
| "content": "<|extratoken_62|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50319": { |
| "content": "<|extratoken_63|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50320": { |
| "content": "<|extratoken_64|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50321": { |
| "content": "<|extratoken_65|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50322": { |
| "content": "<|extratoken_66|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50323": { |
| "content": "<|extratoken_67|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50324": { |
| "content": "<|extratoken_68|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50325": { |
| "content": "<|extratoken_69|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50326": { |
| "content": "<|extratoken_70|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50327": { |
| "content": "<|extratoken_71|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50328": { |
| "content": "<|extratoken_72|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50329": { |
| "content": "<|extratoken_73|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50330": { |
| "content": "<|extratoken_74|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50331": { |
| "content": "<|extratoken_75|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50332": { |
| "content": "<|extratoken_76|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50333": { |
| "content": "<|extratoken_77|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50334": { |
| "content": "<|extratoken_78|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50335": { |
| "content": "<|extratoken_79|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50336": { |
| "content": "<|extratoken_80|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50337": { |
| "content": "<|extratoken_81|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50338": { |
| "content": "<|extratoken_82|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50339": { |
| "content": "<|extratoken_83|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50340": { |
| "content": "<|extratoken_84|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50341": { |
| "content": "<|extratoken_85|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50342": { |
| "content": "<|extratoken_86|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50343": { |
| "content": "<|extratoken_87|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50344": { |
| "content": "<|extratoken_88|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50345": { |
| "content": "<|extratoken_89|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50346": { |
| "content": "<|extratoken_90|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50347": { |
| "content": "<|extratoken_91|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50348": { |
| "content": "<|extratoken_92|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50349": { |
| "content": "<|extratoken_93|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50350": { |
| "content": "<|extratoken_94|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50351": { |
| "content": "<|extratoken_95|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50352": { |
| "content": "<|extratoken_96|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50353": { |
| "content": "<|extratoken_97|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50354": { |
| "content": "<|extratoken_98|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50355": { |
| "content": "<|extratoken_99|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50356": { |
| "content": "<|extratoken_100|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50357": { |
| "content": "<|extratoken_101|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50358": { |
| "content": "<|extratoken_102|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50359": { |
| "content": "<|extratoken_103|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50360": { |
| "content": "<|extratoken_104|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50361": { |
| "content": "<|extratoken_105|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50362": { |
| "content": "<|extratoken_106|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50363": { |
| "content": "<|extratoken_107|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50364": { |
| "content": "<|extratoken_108|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50365": { |
| "content": "<|extratoken_109|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50366": { |
| "content": "<|extratoken_110|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50367": { |
| "content": "<|extratoken_111|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50368": { |
| "content": "<|extratoken_112|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50369": { |
| "content": "<|extratoken_113|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50370": { |
| "content": "<|extratoken_114|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50371": { |
| "content": "<|extratoken_115|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50372": { |
| "content": "<|extratoken_116|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50373": { |
| "content": "<|extratoken_117|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50374": { |
| "content": "<|extratoken_118|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50375": { |
| "content": "<|extratoken_119|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50376": { |
| "content": "<|extratoken_120|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50377": { |
| "content": "<|extratoken_121|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50378": { |
| "content": "<|extratoken_122|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50379": { |
| "content": "<|extratoken_123|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50380": { |
| "content": "<|extratoken_124|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50381": { |
| "content": "<|extratoken_125|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50382": { |
| "content": "<|extratoken_126|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50383": { |
| "content": "<|extratoken_127|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50384": { |
| "content": "<|extratoken_128|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50385": { |
| "content": "<|extratoken_129|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50386": { |
| "content": "<|extratoken_130|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50387": { |
| "content": "<|extratoken_131|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50388": { |
| "content": "<|extratoken_132|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50389": { |
| "content": "<|extratoken_133|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50390": { |
| "content": "<|extratoken_134|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50391": { |
| "content": "<|extratoken_135|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50392": { |
| "content": "<|extratoken_136|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50393": { |
| "content": "<|extratoken_137|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50394": { |
| "content": "<|extratoken_138|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50395": { |
| "content": "<|extratoken_139|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50396": { |
| "content": "<|extratoken_140|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50397": { |
| "content": "<|extratoken_141|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50398": { |
| "content": "<|extratoken_142|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "50399": { |
| "content": "<|extratoken_143|>", |
| "lstrip": false, |
| "normalized": true, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| } |
| }, |
| "bos_token": "<|endoftext|>", |
| "clean_up_tokenization_spaces": true, |
| "eos_token": "<|endoftext|>", |
| "errors": "replace", |
| "model_max_length": 2048, |
| "pad_token": null, |
| "tokenizer_class": "GPT2Tokenizer", |
| "unk_token": "<|endoftext|>" |
| } |
|
|
